STINGS MIGRATIONS SASHI and also add some to scope one

luau/twshell/common/db.luau

@@ -5,39 +5,39 @@
 --- A wrapper around a sqlx PgPool that provides methods to execute queries and fetch results, with values mapped through the DbValueMapper
 export type Db = {
     --- Returns if we support the given type for casting
-    supports: (type: string) -> boolean,
+    supports: (self: Db, type: string) -> boolean,
     --- Casts a Lua value into an OpaqueValue of the specified type.
-    cast: (value: any, type: string) -> DbValue,
+    cast: (self: Db, value: any, type: string) -> DbValue,
     --- Executes a query and returns number of affected rows
-    execute: (sql: string, params: {DbValue}) -> number,
+    execute: (self: Db, sql: string, params: {DbValue}) -> number,
     --- Executes a query and returns all results
-    fetchall: (sql: string, params: {DbValue}) -> {PgRow},
+    fetchall: (self: Db, sql: string, params: {DbValue}) -> {PgRow},
     --- Begins a transaction and returns a DbTx object that can be used to execute queries within the transaction
-    begin: () -> DbTx,
+    begin: (self: Db) -> DbTx,
 }
 
 --- A wrapper around a sqlx Transaction that provides methods to execute queries and fetch results, with values mapped through the DbValueMapper
 export type DbTx = {
     --- Executes a query and returns number of affected rows
-    execute: (sql: string, params: {DbValue}) -> number,
+    execute: (self: DbTx, sql: string, params: {DbValue}) -> number,
     --- Executes a query and returns all results
-    fetchall: (sql: string, params: {DbValue}) -> {PgRow},
+    fetchall: (self: DbTx, sql: string, params: {DbValue}) -> {PgRow},
     --- Commits the transaction
-    commit: () -> (),
+    commit: (self: DbTx) -> (),
     --- Rolls back the transaction
-    rollback: () -> (),
+    rollback: (self: DbTx) -> (),
 }
 
 export type PgRow = {
     --- Gets the value of a column by index (1-based) and type. Does not immediately convert the value to a Lua type, but instead returns a Value object that can be converted on demand.
-    get: (index: number, type: string) -> DbValue,
+    get: (self: PgRow, index: number, type: string) -> DbValue,
 }
 
 export type DbValue = {
     --- Gets the underlying type 
-    type: () -> string,
+    type: (self: DbValue) -> string,
     --- Takes out the underlying value
-    take: () -> any,
+    take: (self: DbValue) -> any,
 }
 
 export type Id = {

luau/twshell/migrations/stings.luau

@@ -0,0 +1,154 @@
+local datetime = require"@antiraid/datetime"
+local typesext = require"@antiraid/typesext"
+local db = require"../common/db"
+
+local OLD_SCOPE = "builtins.stings"
+local KEX_SCOPE = "__kexsbuiltins.stings"
+local FAR_FUTURE_SECS = 100 * 365 * 24 * 60 * 60
+
+return {
+    up = function(db: db.Db)
+        print("[stings] merging old sting data into kex records (pre-kv_scope_unnest)")
+
+        local tx = db:begin()
+
+        local ok, err = pcall(function()
+            local sting_rows = tx:fetchall(
+                "SELECT owner_id, owner_type, key, scopes, value FROM tenant_kv WHERE $1 = ANY(scopes)",
+                { db:cast(OLD_SCOPE, "string") }
+            )
+
+            type StingFields = { owner_id: string, owner_type: string, key: string, userid: string, modid: string?, reason: string, stings: number }
+            local old_stings: {[string]: StingFields} = {}
+            local missing_userid = 0
+            for _, row in sting_rows do
+                local owner_id = row:get(0, "string"):take() :: string
+                local owner_type = row:get(1, "string"):take() :: string
+                local key = row:get(2, "string"):take() :: string
+                local scopes = row:get(3, "{string}"):take() :: {string}
+                local value = row:get(4, "custom@khronosvalue"):take() :: any
+
+                local userid: string? = nil
+                for _, s in scopes do
+                    if s ~= OLD_SCOPE then
+                        userid = s
+                        break
+                    end
+                end
+
+                if not userid then
+                    print(string.format("[stings] sting key=%s owner=%s/%s has no userid in scopes %s; skipping", key, owner_type, owner_id, table.concat(scopes, ",")))
+                    missing_userid += 1
+                    continue
+                end
+
+                local composite = owner_id .. "|" .. owner_type .. "|" .. key
+                old_stings[composite] = {
+                    owner_id = owner_id,
+                    owner_type = owner_type,
+                    key = key,
+                    userid = userid,
+                    modid = value.modId or value.modid,
+                    reason = value.reason or "",
+                    stings = value.stings or 1,
+                }
+            end
+            print(string.format("[stings] loaded %d old sting rows (%d skipped)", #sting_rows - missing_userid, missing_userid))
+
+            local kex_rows = tx:fetchall(
+                "SELECT id, owner_id, owner_type, key, value FROM tenant_kv WHERE $1 = ANY(scopes)",
+                { db:cast(KEX_SCOPE, "string") }
+            )
+            print(string.format("[stings] found %d existing kex rows", #kex_rows))
+
+            local merged = 0
+            local matched: {[string]: boolean} = {}
+
+            for _, row in kex_rows do
+                local id = row:get(0, "string"):take() :: string
+                local owner_id = row:get(1, "string"):take() :: string
+                local owner_type = row:get(2, "string"):take() :: string
+                local key = row:get(3, "string"):take() :: string
+                local value = row:get(4, "custom@khronosvalue"):take() :: any
+
+                local composite = owner_id .. "|" .. owner_type .. "|" .. key
+                local sting = old_stings[composite]
+                if not sting then
+                    print(string.format("[stings] kex key=%s owner=%s/%s has no matching sting data; leaving as-is", key, owner_type, owner_id))
+                    continue
+                end
+                matched[composite] = true
+
+                local new_value = {
+                    expiresat = value.expiresat,
+                    data = {
+                        userid = sting.userid,
+                        modid = sting.modid,
+                        reason = sting.reason,
+                        stings = sting.stings,
+                    },
+                }
+
+                tx:execute(
+                    "UPDATE tenant_kv SET value = $1, last_updated_at = NOW() WHERE id = $2",
+                    {
+                        db:cast(new_value, "custom@khronosvalue"),
+                        db:cast(id, "string"),
+                    }
+                )
+                merged += 1
+            end
+            print(string.format("[stings] merged %d kex rows with matching sting data", merged))
+
+            local inserted = 0
+            local far_future = datetime.UTC:now() + datetime.timedelta_seconds(FAR_FUTURE_SECS)
+            for composite, sting in old_stings do
+                if matched[composite] then continue end
+
+                local new_id = typesext.randstring(64)
+                local new_value = {
+                    expiresat = far_future,
+                    data = {
+                        userid = sting.userid,
+                        modid = sting.modid,
+                        reason = sting.reason,
+                        stings = sting.stings,
+                    },
+                }
+
+                tx:execute(
+                    [[INSERT INTO tenant_kv (id, owner_id, owner_type, key, value, scopes)
+                      VALUES ($1, $2, $3, $4, $5, $6)
+                      ON CONFLICT (owner_id, owner_type, key, scopes) DO UPDATE SET value = EXCLUDED.value, last_updated_at = NOW()]],
+                    {
+                        db:cast(new_id, "string"),
+                        db:cast(sting.owner_id, "string"),
+                        db:cast(sting.owner_type, "string"),
+                        db:cast(sting.key, "string"),
+                        db:cast(new_value, "custom@khronosvalue"),
+                        db:cast({KEX_SCOPE}, "{string}"),
+                    }
+                )
+                inserted += 1
+            end
+            if inserted > 0 then
+                print(string.format("[stings] inserted %d new kex rows (no prior expiry, assigned far-future)", inserted))
+            end
+
+            local deleted = tx:execute(
+                "DELETE FROM tenant_kv WHERE $1 = ANY(scopes)",
+                { db:cast(OLD_SCOPE, "string") }
+            )
+            print(string.format("[stings] deleted %d rows from scope='%s'", deleted, OLD_SCOPE))
+        end)
+
+        if not ok then
+            print("[stings] migration failed: " .. tostring(err))
+            tx:rollback()
+            error(err)
+        end
+
+        tx:commit()
+        print("[stings] committed")
+    end,
+}

src/migrations/kv_scope_unnest.rs

@@ -9,35 +9,39 @@ pub static MIGRATION: Migration = Migration {
 
             let stmts = [
                 "ALTER TABLE tenant_kv ADD COLUMN scopes_2 TEXT;",
-                "ALTER TABLE tenant_kv DROP CONSTRAINT kv_unique_entry;"
+                "ALTER TABLE tenant_kv DROP CONSTRAINT kv_unique_entry;",
             ];
 
             for stmt in stmts.iter() {
-                sqlx::query(stmt)
-                    .execute(&mut *tx)
-                    .await?;
+                sqlx::query(stmt).execute(&mut *tx).await?;
             }
 
             #[derive(sqlx::FromRow, Debug)]
             pub struct TenantKv {
                 owner_id: String,
                 owner_type: String,
                 key: String,
-                scopes: Vec<String>
+                scopes: Vec<String>,
             }
 
-            let rows = sqlx::query_as::<_, TenantKv>("SELECT owner_id, owner_type, key, scopes FROM tenant_kv")
+            let rows = sqlx::query_as::<_, TenantKv>(
+                "SELECT owner_id, owner_type, key, scopes FROM tenant_kv",
+            )
             .fetch_all(&mut *tx)
             .await?;
 
             for row in rows {
                 if row.scopes.is_empty() {
                     panic!("No scopes found for {row:?}!");
                 }
-                if row.scopes.len() > 1 {
-                    panic!("Multiple scopes found for {row:?}!");
-                }
-                let scope = row.scopes[0].clone();
+                let scope = row
+                    .scopes
+                    .iter()
+                    .find(|elem| elem.chars().any(|c| !c.is_numeric()));
+
+                let Some(scope) = scope else {
+                    panic!("No non-numeric scopes found for {row:?}!");
+                };
 
                 sqlx::query("UPDATE tenant_kv SET scopes_2 = $1 WHERE owner_id = $2 and owner_type = $3 and key = $4")
                 .bind(scope)
@@ -56,9 +60,7 @@ pub static MIGRATION: Migration = Migration {
             ];
 
             for stmt in stmts.iter() {
-                sqlx::query(stmt)
-                    .execute(&mut *tx)
-                    .await?;
+                sqlx::query(stmt).execute(&mut *tx).await?;
             }
 
             tx.commit().await?;

src/migrations/mod.rs

@@ -22,8 +22,9 @@ use rust_embed::Embed;
 
 use crate::{master::mainthread::{RunInThreadFn, run_in_thread}, worker::builtins::TemplatingTypes};
 
-pub const RUST_MIGRATIONS: [Migration; 11] = [
-    // Do not change order of migrations without verifying dependencies
+// Rust migrations that must run before the luau migrations in `luau/twshell/migrations/`.
+// Do not change order without verifying dependencies.
+pub const RUST_MIGRATIONS_BEFORE_LUAU: [Migration; 11] = [
     khronosvalue_v2::MIGRATION,
     kv_generic::MIGRATION,
     tenantstate::MIGRATION,
@@ -37,8 +38,10 @@ pub const RUST_MIGRATIONS: [Migration; 11] = [
     tenant_kv_add_bytea::MIGRATION,
 ];
 
-pub const POST_LUAU_RUST_MIGRATIONS: [Migration; 1] = [
-    // Migrations that need to be applied after all Luau migrations have finished
+// Rust migrations that must run after the luau migrations.
+// kv_scope_unnest collapses scopes[] -> scope, which loses the userid that
+// stings.luau needs to read from the per-row scopes array.
+pub const RUST_MIGRATIONS_AFTER_LUAU: [Migration; 1] = [
     kv_scope_unnest::MIGRATION,
 ];
 
@@ -119,19 +122,16 @@ enum MigrationType {
 
 /// Computes the list of migrations to apply, including both hardcoded Rust migrations and Luau migrations embedded in the binary
 fn migrations() -> Result<Vec<MigrationType>, crate::Error> {
-    let mut base_migrations = Vec::new();
+    let mut base_migrations = RUST_MIGRATIONS_BEFORE_LUAU.into_iter().map(MigrationType::Rust).collect::<Vec<_>>();
 
-    for migration in RUST_MIGRATIONS {
-        base_migrations.push(MigrationType::Rust(migration));
-    }
-
-    // Add luau migrations from MigrationsFolder after all base rust once have finished
+    // Add luau migrations from MigrationsFolder
     for entry in MigrationsFolder::iter() {
         base_migrations.push(MigrationType::Luau(entry));
     }
 
-    for migration in POST_LUAU_RUST_MIGRATIONS {
-        base_migrations.push(MigrationType::Rust(migration));
+    // Rust migrations that depend on luau migrations having already run
+    for mig in RUST_MIGRATIONS_AFTER_LUAU.iter() {
+        base_migrations.push(MigrationType::Rust(*mig));
     }
 
     Ok(base_migrations)