From 631b5e1027dc4c13fdfc9924478eb8c369954b55 Mon Sep 17 00:00:00 2001 From: asepharyana Date: Thu, 27 Aug 2026 15:05:30 +0700 Subject: [PATCH] fix: make gateway migrations idempotent + self-heal drizzle history Prevent recurring infinite restart loop (389x crash) caused by drizzle re-applying already-applied migrations when public.__drizzle_migrations tracking is empty/partial. - 0017/0018: ADD COLUMN IF NOT EXISTS (re-run safe) - 0019: DO-block rename that handles all prior states (server_name-only, both columns, or server_nick-only) so it never errors or double-renames - seedDrizzleHistory: reconcile tracked created_at to the journal's latest 'when' when the schema already reflects the latest migration, instead of early-returning on an existing-but-empty/partial tracking table --- ...017_add_username_to_moderation_actions.sql | 2 +- ..._add_server_name_to_moderation_actions.sql | 2 +- ...0019_rename_server_name_to_server_nick.sql | 24 +++- .../src/shared/database/migrate.ts | 122 +++++++++++++----- 4 files changed, 116 insertions(+), 34 deletions(-) diff --git a/services/discord-gateway/drizzle/migrations/0017_add_username_to_moderation_actions.sql b/services/discord-gateway/drizzle/migrations/0017_add_username_to_moderation_actions.sql index 4b9b04e1..4d646521 100644 --- a/services/discord-gateway/drizzle/migrations/0017_add_username_to_moderation_actions.sql +++ b/services/discord-gateway/drizzle/migrations/0017_add_username_to_moderation_actions.sql @@ -1,4 +1,4 @@ -ALTER TABLE "moderation_actions" ADD COLUMN "username" text; +ALTER TABLE "moderation_actions" ADD COLUMN IF NOT EXISTS "username" text; --> statement-breakpoint -- Backfill username from messages table for existing actions UPDATE "moderation_actions" a diff --git a/services/discord-gateway/drizzle/migrations/0018_add_server_name_to_moderation_actions.sql b/services/discord-gateway/drizzle/migrations/0018_add_server_name_to_moderation_actions.sql index ac29ddfe..612b667c 100644 --- a/services/discord-gateway/drizzle/migrations/0018_add_server_name_to_moderation_actions.sql +++ b/services/discord-gateway/drizzle/migrations/0018_add_server_name_to_moderation_actions.sql @@ -1 +1 @@ -ALTER TABLE "moderation_actions" ADD COLUMN "server_name" text; +ALTER TABLE "moderation_actions" ADD COLUMN IF NOT EXISTS "server_name" text; diff --git a/services/discord-gateway/drizzle/migrations/0019_rename_server_name_to_server_nick.sql b/services/discord-gateway/drizzle/migrations/0019_rename_server_name_to_server_nick.sql index 6148ae57..b74ce7fd 100644 --- a/services/discord-gateway/drizzle/migrations/0019_rename_server_name_to_server_nick.sql +++ b/services/discord-gateway/drizzle/migrations/0019_rename_server_name_to_server_nick.sql @@ -1,4 +1,26 @@ -- Rename server_name (guild name) -> server_nick (member's server nickname) -- server_name was added in 0018 but never populated (all NULL), so renaming -- the empty column is safe and keeps retention semantics correct. -ALTER TABLE "moderation_actions" RENAME COLUMN "server_name" TO "server_nick"; +-- Idempotent across all possible prior states: +-- • server_name present, server_nick absent -> rename +-- • both present (backfill created server_nick) -> drop the stale server_name +-- • server_nick present only -> already done, no-op +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'moderation_actions' AND column_name = 'server_name' + ) AND NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'moderation_actions' AND column_name = 'server_nick' + ) THEN + ALTER TABLE "moderation_actions" RENAME COLUMN "server_name" TO "server_nick"; + ELSIF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'moderation_actions' AND column_name = 'server_name' + ) THEN + -- server_nick already exists (e.g. backfilled out-of-band) — drop the + -- now-redundant server_name column instead of renaming into a conflict. + ALTER TABLE "moderation_actions" DROP COLUMN "server_name"; + END IF; +END $$; diff --git a/services/discord-gateway/src/shared/database/migrate.ts b/services/discord-gateway/src/shared/database/migrate.ts index 10b006c9..d9738854 100644 --- a/services/discord-gateway/src/shared/database/migrate.ts +++ b/services/discord-gateway/src/shared/database/migrate.ts @@ -56,25 +56,56 @@ async function getFirstMigrationTag(): Promise { return first.tag; } +/** + * Read the Drizzle migration journal and return the `when` (folderMillis) + * timestamp of the LAST migration entry. Drizzle's PG migrator applies a + * migration only when `tracked_max_created_at < migration.when`, so seeding + * the tracking table with this value marks every existing journal migration + * as already applied. + */ +async function getLastMigrationWhen(): Promise { + const journalPath = join( + process.cwd(), + "drizzle/migrations/meta/_journal.json", + ); + const raw = await readFile(journalPath, "utf-8"); + const journal: MigrationJournal = JSON.parse(raw); + + if (!journal.entries || journal.entries.length === 0) { + throw new Error( + "Migration journal is empty — cannot determine last migration timestamp", + ); + } + + const last = journal.entries.reduce((newest, entry) => + entry.when > newest.when ? entry : newest, + ); + return last.when; +} + /** * Seed Drizzle's __drizzle_migrations tracking table for pre-existing databases * that were created manually or by an earlier migration system (e.g., the old * checkSchemaExists short-circuit). Without this, Drizzle attempts to re-create * all tables from 0000 and fails with "relation already exists". + * + * The Drizzle PG migrator (drizzle-orm 0.45.x) only reads the LAST tracked + * migration (`SELECT ... ORDER BY created_at DESC LIMIT 1` from + * .__drizzle_migrations) and re-applies every journal + * migration whose `when` is newer than that value. If the tracking table is + * empty or missing the latest entries while the DB schema is already up to + * date, re-running the migrations crashes with "column/relation already + * exists" — putting the gateway into an infinite restart loop. + * + * So instead of early-returning when the table exists (which left it empty / + * partial and caused this exact crash), we RECONCILE: when the DB is a + * pre-existing one (app tables already present) and the schema already + * reflects the latest migration, we seed the tracking table up to the latest + * journal `when` so Drizzle skips everything instead of re-applying it. */ async function seedDrizzleHistory(client: PoolClient): Promise { - const exists = await client.query(` - SELECT EXISTS ( - SELECT FROM information_schema.tables - WHERE table_name = '__drizzle_migrations' - ) - `); - const drizzleTableExists = exists.rows[0]?.exists === true; - if (drizzleTableExists) { - return; // already seeded, nothing to do - } - - // Check whether the app tables pre-exist (old ./migrations/ SQL or manual creation). + // Check whether the app tables pre-exist (old ./migrations/ SQL or manual + // creation). If not, this is a brand-new database — let Drizzle handle it. const hasTextCache = await client.query(` SELECT EXISTS ( SELECT FROM information_schema.columns @@ -85,12 +116,19 @@ async function seedDrizzleHistory(client: PoolClient): Promise { return; // brand-new database, let Drizzle handle everything } - const firstMigrationTag = await getFirstMigrationTag(); + // Final schema "at latest migration" sentinel: the moderation_actions + // table at 0019 has a server_nick column. If present, the schema is at (or + // past) migration 0019, so any journal entries not yet tracked are safe to + // mark as applied rather than re-running (which would fail). + const atLatest = await client.query(` + SELECT EXISTS ( + SELECT FROM information_schema.columns + WHERE table_name = 'moderation_actions' AND column_name = 'server_nick' + ) + `); + const schemaAtLatest = atLatest.rows[0]?.exists === true; - logger.info( - { firstMigrationTag }, - "Seeding Drizzle migration history — marking first migration as already applied on this pre-existing database", - ); + const lastMigrationWhen = await getLastMigrationWhen(); // Create the Drizzle tracking table if it does not exist. // PG15+ locks down CREATE on the public schema for non-owner roles, @@ -128,21 +166,43 @@ async function seedDrizzleHistory(client: PoolClient): Promise { throw createErr; } } - // Drizzle's __drizzle_migrations table has no UNIQUE(hash) - // constraint, so check manually before inserting. - const alreadySeeded = await client.query( - `SELECT 1 FROM "__drizzle_migrations" WHERE hash = $1 LIMIT 1`, - [firstMigrationTag], - ); - if (alreadySeeded.rows.length === 0) { - await client.query( - `INSERT INTO "__drizzle_migrations" (hash, created_at) VALUES ($1, $2)`, - [firstMigrationTag, Date.now()], - ); + + // Current max tracked created_at. Drizzle only re-applies migrations newer + // than this, so if it already reaches the journal's last `when`, nothing to do. + const tracked = await client.query(` + SELECT COALESCE(MAX(created_at), 0) AS max_created FROM "__drizzle_migrations" + `); + const trackedMax = Number(tracked.rows[0]?.max_created ?? 0); + + if (trackedMax >= lastMigrationWhen) { + return; // already fully tracked — Drizzle will apply only genuinely-pending ones } - logger.info( - { firstMigrationTag }, - "Drizzle history seeded — first migration marked applied", + + if (!schemaAtLatest) { + // Schema is NOT yet at the latest journal migration — this is a genuinely + // old DB that needs the pending migrations to run for real. Let Drizzle + // apply them. (Idempotent migration SQL guards legacy partial states.) + logger.info( + { trackedMax, lastMigrationWhen }, + "DB schema behind latest journal migration — leaving pending migrations to run", + ); + return; + } + + // Schema already reflects the latest migration but the tracking table is + // behind (e.g. migrations applied manually/out-of-band without being + // recorded). Seed up to the last journal `when` so Drizzle skips them + // instead of re-running already-applied DDL and crashing. This is the + // recovery that broke the gateway's infinite-restart loop. + const journalTag = + (await getFirstMigrationTag()) + `@${lastMigrationWhen}-reconciled`; + await client.query( + `INSERT INTO "__drizzle_migrations" (hash, created_at) VALUES ($1, $2)`, + [journalTag, lastMigrationWhen], + ); + logger.warn( + { lastMigrationWhen }, + "Reconciled Drizzle migration history — marked schema-at-latest migrations as applied (preventing re-run of already-applied DDL)", ); }