From a45e7df75df1d654668d15f04a340ddf9917f453 Mon Sep 17 00:00:00 2001 From: ditadi Date: Mon, 12 Oct 2026 13:36:54 +0100 Subject: [PATCH] feat(appkit): add appkit db introspect `appkit db introspect` reads an existing Postgres schema (default: app) from pg_catalog and writes config/database/schema.ts from it, keeping the previous file as schema.ts.bak. It also writes the migration that brings history up to the database, 0000_baseline first and NNNN_introspect after, and records it as applied. That is only true on the branch it read, so it runs only on the developer's dev or session branch, which `appkit db init` creates. The deployed app adopts 0000_baseline on start when its tables already exist. What the schema builder cannot express is skipped and reported, and what it declares differently from the database (a timestamp precision, an index) is listed. Objects it does not read (views, triggers, policies, functions, standalone sequences) are named in a warning, since a database the migrations create will not have them. It refuses when the branch's journal and the checkout's migrations differ, and it refuses a migration that drops something schema.ts left out, since the deployed app runs that migration on start. Recording takes the migration lock and checks the journal again, and a run interrupted before recording is finished by the next one. If anything fails after schema.ts is written, schema.ts, its backup, database.d.ts and the migrations are put back. Signed-off-by: ditadi --- .../appkit/src/database/migrations/apply.ts | 120 ++- .../migrations/tests/apply.pg.test.ts | 129 +++ .../database/schema-builder/define-schema.ts | 26 +- packages/appkit/src/database/tooling/index.ts | 5 + .../database/tooling/introspect/catalog.ts | 322 +++++++ .../src/database/tooling/introspect/index.ts | 315 +++++++ .../src/database/tooling/introspect/render.ts | 864 ++++++++++++++++++ .../introspect/tests/catalog.pg.test.ts | 147 +++ .../tooling/introspect/tests/catalog.test.ts | 136 +++ .../tooling/introspect/tests/fixture.ts | 293 ++++++ .../tests/golden/rich.schema.ts.txt | 51 ++ .../introspect/tests/introspect.test.ts | 741 +++++++++++++++ .../tooling/introspect/tests/render.test.ts | 623 +++++++++++++ .../introspect/tests/roundtrip.test.ts | 368 ++++++++ .../appkit/src/database/tooling/migrate.ts | 2 +- .../appkit/src/database/tooling/migrations.ts | 18 +- .../database/tooling/tests/contract.test.ts | 1 + .../database/tooling/tests/migrations.test.ts | 70 ++ packages/shared/src/cli/commands/db/index.ts | 4 +- .../src/cli/commands/db/introspect.test.ts | 171 ++++ .../shared/src/cli/commands/db/introspect.ts | 43 + packages/shared/src/cli/commands/db/output.ts | 48 +- .../src/cli/commands/db/tooling-contract.ts | 24 + 23 files changed, 4510 insertions(+), 11 deletions(-) create mode 100644 packages/appkit/src/database/tooling/introspect/catalog.ts create mode 100644 packages/appkit/src/database/tooling/introspect/index.ts create mode 100644 packages/appkit/src/database/tooling/introspect/render.ts create mode 100644 packages/appkit/src/database/tooling/introspect/tests/catalog.pg.test.ts create mode 100644 packages/appkit/src/database/tooling/introspect/tests/catalog.test.ts create mode 100644 packages/appkit/src/database/tooling/introspect/tests/fixture.ts create mode 100644 packages/appkit/src/database/tooling/introspect/tests/golden/rich.schema.ts.txt create mode 100644 packages/appkit/src/database/tooling/introspect/tests/introspect.test.ts create mode 100644 packages/appkit/src/database/tooling/introspect/tests/render.test.ts create mode 100644 packages/appkit/src/database/tooling/introspect/tests/roundtrip.test.ts create mode 100644 packages/shared/src/cli/commands/db/introspect.test.ts create mode 100644 packages/shared/src/cli/commands/db/introspect.ts diff --git a/packages/appkit/src/database/migrations/apply.ts b/packages/appkit/src/database/migrations/apply.ts index c8e5f9c3f..b9f528640 100644 --- a/packages/appkit/src/database/migrations/apply.ts +++ b/packages/appkit/src/database/migrations/apply.ts @@ -147,7 +147,7 @@ export class MigrationBaselineMismatchError extends Error { constructor(schemaName: string, problems: readonly string[]) { super( `Cannot adopt ${BASELINE_TAG}: schema "${schemaName}" has no recorded migrations and only part of the baseline: ${problems.join("; ")}. ` + - "Create the missing tables and columns (or drop the partial ones), or regenerate the baseline against this database.", + "Create the missing tables and columns (or drop the partial ones), or regenerate the baseline with `appkit db introspect` against this database.", ); this.name = "MigrationBaselineMismatchError"; this.problems = Object.freeze([...problems]); @@ -575,6 +575,124 @@ function duration(ms: number): string { return `${ms}ms`; } +/** How the branch's recorded migrations differ from the committed ones. */ +interface MigrationDrift { + /** Committed, not recorded, newer than every recorded one. */ + readonly pending: readonly string[]; + /** Committed, not recorded, older than the last recorded one. */ + readonly stranded: readonly string[]; + /** Recorded rows no committed migration matches: the branch is ahead of the checkout. */ + readonly unknown: number; +} + +function driftOf( + history: ReturnType, +): MigrationDrift { + return { + pending: tags(history.pending), + stranded: tags(history.stranded), + unknown: history.unknown.length, + }; +} + +function committedMigrations(folder: string): MigrationFile[] { + const journal = readJournal(folder); + return journal ? readMigrations(folder, journal) : []; +} + +/** Read without the lock; markMigrationsApplied checks again under it. */ +export async function readMigrationDrift( + client: QueryClient, + options: ApplyMigrationsOptions, +): Promise { + return driftOf( + reconcileMigrations( + committedMigrations(options.migrationsFolder), + await readRecordedMigrations(client, options.schemaName), + ), + ); +} + +/** Why the branch's history is not the committed one plus toRecord; undefined when it is. */ +function driftReason( + drift: MigrationDrift, + toRecord: readonly string[], +): string | undefined { + const notApplied = drift.pending.filter((tag) => !toRecord.includes(tag)); + const recorded = toRecord.filter((tag) => !drift.pending.includes(tag)); + const reasons = [ + drift.unknown > 0 + ? `it records ${drift.unknown} migration${drift.unknown === 1 ? "" : "s"} this checkout does not have` + : "", + drift.stranded.length > 0 ? `it skipped ${drift.stranded.join(", ")}` : "", + notApplied.length > 0 ? `it has not applied ${notApplied.join(", ")}` : "", + recorded.length > 0 ? `it already records ${recorded.join(", ")}` : "", + ].filter(Boolean); + return reasons.length > 0 ? reasons.join("; ") : undefined; +} + +/** + * Record migrations as applied without running them: the branch already holds + * what they describe. Under the migration lock, and only when they are exactly + * the branch's pending migrations, so DDL the branch never ran is not recorded. + */ +export async function markMigrationsApplied( + pool: Pick, + options: ApplyMigrationsOptions, + toRecord: readonly string[], +): Promise { + const files = committedMigrations(options.migrationsFolder); + const missing = toRecord.filter( + (tag) => !files.some((file) => file.tag === tag), + ); + if (missing.length > 0) { + throw new MigrationFilesError( + `Cannot record ${missing.join(", ")} as applied: not in ${options.migrationsFolder}/meta/_journal.json.`, + ); + } + const { schemaName } = options; + const client = await pool.connect(); + const reset: string[] = []; + let locked = false; + try { + await prepareMigrationSession(client, reset); + await acquireMigrationLock( + client, + schemaName, + options.lockWaitMs ?? LOCK_WAIT_MS, + ); + locked = true; + const history = reconcileMigrations( + files, + await readRecordedMigrations(client, schemaName), + ); + const reason = driftReason(driftOf(history), toRecord); + if (reason) { + throw new Error( + `Schema "${schemaName}" changed while this ran (${reason}), so nothing was recorded.`, + ); + } + await inTransaction(client, async () => { + await ensureJournalTable(client, schemaName); + for (const file of history.pending) { + await recordMigration(client, schemaName, file); + } + }); + } catch (error) { + if (!(error instanceof MigrationLockWaitError)) throw error; + throw new Error( + `${error.holder ? `Session ${error.holder}` : "Another session"} held the migration lock of schema "${schemaName}" for over ${duration(options.lockWaitMs ?? LOCK_WAIT_MS)}, so nothing was recorded. Rerun once it finishes.`, + { cause: error }, + ); + } finally { + await releaseMigrationClient( + client, + locked ? migrationLockKey(schemaName) : undefined, + reset, + ); + } +} + /** * Apply every committed migration the database has not recorded yet. * diff --git a/packages/appkit/src/database/migrations/tests/apply.pg.test.ts b/packages/appkit/src/database/migrations/tests/apply.pg.test.ts index 7e4875576..c3b2d7b99 100644 --- a/packages/appkit/src/database/migrations/tests/apply.pg.test.ts +++ b/packages/appkit/src/database/migrations/tests/apply.pg.test.ts @@ -16,8 +16,11 @@ import { applyPendingMigrations, MigrationApplyError, MigrationHistoryError, + markMigrationsApplied, migrationLockKey, + readMigrationDrift, } from "../apply"; +import { MigrationFilesError } from "../paths"; import { hashOf, removeMigrationFolders, writeMigrations } from "./fixtures"; // A real server, for what a fake cannot show: transactions, enum rules, locks @@ -405,4 +408,130 @@ describe.skipIf(!url)("applyPendingMigrations against Postgres", () => { }); expect(await columns("notes")).toEqual(["id", "body"]); }); + + describe("recording migrations the branch already has", () => { + const withTags = () => + writeMigrations([ + { + tag: "0000_notes", + when: 1000, + sql: [`CREATE TABLE ${q("notes")} ("id" integer PRIMARY KEY);`], + }, + { + tag: "0001_introspect", + when: 3000, + sql: [`ALTER TABLE ${q("notes")} ADD COLUMN "body" text;`], + }, + ]); + /** The branch applied 0000_notes and recorded a 2000 migration this checkout lacks. */ + const aheadOfCheckout = async (folder: string) => { + await applyPendingMigrations(newPool(), { + ...options(folder), + migrationsFolder: notes(), + }); + await admin.query( + `delete from ${q("__drizzle_migrations")} where created_at <> 1000`, + ); + await admin.query( + `insert into ${q("__drizzle_migrations")} (hash, created_at) values ('teammate', 2000)`, + ); + }; + + test("the drift lists unrecorded committed migrations and recorded ones the checkout lacks", async () => { + const folder = withTags(); + expect(await readMigrationDrift(newPool(), options(folder))).toEqual({ + pending: ["0000_notes", "0001_introspect"], + stranded: [], + unknown: 0, + }); + + await aheadOfCheckout(folder); + + expect(await readMigrationDrift(newPool(), options(folder))).toEqual({ + pending: ["0001_introspect"], + stranded: [], + unknown: 1, + }); + }); + + test("records exactly the pending migrations, without running them", async () => { + await admin.query(`create schema "${schema}"`); + const folder = withTags(); + + await markMigrationsApplied(newPool(), options(folder), [ + "0000_notes", + "0001_introspect", + ]); + + expect(await recorded()).toEqual([ + { hash: hashOf(folder, "0000_notes"), createdAt: 1000 }, + { hash: hashOf(folder, "0001_introspect"), createdAt: 3000 }, + ]); + expect(await columns("notes")).toEqual([]); + }); + + test.each([ + [ + "records a migration the checkout lacks", + aheadOfCheckout, + ["0001_introspect"], + "it records 1 migration this checkout does not have", + ], + [ + "has not applied an earlier one", + async () => { + await admin.query(`create schema "${schema}"`); + }, + ["0001_introspect"], + "it has not applied 0000_notes", + ], + ])( + "refuses when the branch %s, and records nothing", + async (_label, arrange, toRecord, reason) => { + const folder = withTags(); + await arrange(folder); + const before = await recorded().catch(() => []); + + await expect( + markMigrationsApplied(newPool(), options(folder), toRecord), + ).rejects.toThrow( + `Schema "${schema}" changed while this ran (${reason}), so nothing was recorded.`, + ); + expect(await recorded().catch(() => [])).toEqual(before); + }, + ); + + test("a tag missing from the journal fails before connecting", async () => { + const pool = newPool(); + const connect = vi.spyOn(pool, "connect"); + + await expect( + markMigrationsApplied(pool, options(withTags()), ["0002_introspect"]), + ).rejects.toBeInstanceOf(MigrationFilesError); + expect(connect).not.toHaveBeenCalled(); + }); + + test("the wait for the migration lock has a limit, and nothing is recorded", async () => { + await admin.query(`create schema "${schema}"`); + const holder = await admin.connect(); + held.push(holder); + await holder.query("set application_name = 'another-deploy'"); + await holder.query("select pg_advisory_lock($1::bigint)", [ + migrationLockKey(schema), + ]); + + await expect( + markMigrationsApplied( + newPool(), + { ...options(withTags()), lockWaitMs: 300 }, + ["0000_notes", "0001_introspect"], + ), + ).rejects.toThrow( + new RegExp( + `^Session pid \\d+ \\(another-deploy, idle\\) held the migration lock of schema "${schema}" for over 300ms, so nothing was recorded\\. Rerun once it finishes\\.$`, + ), + ); + expect(await recorded().catch(() => [])).toEqual([]); + }); + }); }); diff --git a/packages/appkit/src/database/schema-builder/define-schema.ts b/packages/appkit/src/database/schema-builder/define-schema.ts index 5d8cf3d91..8d8038d08 100644 --- a/packages/appkit/src/database/schema-builder/define-schema.ts +++ b/packages/appkit/src/database/schema-builder/define-schema.ts @@ -72,9 +72,27 @@ function assertName(value: string, label: string): void { } } +/** Table names defineSchema() refuses. */ +export function isReservedTableName(name: string): boolean { + return ( + !name || + RESERVED_OBJECT_KEYS.has(name) || + RESERVED_DATABASE_EXPORT_KEYS.has(name) + ); +} + +/** Column keys that collide with the runtime metadata on a table handle. */ +function collidesWithMetadata(key: string): boolean { + return key.startsWith("$") || key === "and" || key === "or"; +} + +/** Column names defineSchema() refuses. */ +export function isReservedColumnName(name: string): boolean { + return !name || RESERVED_OBJECT_KEYS.has(name) || collidesWithMetadata(name); +} + function assertTableName(value: string): void { - assertName(value, "Table name"); - if (RESERVED_DATABASE_EXPORT_KEYS.has(value)) { + if (isReservedTableName(value)) { throw new SchemaBuildError(`Table name "${value}" is reserved`); } } @@ -120,7 +138,7 @@ function declareTable>( const handle = Object.create(null) as Record; for (const [key, column] of Object.entries(columns)) { assertName(key, `Column name on table "${name}"`); - if (key.startsWith("$") || key === "and" || key === "or") { + if (collidesWithMetadata(key)) { throw new SchemaBuildError( `Column "${name}.${key}" collides with AppKit runtime metadata`, ); @@ -344,7 +362,7 @@ export function assertFinalizedSchema(value: unknown): asserts value is Schema { * the deployed app's service principal can create and own a schema of its * own, but has no CREATE on `public`. */ -const DEFAULT_SCHEMA_NAME = "app"; +export const DEFAULT_SCHEMA_NAME = "app"; /** * Compile one declared schema. The returned type keeps the table names the diff --git a/packages/appkit/src/database/tooling/index.ts b/packages/appkit/src/database/tooling/index.ts index bbec5337a..29adba529 100644 --- a/packages/appkit/src/database/tooling/index.ts +++ b/packages/appkit/src/database/tooling/index.ts @@ -7,4 +7,9 @@ * the same AppKit release should import it. */ export { createDbContext, type DbContext } from "./context"; +export { + type IntrospectOptions, + type IntrospectReport, + runIntrospect, +} from "./introspect"; export { type MigrateOptions, type MigrateReport, runMigrate } from "./migrate"; diff --git a/packages/appkit/src/database/tooling/introspect/catalog.ts b/packages/appkit/src/database/tooling/introspect/catalog.ts new file mode 100644 index 000000000..07703b0aa --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/catalog.ts @@ -0,0 +1,322 @@ +import { MIGRATIONS_TABLE } from "../../migrations/paths"; +import type { ReferentialAction } from "../../schema-builder/types"; + +/** Run one parameterized query and return its rows (a pg Pool fits: `(t, v) => pool.query(t, v).then(r => r.rows)`). */ +export type CatalogQuery = ( + text: string, + values: readonly unknown[], +) => Promise; + +export interface IntrospectedEnum { + readonly name: string; + readonly schema: string; + readonly values: readonly string[]; +} + +export interface IntrospectedColumn { + readonly name: string; + /** format_type(), e.g. "character varying(120)". */ + readonly type: string; + readonly notNull: boolean; + /** attidentity: "" none, "a" always, "d" by default. */ + readonly identity: "" | "a" | "d"; + /** Generated column, stored or virtual (attgenerated 's' or 'v'). */ + readonly generated: boolean; + /** pg_get_expr() of the default (or generation expression). */ + readonly defaultExpr: string | null; + readonly enum?: IntrospectedEnum; +} + +export interface IntrospectedForeignKey { + readonly name: string; + readonly columns: readonly string[]; + readonly schema: string; + readonly table: string; + readonly refColumns: readonly string[]; + readonly onDelete: ReferentialAction; + readonly onUpdate: ReferentialAction; + /** DEFERRABLE or MATCH FULL, neither of which the builder declares. */ + readonly deferrable: boolean; + readonly matchFull: boolean; +} + +export interface IntrospectedUnique { + readonly name: string; + readonly columns: readonly string[]; +} + +export interface IntrospectedTable { + readonly name: string; + /** relkind 'p': partitioned in the database. */ + readonly partitioned: boolean; + readonly columns: readonly IntrospectedColumn[]; + readonly primaryKey: readonly string[]; + readonly unique: readonly IntrospectedUnique[]; + readonly foreignKeys: readonly IntrospectedForeignKey[]; + /** CHECK and EXCLUDE constraint names. */ + readonly checks: readonly string[]; + /** Index names, except the indexes behind primary-key, unique and exclusion constraints. */ + readonly indexes: readonly string[]; +} + +/** pg_constraint.confdeltype / confupdtype. */ +const ACTIONS: Record = { + a: "no action", + r: "restrict", + c: "cascade", + n: "set null", + d: "set default", +}; + +export interface ColumnRow { + table_name: string; + name: string; + type: string; + not_null: boolean; + identity: string; + generated: string; + default_expr: string | null; + type_kind: string; + type_schema: string; + type_name: string; + enum_values: string[] | null; +} + +export interface TableRow { + name: string; + partitioned: boolean; +} + +export interface ConstraintRow { + table_name: string; + name: string; + kind: "p" | "u" | "f" | "c" | "x"; + columns: string[]; + ref_schema: string | null; + ref_table: string | null; + ref_columns: string[] | null; + on_delete: string; + on_update: string; + deferrable?: boolean; + /** confmatchtype: "f" full, "s" simple. */ + match_type?: string; +} + +export interface IndexRow { + table_name: string; + name: string; +} + +// pg_catalog, never information_schema: it hides objects the role cannot access. +// Ordinary and partitioned tables only; partitions belong to their parent. +const TABLES_SQL = ` +select c.relname::text as name, c.relkind = 'p' as partitioned + from pg_catalog.pg_class c + join pg_catalog.pg_namespace n on n.oid = c.relnamespace + where n.nspname = $1 and c.relkind in ('r', 'p') and not c.relispartition + and c.relname <> $2 + order by c.relname`; + +const COLUMNS_SQL = ` +select c.relname::text as table_name, a.attname::text as name, + pg_catalog.format_type(a.atttypid, a.atttypmod) as type, + a.attnotnull as not_null, a.attidentity::text as identity, + a.attgenerated::text as generated, + pg_catalog.pg_get_expr(d.adbin, d.adrelid) as default_expr, + t.typtype::text as type_kind, tn.nspname::text as type_schema, + t.typname::text as type_name, + case when t.typtype = 'e' then array( + select e.enumlabel::text from pg_catalog.pg_enum e + where e.enumtypid = t.oid order by e.enumsortorder) + end as enum_values + from pg_catalog.pg_attribute a + join pg_catalog.pg_class c on c.oid = a.attrelid + join pg_catalog.pg_namespace n on n.oid = c.relnamespace + join pg_catalog.pg_type t on t.oid = a.atttypid + join pg_catalog.pg_namespace tn on tn.oid = t.typnamespace + left join pg_catalog.pg_attrdef d on d.adrelid = a.attrelid and d.adnum = a.attnum + where n.nspname = $1 and c.relkind in ('r', 'p') and not c.relispartition + and c.relname <> $2 and a.attnum > 0 and not a.attisdropped + order by c.relname, a.attnum`; + +const CONSTRAINTS_SQL = ` +select c.relname::text as table_name, k.conname::text as name, k.contype::text as kind, + array(select a.attname::text + from unnest(k.conkey) with ordinality as u(attnum, ord) + join pg_catalog.pg_attribute a on a.attrelid = k.conrelid and a.attnum = u.attnum + order by u.ord) as columns, + rn.nspname::text as ref_schema, rc.relname::text as ref_table, + array(select a.attname::text + from unnest(k.confkey) with ordinality as u(attnum, ord) + join pg_catalog.pg_attribute a on a.attrelid = k.confrelid and a.attnum = u.attnum + order by u.ord) as ref_columns, + k.confdeltype::text as on_delete, k.confupdtype::text as on_update, + k.condeferrable as deferrable, k.confmatchtype::text as match_type + from pg_catalog.pg_constraint k + join pg_catalog.pg_class c on c.oid = k.conrelid + join pg_catalog.pg_namespace n on n.oid = c.relnamespace + left join pg_catalog.pg_class rc on rc.oid = k.confrelid + left join pg_catalog.pg_namespace rn on rn.oid = rc.relnamespace + where n.nspname = $1 and k.contype in ('p', 'u', 'f', 'c', 'x') and c.relname <> $2 + -- A foreign key to a partitioned table has one copy per partition; keep the original. + and k.conparentid = 0 + order by c.relname, k.conname`; + +// Indexes no constraint stands for: schema.ts cannot declare them. +const INDEXES_SQL = ` +select c.relname::text as table_name, i.relname::text as name + from pg_catalog.pg_index x + join pg_catalog.pg_class c on c.oid = x.indrelid + join pg_catalog.pg_class i on i.oid = x.indexrelid + join pg_catalog.pg_namespace n on n.oid = c.relnamespace + where n.nspname = $1 and c.relname <> $2 + and not exists ( + select 1 from pg_catalog.pg_constraint k + where k.conrelid = x.indrelid and k.conindid = x.indexrelid + and k.contype in ('p', 'u', 'x')) + order by c.relname, i.relname`; + +// What readCatalog does not read, so the report can name it. Objects an +// extension owns are left out: the extension recreates them. +const UNREAD_SQL = ` +with ns as (select oid from pg_catalog.pg_namespace where nspname = $1), +objects (label, classid, objid) as ( + select case c.relkind when 'v' then 'view ' when 'm' then 'materialized view ' + when 'f' then 'foreign table ' else 'sequence ' end || c.relname, + 'pg_catalog.pg_class'::regclass, c.oid + from pg_catalog.pg_class c + where c.relnamespace = (select oid from ns) and c.relkind in ('v', 'm', 'f', 'S') + -- serial and identity sequences belong to their column + and not exists ( + select 1 from pg_catalog.pg_depend d + where d.classid = 'pg_catalog.pg_class'::regclass and d.objid = c.oid + and d.refclassid = 'pg_catalog.pg_class'::regclass and d.deptype in ('a', 'i')) + union all + select 'trigger ' || t.tgname || ' on ' || c.relname, 'pg_catalog.pg_trigger'::regclass, t.oid + from pg_catalog.pg_trigger t + join pg_catalog.pg_class c on c.oid = t.tgrelid + where c.relnamespace = (select oid from ns) and not t.tgisinternal + union all + select 'policy ' || p.polname || ' on ' || c.relname, 'pg_catalog.pg_policy'::regclass, p.oid + from pg_catalog.pg_policy p + join pg_catalog.pg_class c on c.oid = p.polrelid + where c.relnamespace = (select oid from ns) + union all + select 'row-level security on ' || c.relname, 'pg_catalog.pg_class'::regclass, c.oid + from pg_catalog.pg_class c + where c.relnamespace = (select oid from ns) and c.relrowsecurity + union all + select 'inheritance of ' || c.relname || ' from ' || p.relname, 'pg_catalog.pg_class'::regclass, c.oid + from pg_catalog.pg_inherits i + join pg_catalog.pg_class c on c.oid = i.inhrelid + join pg_catalog.pg_class p on p.oid = i.inhparent + where c.relnamespace = (select oid from ns) and not c.relispartition + union all + select case p.prokind when 'p' then 'procedure ' when 'a' then 'aggregate ' else 'function ' end + || p.proname || '(' || pg_catalog.pg_get_function_identity_arguments(p.oid) || ')', + 'pg_catalog.pg_proc'::regclass, p.oid + from pg_catalog.pg_proc p + where p.pronamespace = (select oid from ns) + union all + select case t.typtype when 'd' then 'domain ' else 'type ' end || t.typname, + 'pg_catalog.pg_type'::regclass, t.oid + from pg_catalog.pg_type t + where t.typnamespace = (select oid from ns) + and (t.typtype in ('d', 'r') + or (t.typtype = 'c' and exists ( + select 1 from pg_catalog.pg_class c where c.oid = t.typrelid and c.relkind = 'c')) + -- an enum is read through the columns that use it + or (t.typtype = 'e' and not exists ( + select 1 from pg_catalog.pg_attribute a + join pg_catalog.pg_class c on c.oid = a.attrelid + where a.atttypid = t.oid and c.relkind in ('r', 'p') and not a.attisdropped))) +) +select o.label from objects o + where not exists ( + select 1 from pg_catalog.pg_depend d + where d.classid = o.classid and d.objid = o.objid and d.deptype = 'e') + order by o.label`; + +function toColumn(row: ColumnRow): IntrospectedColumn { + const identity = + row.identity === "a" || row.identity === "d" ? row.identity : ""; + return { + name: row.name, + type: row.type, + notNull: row.not_null, + identity, + generated: row.generated !== "", + defaultExpr: row.default_expr, + ...(row.type_kind === "e" + ? { + enum: { + name: row.type_name, + schema: row.type_schema, + values: row.enum_values ?? [], + }, + } + : {}), + }; +} + +/** Assemble the model from the catalog result sets (exported for fixture tests). */ +export function buildTables( + tables: readonly TableRow[], + columns: readonly ColumnRow[], + constraints: readonly ConstraintRow[], + indexes: readonly IndexRow[] = [], +): IntrospectedTable[] { + return tables.map(({ name, partitioned }) => { + const own = constraints.filter((row) => row.table_name === name); + return { + name, + partitioned, + columns: columns.filter((row) => row.table_name === name).map(toColumn), + primaryKey: own.find((row) => row.kind === "p")?.columns ?? [], + unique: own + .filter((row) => row.kind === "u") + .map((row) => ({ name: row.name, columns: row.columns })), + foreignKeys: own + .filter((row) => row.kind === "f" && row.ref_table) + .map((row) => ({ + name: row.name, + columns: row.columns, + schema: row.ref_schema ?? "", + table: row.ref_table ?? "", + refColumns: row.ref_columns ?? [], + onDelete: ACTIONS[row.on_delete] ?? "no action", + onUpdate: ACTIONS[row.on_update] ?? "no action", + deferrable: row.deferrable === true, + matchFull: row.match_type === "f", + })), + checks: own + .filter((row) => row.kind === "c" || row.kind === "x") + .map((row) => row.name), + indexes: indexes + .filter((row) => row.table_name === name) + .map((row) => row.name), + }; + }); +} + +/** Tables of one schema from pg_catalog, excluding the migration journal. */ +export async function readCatalog( + query: CatalogQuery, + schemaName: string, +): Promise { + const params = [schemaName, MIGRATIONS_TABLE]; + const tables = await query(TABLES_SQL, params); + const columns = await query(COLUMNS_SQL, params); + const constraints = await query(CONSTRAINTS_SQL, params); + const indexes = await query(INDEXES_SQL, params); + return buildTables(tables, columns, constraints, indexes); +} + +/** Objects of the schema readCatalog does not read, e.g. "trigger audit on notes". */ +export async function readUnreadObjects( + query: CatalogQuery, + schemaName: string, +): Promise { + const rows = await query<{ label: string }>(UNREAD_SQL, [schemaName]); + return rows.map((row) => row.label); +} diff --git a/packages/appkit/src/database/tooling/introspect/index.ts b/packages/appkit/src/database/tooling/introspect/index.ts new file mode 100644 index 000000000..66ce9bd7d --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/index.ts @@ -0,0 +1,315 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { + DEFAULT_SCHEMA_FILE, + loadDefaultDatabaseSchema, +} from "../../../plugins/database/load-schema"; +import { writeDatabaseTypes } from "../../../type-generator/database"; +import { + markMigrationsApplied, + readMigrationDrift, +} from "../../migrations/apply"; +import { + journalFile, + migrationsFolder, + readJournal, +} from "../../migrations/paths"; +import { DEFAULT_SCHEMA_NAME } from "../../schema-builder/define-schema"; +import { type DbContext, NO_ENV_BRANCH, withBranchPool } from "../context"; +import { notYourBranch, resolveEnvBranch, takesWrites } from "../guard"; +import { DATABASE_TYPES_PATH } from "../migrate"; +import { + breakingStatements, + discardMigrationFiles, + type GeneratedMigration, + generateMigrationFiles, +} from "../migrations"; +import { resourceId } from "../targets"; +import { readCatalog, readUnreadObjects } from "./catalog"; +import { + renderSchemaSource, + type Skip, + shown, + type Unexpressed, +} from "./render"; + +export interface IntrospectOptions { + /** false: an ambiguous rename against existing history throws instead of prompting. */ + readonly interactive: boolean; + /** Schema to read; defaults to DEFAULT_SCHEMA_NAME. The current schema.ts is never read. */ + readonly schemaName?: string; + /** Module the written schema.ts imports the builder from. Default "@databricks/appkit/beta". */ + readonly importSource?: string; +} + +export interface IntrospectReport { + readonly branch: string; + readonly schemaName: string; + /** Tables written to schema.ts. */ + readonly tables: readonly string[]; + readonly schemaFile: string; + /** Where the previous schema.ts went, when this run replaced a different one. */ + readonly backupFile: string | null; + readonly skipped: readonly Skip[]; + readonly warnings: readonly string[]; + /** What schema.ts declares differently from the database, which keeps it as it is. */ + readonly approximated: readonly string[]; + /** 0000_baseline without history, NNNN_introspect with it; null when nothing changed. */ + readonly migration: GeneratedMigration | null; + /** Tags recorded as applied: the migration, and any an interrupted run wrote but did not record. */ + readonly recorded: readonly string[]; + readonly typesFile: string; +} + +/** Introspect would record history the branch or the deployed app contradicts. */ +export class IntrospectRefusedError extends Error { + constructor(message: string) { + super(message); + this.name = "IntrospectRefusedError"; + } +} + +/** Tags introspect writes. Pending ones match the branch when a fresh render adds nothing. */ +const INTROSPECT_TAG = /^\d{4}_(introspect|baseline)$/; + +/** File contents, or null when it does not exist. */ +function readIfExists(file: string): string | null { + return fs.existsSync(file) ? fs.readFileSync(file, "utf-8") : null; +} + +function writeOrRemove(file: string, contents: string | null): void { + if (contents === null) fs.rmSync(file, { force: true }); + else fs.writeFileSync(file, contents); +} + +const plural = (count: number, word: string) => + `${count} ${word}${count === 1 ? "" : "s"}`; + +/** The statement drizzle-kit 0.31 writes to remove it. */ +function dropStatement(item: Unexpressed, schemaName: string): string { + // drizzle-kit leaves the public schema out of the name. + const named = + schemaName === "public" + ? `"${item.name}"` + : `"${schemaName}"."${item.name}"`; + switch (item.kind) { + case "table": + return `DROP TABLE ${named} CASCADE;`; + case "column": + return `ALTER TABLE ${named} DROP COLUMN "${item.member}";`; + case "default": + return `ALTER TABLE ${named} ALTER COLUMN "${item.member}" DROP DEFAULT;`; + case "constraint": + return `ALTER TABLE ${named} DROP CONSTRAINT "${item.member}";`; + case "type": + return `DROP TYPE ${named};`; + } +} + +/** + * With history, the migration is recorded here as applied and runs on deploy. + * A drop of what the branch still has but schema.ts cannot express would be + * recorded falsely here, and would remove it from every deployed database. + */ +function refuseDropsOfUnexpressed( + statements: readonly string[], + unexpressed: readonly Unexpressed[], + schemaName: string, +): void { + const reasons = new Map( + unexpressed.map((item) => [dropStatement(item, schemaName), item.reason]), + ); + const refused = statements + .map((statement) => statement.trim()) + .filter((statement) => reasons.has(statement)); + if (refused.length === 0) return; + throw new IntrospectRefusedError( + `Refusing to write a migration that drops what the branch still has but schema.ts cannot express: ${refused + .map( + (statement) => + `${shown(statement)} (${shown(reasons.get(statement) ?? "")})`, + ) + .join( + " ", + )}. The deployed app would run ${refused.length === 1 ? "it" : "them"} on start.`, + ); +} + +/** + * `appkit db introspect`: read the `.env` branch's app schema from pg_catalog, + * write schema.ts (the previous file goes to schema.ts.bak), write the + * migration that brings history up to the database, and record it as applied. + * Runs only on your dev or session branch, and only when the branch records + * exactly the committed migrations. A failure puts back schema.ts, the backup, + * database.d.ts and the migrations folder. + */ +export async function runIntrospect( + ctx: DbContext, + options: IntrospectOptions, +): Promise { + const schemaFile = path.join(ctx.root, DEFAULT_SCHEMA_FILE); + // Introspect replaces schema.ts, so the current file (missing, empty or broken) is never read. + const schemaName = options.schemaName || DEFAULT_SCHEMA_NAME; + const folder = migrationsFolder(ctx.root); + const migrationOptions = { migrationsFolder: folder, schemaName }; + const hasHistory = (readJournal(folder)?.entries.length ?? 0) > 0; + + const env = await resolveEnvBranch(ctx); + if (!env) throw new Error(`Did not introspect: ${NO_ENV_BRANCH}.`); + if (!takesWrites(env.kind)) { + throw new IntrospectRefusedError( + `Refusing to introspect: ${notYourBranch(env)}. Introspect records the migration it writes as applied, so it runs only on branches \`appkit db init\` created for you. ` + + `Run \`appkit db init\`: it creates your dev branch from the project's default branch, and introspects it when there is no ${DEFAULT_SCHEMA_FILE}.`, + ); + } + const branchId = resourceId(env.branch.name); + + return withBranchPool(ctx, async (pool) => { + // The diff starts from the latest committed snapshot, so the branch must + // record exactly the committed migrations: one it lacks would come out + // reversed, and one the checkout lacks would come out again. + const drift = await readMigrationDrift(pool, migrationOptions); + const refuse = (reason: string, advice: string) => + new IntrospectRefusedError( + `Refusing to introspect ${branchId}: ${reason} in schema "${schemaName}". ${advice}`, + ); + if (drift.unknown > 0) { + throw refuse( + `it records ${plural(drift.unknown, "migration")} this checkout does not have`, + "Pull the latest config/database/migrations, then rerun.", + ); + } + if (drift.stranded.length > 0) { + throw refuse( + `it skipped ${drift.stranded.join(", ")}`, + "Run `appkit db migrate`, which says how to fix it.", + ); + } + // An interrupted run leaves its migration written but not recorded. + const unrecorded = drift.pending.every((tag) => INTROSPECT_TAG.test(tag)) + ? drift.pending + : []; + const notApplied = (them: string) => + refuse( + `it has not applied ${drift.pending.join(", ")}`, + `${them} Run \`appkit db migrate\` first.`, + ); + if (drift.pending.length > unrecorded.length) { + throw notApplied( + `The migration would undo ${drift.pending.length === 1 ? "it" : "them"}.`, + ); + } + + const query = async (text: string, values: readonly unknown[]) => + (await pool.query(text, [...values])).rows as T[]; + const tables = await readCatalog(query, schemaName); + const unread = await readUnreadObjects(query, schemaName); + const rendered = renderSchemaSource(tables, schemaName, { + importSource: options.importSource, + }); + // An app on another schema that forgot --schema-name must not lose its + // schema.ts to an empty one; with history, an empty schema drops every table. + if ( + rendered.tables.length === 0 && + (hasHistory || fs.existsSync(schemaFile)) + ) { + const reasons = rendered.skipped + .filter((skip) => !skip.column) + .map((skip) => `${skip.table}: ${skip.reason}`); + throw new IntrospectRefusedError( + `Schema "${schemaName}" on ${branchId} has no tables${tables.length > 0 ? ` introspect can write (${reasons.join("; ")})` : ""}, so introspect would ${hasHistory ? "write a migration that drops every table" : `replace ${DEFAULT_SCHEMA_FILE} with an empty one`}. Pass --schema-name to read another schema.`, + ); + } + + const backupFile = `${schemaFile}.bak`; + const typesFile = path.join(ctx.root, DATABASE_TYPES_PATH); + const previous = readIfExists(schemaFile); + const previousBackup = readIfExists(backupFile); + const previousTypes = readIfExists(typesFile); + const backsUp = previous !== null && previous !== rendered.source; + const journalBefore = readIfExists(journalFile(folder)); + let migration: GeneratedMigration | null = null; + let recorded: string[] = []; + try { + if (backsUp) fs.writeFileSync(backupFile, previous); + fs.mkdirSync(path.dirname(schemaFile), { recursive: true }); + fs.writeFileSync(schemaFile, rendered.source); + let schema: Awaited>; + try { + schema = await loadDefaultDatabaseSchema(ctx.root, { + showImportError: true, + }); + } catch (error) { + throw new Error( + `The schema.ts introspect generated does not load. This is a bug in \`appkit db introspect\`; please report it with the catalog of schema "${schemaName}".`, + { cause: error }, + ); + } + + migration = await generateMigrationFiles({ + folder, + schema, + tag: hasHistory ? "introspect" : "baseline", + interactive: options.interactive, + ...(hasHistory + ? { + beforeWrite: (statements: readonly string[]) => + refuseDropsOfUnexpressed( + statements, + rendered.unexpressed, + schemaName, + ), + } + : {}), + }); + if (unrecorded.length > 0 && migration) { + throw notApplied( + `Its tables differ from what ${unrecorded.length === 1 ? "it describes" : "they describe"}.`, + ); + } + await writeDatabaseTypes(schema, typesFile); + // The database already holds what the migrations describe; recording is the last step. + recorded = [...unrecorded, ...(migration ? [migration.tag] : [])]; + if (recorded.length > 0) { + await markMigrationsApplied(pool, migrationOptions, recorded); + } + } catch (error) { + writeOrRemove(schemaFile, previous); + if (backsUp) writeOrRemove(backupFile, previousBackup); + writeOrRemove(typesFile, previousTypes); + if (migration) discardMigrationFiles(folder, migration, journalBefore); + if (error instanceof IntrospectRefusedError) throw error; + throw new Error( + `Introspect changed nothing: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } + + const warnings = [...rendered.warnings]; + if (unread.length > 0) { + warnings.push( + `Not read: ${unread.map(shown).join(", ")}. schema.ts and its migrations leave ${unread.length === 1 ? "it" : "them"} out, so a database the migrations create will not have ${unread.length === 1 ? "it" : "them"}.`, + ); + } + const breaking = breakingStatements(migration?.statements ?? []); + if (migration && breaking.length > 0) { + warnings.push( + `${migration.tag} breaks instances still running the previous release, which keep serving while the new one migrates on start: ${breaking.join(" ")}`, + ); + } + return { + branch: env.branch.name, + schemaName, + tables: rendered.tables, + schemaFile, + backupFile: backsUp ? backupFile : null, + skipped: rendered.skipped, + warnings, + approximated: rendered.approximated, + migration, + recorded, + typesFile, + }; + }); +} diff --git a/packages/appkit/src/database/tooling/introspect/render.ts b/packages/appkit/src/database/tooling/introspect/render.ts new file mode 100644 index 000000000..b8378f5e5 --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/render.ts @@ -0,0 +1,864 @@ +import { + DEFAULT_SCHEMA_NAME, + isReservedColumnName, + isReservedTableName, +} from "../../schema-builder/define-schema"; +import type { + IntrospectedColumn, + IntrospectedForeignKey, + IntrospectedTable, +} from "./catalog"; + +/** + * Something introspect left out of schema.ts, with an `// appkit: skipped …` comment. + * Display text: catalog names come with control characters escaped (see shown()). + */ +export interface Skip { + readonly table: string; + readonly column?: string; + readonly reason: string; +} + +/** + * Something the branch has that schema.ts leaves out, by raw catalog name. A + * migration that drops it must not be recorded as applied: the branch keeps it. + */ +export interface Unexpressed { + readonly kind: "table" | "column" | "default" | "constraint" | "type"; + /** The table, or the enum type for "type". */ + readonly name: string; + /** The column for "column" and "default", the constraint for "constraint". */ + readonly member?: string; + readonly reason: string; +} + +interface SchemaSource { + readonly source: string; + /** Tables written to schema.ts. */ + readonly tables: readonly string[]; + readonly skipped: readonly Skip[]; + readonly unexpressed: readonly Unexpressed[]; + /** Omissions that change behavior, e.g. a NOT NULL column the API can no longer fill. */ + readonly warnings: readonly string[]; + /** What schema.ts declares differently from the database, which keeps it as it is. */ + readonly approximated: readonly string[]; +} + +interface RenderOptions { + /** Module the builder is imported from. Default "@databricks/appkit/beta". */ + readonly importSource?: string; +} + +/** Storage as the builder sees it; id/bigid are server-generated integer/bigint keys. */ +type Storage = + | "id" + | "bigid" + | "integer" + | "bigint" + | "text" + | "varchar" + | "boolean" + | "uuid" + | "timestamp" + | "jsonb" + | "enum"; + +interface BaseType { + /** Builder call, e.g. `varchar(120)`. */ + readonly call: string; + /** Import it needs ("" for the context's enum). */ + readonly importName: string; + readonly storage: Storage; + readonly varcharLength?: number; + /** How the call differs from the database type. */ + readonly approximation?: string; +} + +interface ColumnPlan { + readonly column: IntrospectedColumn; + base: BaseType; + /** Set when `fk(() => .)` replaces the base call. */ + fk?: { + readonly table: string; + readonly column: string; + readonly actions: string[]; + }; + readonly isPrimaryKey: boolean; + readonly isUnique: boolean; + defaultCall?: string; + readonly comments: string[]; +} + +interface TablePlan { + readonly table: IntrospectedTable; + readonly columns: ColumnPlan[]; + /** Table-level comments, rendered first. */ + readonly comments: string[]; + /** Columns and skipped-column comments, in catalog order. */ + readonly body: Array; + variable: string; +} + +const JS_RESERVED = new Set( + ( + "break case catch class const continue debugger default delete do else enum export extends false finally for " + + "function if import in instanceof new null return super switch this throw true try typeof var void while with " + + "yield let static implements interface package private protected public await arguments eval undefined" + ).split(" "), +); +const IDENTIFIER = /^[A-Za-z_$][A-Za-z0-9_$]*$/; +const INT32 = { min: -2147483648, max: 2147483647 }; + +/** + * Line separators end a `//` comment, and bidi controls reorder what an editor + * shows. JSON.stringify keeps both. + */ +const UNSAFE_IN_SOURCE = + /[\u2028\u2029\u200e\u200f\u202a-\u202e\u2066-\u2069]/g; +const escaped = (char: string) => + `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`; +const q = (value: string) => + JSON.stringify(value).replace(UNSAFE_IN_SOURCE, escaped); +/** + * Catalog text for a `//` comment or a terminal line. A quoted Postgres name can + * hold a newline, which would end the comment and turn the rest of the name + * into code. + */ +export const shown = (text: string) => + Array.from(text, (char) => { + const code = char.charCodeAt(0); + return code < 0x20 || code === 0x7f ? escaped(char) : char; + }) + .join("") + .replace(UNSAFE_IN_SOURCE, escaped); +/** Postgres keeps the first 63 bytes of a name, cut on a character boundary. */ +function pgName(name: string): string { + let bytes = 0; + let kept = ""; + for (const char of name) { + bytes += Buffer.byteLength(char); + if (bytes > 63) break; + kept += char; + } + return kept; +} +const key = (name: string) => (IDENTIFIER.test(name) ? name : q(name)); +const access = (variable: string, name: string) => + IDENTIFIER.test(name) ? `${variable}.${name}` : `${variable}[${q(name)}]`; +const list = (names: readonly string[]) => names.join(", "); + +function baseType( + column: IntrospectedColumn, + schemaName: string, +): BaseType | string { + const type = column.type; + if (column.enum) { + if (column.enum.schema !== schemaName) { + return `enum type "${column.enum.schema}.${column.enum.name}" lives outside schema "${schemaName}"`; + } + if (column.enum.values.length === 0) { + return `enum type "${column.enum.name}" has no values`; + } + return { + call: `enumType(${q(column.enum.name)}, [${column.enum.values.map(q).join(", ")}])`, + importName: "", + storage: "enum", + }; + } + switch (type) { + case "integer": + return { call: "integer()", importName: "integer", storage: "integer" }; + case "bigint": + return { call: "bigint()", importName: "bigint", storage: "bigint" }; + case "text": + return { call: "text()", importName: "text", storage: "text" }; + case "character varying": + return { + call: "text()", + importName: "text", + storage: "text", + approximation: + "character varying without a length is declared as text()", + }; + case "boolean": + return { call: "boolean()", importName: "boolean", storage: "boolean" }; + case "uuid": + return { call: "uuid()", importName: "uuid", storage: "uuid" }; + case "jsonb": + return { call: "jsonb()", importName: "jsonb", storage: "jsonb" }; + } + const varchar = /^character varying\((\d+)\)$/.exec(type); + if (varchar) { + const length = Number(varchar[1]); + return { + call: `varchar(${length})`, + importName: "varchar", + storage: "varchar", + varcharLength: length, + }; + } + const timestamp = /^timestamp(\(\d\))? (with|without) time zone$/.exec(type); + if (timestamp) { + return { + call: + timestamp[2] === "with" + ? "timestamp({ withTimezone: true })" + : "timestamp()", + importName: "timestamp", + storage: "timestamp", + ...(timestamp[1] + ? { approximation: `${type} is declared without its precision` } + : {}), + }; + } + return `type "${type}" is not supported`; +} + +const isSequenceDefault = (expr: string | null) => + !!expr && /^nextval\(/.test(expr); + +/** A builder default call, or why the database default cannot be expressed. */ +function defaultCall( + expr: string, + base: BaseType, + enumValues: readonly string[] | undefined, +): { call: string } | { skip: string } { + const unsupported = { skip: `default ${expr} is not supported` }; + switch (base.storage) { + case "timestamp": + return /^(now\(\)|CURRENT_TIMESTAMP)$/i.test(expr) + ? { call: ".defaultNow()" } + : unsupported; + case "uuid": + return expr === "gen_random_uuid()" + ? { call: ".defaultRandom()" } + : unsupported; + case "boolean": + return expr === "true" || expr === "false" + ? { call: `.default(${expr})` } + : unsupported; + case "integer": { + const match = /^\(?'?(-?\d+)'?(?:::integer)?\)?$/.exec(expr); + const value = match ? Number(match[1]) : Number.NaN; + return Number.isInteger(value) && value >= INT32.min && value <= INT32.max + ? { call: `.default(${value})` } + : unsupported; + } + case "text": + case "varchar": + case "enum": { + const match = /^'((?:[^']|'')*)'(?:::[\w." ]+)?$/.exec(expr); + if (!match) return unsupported; + const value = (match[1] ?? "").replace(/''/g, "'"); + if ( + base.storage === "varchar" && + value.length > (base.varcharLength ?? 0) + ) { + return unsupported; + } + if (base.storage === "enum" && !enumValues?.includes(value)) { + return unsupported; + } + return { call: `.default(${q(value)})` }; + } + default: + // The builder accepts no literal defaults for bigint and jsonb. + return unsupported; + } +} + +function importLine(names: readonly string[], source: string): string { + const line = `import { ${names.join(", ")} } from ${q(source)};`; + if (line.length <= 80) return line; + return `import {\n${names.map((name) => ` ${name},`).join("\n")}\n} from ${q(source)};`; +} + +function makeSkip( + table: string, + column: string | undefined, + reason: string, +): Skip { + return { + table: shown(table), + ...(column === undefined ? {} : { column: shown(column) }), + reason: shown(reason), + }; +} + +function skipComment(skip: Skip): string { + return `// appkit: skipped ${skip.column ? `column "${skip.column}"` : `table "${skip.table}"`}: ${skip.reason}`; +} + +/** Defaults the builder allows only on uuid / timestamp columns, never on fk(). */ +const TYPED_DEFAULTS = new Set([".defaultRandom()", ".defaultNow()"]); + +/** The storage an fk() column inherits from its target. */ +function mirrored(target: BaseType): string { + if (target.storage === "id") return "integer()"; + if (target.storage === "bigid") return "bigint()"; + return target.call; +} + +/** + * Render pg_catalog tables as a `defineSchema` source the builder accepts. + * What the builder cannot express is left out with an `// appkit: skipped …` + * comment; what it declares differently (a timestamp without its precision, + * say) is listed in `approximated`. + */ +export function renderSchemaSource( + tables: readonly IntrospectedTable[], + schemaName: string, + options: RenderOptions = {}, +): SchemaSource { + const skipped: Skip[] = []; + const unexpressed: Unexpressed[] = []; + const warnings: string[] = []; + const approximated: string[] = []; + /** Constraints whose database name is not the one migrations use: "db (migrations)". */ + const renamed: string[] = []; + const tableComments: string[] = []; + const qualified = (table: string, column?: string) => + `${schemaName}.${table}${column ? `.${column}` : ""}`; + const skipTable = (table: string, reason: string) => { + const skip = makeSkip(table, undefined, reason); + skipped.push(skip); + unexpressed.push({ kind: "table", name: table, reason }); + tableComments.push(skipComment(skip)); + }; + + // 1. Tables the builder can key: one primary-key column, a usable name. + const names = new Set(tables.map((table) => table.name)); + const plans = new Map(); + for (const table of tables) { + if (isReservedTableName(table.name)) { + skipTable(table.name, `the name "${table.name}" is reserved by AppKit`); + } else if ( + table.name.endsWith("Relations") && + names.has(table.name.slice(0, -"Relations".length)) + ) { + skipTable( + table.name, + `the name collides with the relation metadata AppKit generates for "${table.name.slice(0, -"Relations".length)}"`, + ); + } else if (table.primaryKey.length === 0) { + skipTable(table.name, "it has no primary key"); + } else if (table.primaryKey.length > 1) { + skipTable( + table.name, + `composite primary key (${list(table.primaryKey)}) is not supported`, + ); + } else { + plans.set(table.name, { + table, + columns: [], + comments: [], + body: [], + variable: "", + }); + } + } + + // 2. Columns. A table whose primary-key column cannot be expressed is skipped whole. + for (const plan of [...plans.values()]) { + const { table } = plan; + const pk = table.primaryKey[0]; + const tableSkips: Skip[] = []; + const tableWarnings: string[] = []; + const tableApproximated: string[] = []; + const tableRenamed: string[] = []; + const columnReasons = new Map(); + /** Single-column unique constraints: column → constraint name. */ + const singleUnique = new Map( + table.unique + .filter((unique) => unique.columns.length === 1) + .map((unique) => [unique.columns[0] as string, unique.name]), + ); + const tableUnexpressed: Unexpressed[] = []; + for (const unique of table.unique.filter((u) => u.columns.length > 1)) { + const reason = `unique constraint (${list(unique.columns)}) spans several columns, which is not supported`; + const skip = makeSkip(table.name, undefined, reason); + tableSkips.push(skip); + tableUnexpressed.push({ + kind: "constraint", + name: table.name, + member: unique.name, + reason, + }); + plan.comments.push(`// appkit: skipped ${skip.reason}`); + } + + for (const column of table.columns) { + const skipColumn = (reason: string) => { + const skip = makeSkip(table.name, column.name, reason); + tableSkips.push(skip); + tableUnexpressed.push({ + kind: "column", + name: table.name, + member: column.name, + reason, + }); + columnReasons.set(column.name, reason); + plan.body.push(skipComment(skip)); + const generatesValue = + column.identity !== "" || column.generated || !!column.defaultExpr; + if (column.notNull && !generatesValue) { + tableWarnings.push( + `${qualified(table.name, column.name)} is NOT NULL with no default but is not in schema.ts (${reason}). Inserts through the API will fail until it gets a default, becomes nullable, or is dropped.`, + ); + } + }; + + if (isReservedColumnName(column.name)) { + skipColumn(`the name "${column.name}" is reserved by AppKit`); + continue; + } + if (column.generated) { + skipColumn("generated columns are not supported"); + continue; + } + const base = baseType(column, schemaName); + if (typeof base === "string") { + skipColumn(base); + continue; + } + const isPrimaryKey = column.name === pk; + if (column.identity !== "" && !isPrimaryKey) { + skipColumn( + "identity columns other than the primary key are not supported", + ); + continue; + } + + const columnPlan: ColumnPlan = { + column, + base, + isPrimaryKey, + isUnique: singleUnique.has(column.name) && !isPrimaryKey, + comments: [], + }; + if (base.approximation) { + tableApproximated.push( + `${qualified(table.name, column.name)}: ${base.approximation}`, + ); + } + const uniqueName = singleUnique.get(column.name); + const migrationsUnique = pgName(`${table.name}_${column.name}_unique`); + if (columnPlan.isUnique && uniqueName !== migrationsUnique) { + tableRenamed.push(`${uniqueName} (${migrationsUnique})`); + } + const generatesKey = + column.identity !== "" || isSequenceDefault(column.defaultExpr); + if ( + isPrimaryKey && + generatesKey && + (base.storage === "integer" || base.storage === "bigint") + ) { + // A serial or identity key is the builder's server-generated id()/bigid(). + columnPlan.base = + base.storage === "integer" + ? { call: "id()", importName: "id", storage: "id" } + : { call: "bigid()", importName: "bigid", storage: "bigid" }; + if (column.identity !== "d") { + tableApproximated.push( + `${qualified(table.name, column.name)}: ${column.identity === "a" ? "GENERATED ALWAYS AS IDENTITY" : "a serial key"} is declared as ${columnPlan.base.call}, an identity generated BY DEFAULT`, + ); + } + } else if (column.defaultExpr) { + const parsed = defaultCall( + column.defaultExpr, + base, + column.enum?.values, + ); + if ("call" in parsed) { + columnPlan.defaultCall = parsed.call; + } else { + const skip = makeSkip(table.name, column.name, parsed.skip); + tableSkips.push(skip); + tableUnexpressed.push({ + kind: "default", + name: table.name, + member: column.name, + reason: parsed.skip, + }); + columnPlan.comments.push(`// appkit: skipped ${skip.reason}`); + if (column.notNull) { + tableWarnings.push( + `${qualified(table.name, column.name)}: its ${parsed.skip.replace(/ is not supported$/, "")} is not in schema.ts, so the API requires a value for it on insert.`, + ); + } + } + } + plan.columns.push(columnPlan); + plan.body.push(columnPlan); + } + + if (!plan.columns.some((c) => c.isPrimaryKey)) { + plans.delete(table.name); + skipTable( + table.name, + `its primary key column "${pk}" cannot be expressed (${columnReasons.get(pk as string) ?? "unsupported"})`, + ); + continue; + } + skipped.push(...tableSkips); + unexpressed.push(...tableUnexpressed); + warnings.push(...tableWarnings); + if (table.partitioned) { + tableApproximated.push( + `${qualified(table.name)}: a partitioned table is declared as a plain table`, + ); + } + for (const [one, many, names] of [ + [ + "CHECK or EXCLUDE constraint", + "CHECK or EXCLUDE constraints", + table.checks, + ], + ["index", "indexes", table.indexes], + ] as const) { + if (names.length > 0) { + tableApproximated.push( + `${qualified(table.name)}: ${names.length === 1 ? `${one} ${names[0]} is` : `${many} ${list(names)} are`} not declared`, + ); + } + } + approximated.push(...tableApproximated); + renamed.push(...tableRenamed); + } + + // 3. Foreign keys the builder can express; the rest stay plain columns. + const edges = new Map>(); + const reaches = (from: string, to: string): boolean => { + const seen = new Set(); + const stack = [from]; + while (stack.length > 0) { + const node = stack.pop() as string; + if (node === to) return true; + if (seen.has(node)) continue; + seen.add(node); + stack.push(...(edges.get(node) ?? [])); + } + return false; + }; + /** Why the builder cannot express this FK; null when it can. */ + const fkProblem = ( + plan: TablePlan, + foreignKey: IntrospectedForeignKey, + source: ColumnPlan | undefined, + seenTargets: ReadonlySet, + ): string | null => { + if (foreignKey.columns.length !== 1) { + return "foreign keys over several columns are not supported"; + } + if (!source) return "its column is skipped"; + if (foreignKey.schema !== schemaName) { + return `it references schema "${foreignKey.schema}"`; + } + if (foreignKey.table === plan.table.name) { + // The builder takes one with a TableHandle annotation, which introspect does not write. + return "introspect does not write self-references"; + } + const target = plans.get(foreignKey.table); + if (!target) return `table "${foreignKey.table}" is skipped`; + const targetColumn = target.columns.find( + (c) => c.column.name === foreignKey.refColumns[0], + ); + if (!targetColumn) { + return `column ${foreignKey.table}.${foreignKey.refColumns[0]} is skipped`; + } + if (!targetColumn.isPrimaryKey && !targetColumn.isUnique) { + return `${foreignKey.table}.${foreignKey.refColumns[0]} is not a primary key or single-column unique key`; + } + if (seenTargets.has(foreignKey.table)) { + return `a second foreign key to "${foreignKey.table}" would make the relation ambiguous`; + } + if (plan.table.columns.some((c) => c.name === foreignKey.table)) { + return `the relation "${foreignKey.table}" would clash with the column of the same name`; + } + if (target.table.columns.some((c) => c.name === plan.table.name)) { + return `the reverse relation "${plan.table.name}" would clash with column ${foreignKey.table}.${plan.table.name}`; + } + if (reaches(foreignKey.table, plan.table.name)) { + // TypeScript cannot infer a cycle of fk() thunks without TableHandle annotations (TS7022). + return "it would close a cycle of foreign keys"; + } + if (mirrored(targetColumn.base) !== source.base.call) { + return `its type ${source.base.call} differs from ${foreignKey.table}.${foreignKey.refColumns[0]}`; + } + const actions = [foreignKey.onDelete, foreignKey.onUpdate]; + if (actions.includes("set null") && source.column.notNull) { + return "SET NULL on a NOT NULL column"; + } + const fkDefault = + source.defaultCall && !TYPED_DEFAULTS.has(source.defaultCall); + if (actions.includes("set default") && !fkDefault) { + return "SET DEFAULT without a default the builder can express"; + } + return null; + }; + + for (const plan of plans.values()) { + const targets = new Set(); + for (const foreignKey of plan.table.foreignKeys) { + const source = plan.columns.find( + (c) => c.column.name === foreignKey.columns[0], + ); + const problem = fkProblem(plan, foreignKey, source, targets); + if (problem) { + const reason = `foreign key to ${foreignKey.table}(${list(foreignKey.refColumns)}) is not kept: ${problem}`; + // Migrations name it after its column; a schema.ts that declared it gave it that name. + const constraintNames = [foreignKey.name]; + if (foreignKey.columns.length === 1) { + constraintNames.push( + pgName( + `${plan.table.name}_${foreignKey.columns[0]}_${foreignKey.table}_${foreignKey.refColumns[0]}_fk`, + ), + ); + } + for (const member of new Set(constraintNames)) { + unexpressed.push({ + kind: "constraint", + name: plan.table.name, + member, + reason, + }); + } + // A skipped column already carries its own comment. + if (source || foreignKey.columns.length > 1) { + const skip = makeSkip( + plan.table.name, + foreignKey.columns.join(", "), + reason, + ); + skipped.push(skip); + (source?.comments ?? plan.comments).push( + `// appkit: skipped ${skip.reason}`, + ); + } + continue; + } + const sourceColumn = source as ColumnPlan; + const target = plans.get(foreignKey.table) as TablePlan; + const targetColumn = target.columns.find( + (c) => c.column.name === foreignKey.refColumns[0], + ) as ColumnPlan; + sourceColumn.base = { ...targetColumn.base }; + if ( + sourceColumn.defaultCall && + TYPED_DEFAULTS.has(sourceColumn.defaultCall) + ) { + // fk() takes no defaultRandom()/defaultNow(); the key matters more than the default. + const reason = `default ${sourceColumn.column.defaultExpr} is not supported on a foreign key column`; + const skip = makeSkip( + plan.table.name, + sourceColumn.column.name, + reason, + ); + skipped.push(skip); + unexpressed.push({ + kind: "default", + name: plan.table.name, + member: sourceColumn.column.name, + reason, + }); + sourceColumn.comments.push(`// appkit: skipped ${skip.reason}`); + delete sourceColumn.defaultCall; + if (sourceColumn.column.notNull) { + warnings.push( + `${qualified(plan.table.name, sourceColumn.column.name)}: its default ${sourceColumn.column.defaultExpr} is not in schema.ts, so the API requires a value for it on insert.`, + ); + } + } + for (const [flag, what] of [ + [foreignKey.deferrable, "DEFERRABLE"], + [foreignKey.matchFull, "MATCH FULL"], + ] as const) { + if (flag) { + approximated.push( + `${qualified(plan.table.name, sourceColumn.column.name)}: the foreign key is declared without ${what}`, + ); + } + } + const migrationsFk = pgName( + `${plan.table.name}_${sourceColumn.column.name}_${foreignKey.table}_${foreignKey.refColumns[0]}_fk`, + ); + if (foreignKey.name !== migrationsFk) { + renamed.push(`${foreignKey.name} (${migrationsFk})`); + } + sourceColumn.fk = { + table: foreignKey.table, + column: foreignKey.refColumns[0] as string, + actions: [ + ...(foreignKey.onDelete !== "no action" + ? [`.onDelete(${q(foreignKey.onDelete)})`] + : []), + ...(foreignKey.onUpdate !== "no action" + ? [`.onUpdate(${q(foreignKey.onUpdate)})`] + : []), + ], + }; + targets.add(foreignKey.table); + const out = edges.get(plan.table.name) ?? new Set(); + out.add(foreignKey.table); + edges.set(plan.table.name, out); + } + } + + const declaredEnums = new Set( + [...plans.values()].flatMap((plan) => + plan.columns.flatMap((c) => + c.base.storage === "enum" && c.column.enum ? [c.column.enum.name] : [], + ), + ), + ); + const undeclaredEnums = new Set( + tables.flatMap((table) => + table.columns.flatMap((column) => + column.enum?.schema === schemaName && + !declaredEnums.has(column.enum.name) + ? [column.enum.name] + : [], + ), + ), + ); + for (const name of undeclaredEnums) { + unexpressed.push({ + kind: "type", + name, + reason: "only what schema.ts leaves out uses it", + }); + } + + // 4. Declaration order: targets before the tables that reference them. + const ordered: TablePlan[] = []; + const placed = new Set(); + const place = (plan: TablePlan) => { + if (placed.has(plan.table.name)) return; + placed.add(plan.table.name); + for (const target of [...(edges.get(plan.table.name) ?? [])].sort()) { + const targetPlan = plans.get(target); + if (targetPlan) place(targetPlan); + } + ordered.push(plan); + }; + for (const plan of plans.values()) place(plan); + + // 5. Identifiers and imports. + const imports = new Set(["defineSchema"]); + for (const plan of ordered) { + for (const column of plan.columns) { + if (column.fk) imports.add("fk"); + else if (column.base.importName) imports.add(column.base.importName); + } + } + const usesEnum = ordered.some((plan) => + plan.columns.some((c) => !c.fk && c.base.storage === "enum"), + ); + const used = new Set([...imports, "table", "enumType", "schema"]); + for (const plan of ordered) { + let candidate = plan.table.name.replace(/[^A-Za-z0-9_$]/g, "_"); + if (!/^[A-Za-z_$]/.test(candidate)) candidate = `_${candidate}`; + if (used.has(candidate) || JS_RESERVED.has(candidate)) { + candidate = `${candidate}Table`; + } + let variable = candidate; + for (let n = 2; used.has(variable); n++) variable = `${candidate}${n}`; + used.add(variable); + plan.variable = variable; + } + + // 6. Source. + const lines: string[] = [ + `// Generated by \`appkit db introspect\` from schema "${shown(schemaName)}". Edit tables here,`, + "// then run `npx appkit db migrate` on your dev branch.", + importLine( + [...imports].sort(), + options.importSource ?? "@databricks/appkit/beta", + ), + "", + ]; + // Only a schema other than the default is passed to defineSchema(). + const schemaOption = + schemaName === DEFAULT_SCHEMA_NAME + ? undefined + : `{ schemaName: ${q(schemaName)} }`; + if (renamed.length > 0) { + warnings.push( + `Constraint names differ from the ones migrations use: ${list(renamed)}. A later migration that drops or changes one of them fails until it is renamed in the database (ALTER TABLE … RENAME CONSTRAINT).`, + ); + } + const result = (source: string): SchemaSource => ({ + source, + tables: ordered.map((plan) => plan.table.name), + skipped, + unexpressed, + warnings: warnings.map(shown), + approximated: approximated.map(shown), + }); + if (ordered.length === 0) { + lines.push(...tableComments); + lines.push( + `export const schema = defineSchema(() => ({})${schemaOption ? `, ${schemaOption}` : ""});`, + "", + ); + return result(lines.join("\n")); + } + + // The builder's body, indented one level. + const body: string[] = []; + for (const plan of ordered) { + body.push(` const ${plan.variable} = table(${q(plan.table.name)}, {`); + for (const comment of plan.comments) body.push(` ${comment}`); + for (const entry of plan.body) { + if (typeof entry === "string") { + body.push(` ${entry}`); + continue; + } + const column = entry; + for (const comment of column.comments) body.push(` ${comment}`); + const chain: string[] = []; + const head = column.fk + ? `fk(() => ${access((plans.get(column.fk.table) as TablePlan).variable, column.fk.column)})` + : column.base.call; + const serverKey = + !column.fk && + (column.base.storage === "id" || column.base.storage === "bigid"); + if (column.isPrimaryKey && !serverKey) chain.push(".primaryKey()"); + else if (column.column.notNull && !serverKey) chain.push(".notNull()"); + if (column.isUnique) chain.push(".unique()"); + if (column.defaultCall) chain.push(column.defaultCall); + chain.push(...(column.fk?.actions ?? [])); + body.push(` ${key(column.column.name)}: ${head}${chain.join("")},`); + } + body.push(" });", ""); + } + for (const comment of tableComments) body.push(` ${comment}`); + if (tableComments.length > 0) body.push(""); + const entries = ordered.map((plan) => + plan.variable === plan.table.name + ? plan.variable + : `${key(plan.table.name)}: ${plan.variable}`, + ); + body.push(` return { ${entries.join(", ")} };`); + + const builder = `({ table${usesEnum ? ", enum: enumType" : ""} }) =>`; + if (schemaOption) { + lines.push( + "export const schema = defineSchema(", + ` ${builder} {`, + ...body.map((line) => (line ? ` ${line}` : line)), + " },", + ` ${schemaOption},`, + ");", + "", + ); + } else { + lines.push( + `export const schema = defineSchema(${builder} {`, + ...body, + "});", + "", + ); + } + return result(lines.join("\n")); +} diff --git a/packages/appkit/src/database/tooling/introspect/tests/catalog.pg.test.ts b/packages/appkit/src/database/tooling/introspect/tests/catalog.pg.test.ts new file mode 100644 index 000000000..a7334f852 --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/tests/catalog.pg.test.ts @@ -0,0 +1,147 @@ +import { randomUUID } from "node:crypto"; + +import { Pool } from "pg"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; + +import { type CatalogQuery, readCatalog, readUnreadObjects } from "../catalog"; + +// The catalog SQL against a real server. Point it at a scratch database, e.g. +// APPKIT_TEST_PG_URL=postgres://localhost:5432/postgres +const url = process.env.APPKIT_TEST_PG_URL; + +describe.skipIf(!url)("readCatalog against Postgres", () => { + const schema = `appkit_catalog_${randomUUID().slice(0, 8)}`; + let pool: Pool; + const query: CatalogQuery = async ( + text: string, + values: readonly unknown[], + ) => (await pool.query(text, [...values])).rows as T[]; + + beforeAll(async () => { + pool = new Pool({ connectionString: url, max: 1 }); + const s = `"${schema}"`; + await pool.query(` + create schema ${s}; + create table ${s}.__drizzle_migrations (id serial primary key, hash text not null, created_at bigint); + create type ${s}.ticket_status as enum ('open', 'closed'); + create table ${s}.authors ( + id serial primary key, + email varchar(255) not null unique, + created_at timestamptz not null default now() + ); + create table ${s}.posts ( + id integer generated by default as identity primary key, + author_id integer not null references ${s}.authors (id) on delete cascade, + status ${s}.ticket_status not null default 'open', + total integer generated always as (id * 2) stored, + score numeric(10, 2) check (score > 0) + ); + create index posts_status_idx on ${s}.posts (status); + create table ${s}.events (a integer, b integer, primary key (a, b)); + create table ${s}.parts (id integer primary key) partition by range (id); + create table ${s}.parts_1 partition of ${s}.parts for values from (0) to (100); + create table ${s}.refs (id integer primary key, part_id integer references ${s}.parts (id)); + + create view ${s}.active_authors as select * from ${s}.authors; + create materialized view ${s}.mv as select 1 as x; + create sequence ${s}.standalone; + create function ${s}.touch() returns trigger language plpgsql as 'begin return new; end'; + create trigger audit before update on ${s}.authors for each row execute function ${s}.touch(); + alter table ${s}.authors enable row level security; + create policy own on ${s}.authors using (true); + create table ${s}.base (id integer primary key); + create table ${s}.child (extra integer) inherits (${s}.base); + create domain ${s}.positive as integer check (value > 0); + create type ${s}.pair as (a integer, b integer); + create type ${s}.unused_kind as enum ('x'); + `); + }); + afterAll(async () => { + await pool.query(`drop schema if exists "${schema}" cascade`); + await pool.end(); + }); + + test("reads tables, keys, foreign keys, enums, generated and identity columns, and what schema.ts cannot declare", async () => { + const tables = await readCatalog(query, schema); + const byName = Object.fromEntries(tables.map((t) => [t.name, t])); + + expect(tables.map((t) => t.name)).toEqual([ + "authors", + "base", + "child", + "events", + "parts", + "posts", + "refs", + ]); + expect(byName.authors).toMatchObject({ + primaryKey: ["id"], + unique: [{ name: "authors_email_key", columns: ["email"] }], + columns: [ + { + name: "id", + type: "integer", + notNull: true, + identity: "", + defaultExpr: `nextval('${schema}.authors_id_seq'::regclass)`, + }, + { name: "email", type: "character varying(255)", notNull: true }, + { + name: "created_at", + type: "timestamp with time zone", + defaultExpr: "now()", + }, + ], + }); + expect(byName.posts).toMatchObject({ + primaryKey: ["id"], + foreignKeys: [ + { + name: "posts_author_id_fkey", + columns: ["author_id"], + schema, + table: "authors", + refColumns: ["id"], + onDelete: "cascade", + onUpdate: "no action", + }, + ], + checks: ["posts_score_check"], + indexes: ["posts_status_idx"], + }); + const posts = Object.fromEntries( + (byName.posts?.columns ?? []).map((c) => [c.name, c]), + ); + expect(posts.id?.identity).toBe("d"); + expect(posts.total?.generated).toBe(true); + expect(posts.status?.enum).toEqual({ + name: "ticket_status", + schema, + values: ["open", "closed"], + }); + expect(byName.events?.primaryKey).toEqual(["a", "b"]); + expect(byName.parts?.partitioned).toBe(true); + // One foreign key, not one more per partition of its target. + expect(byName.refs?.foreignKeys.map((fk) => fk.table)).toEqual(["parts"]); + }); + + test("names what it does not read, leaving out sequences that belong to a column", async () => { + const unread = await readUnreadObjects(query, schema); + + expect([...unread].sort()).toEqual( + [ + "domain positive", + "function touch()", + "inheritance of child from base", + "materialized view mv", + "policy own on authors", + "row-level security on authors", + "sequence standalone", + "trigger audit on authors", + "type pair", + "type unused_kind", + "view active_authors", + ].sort(), + ); + }); +}); diff --git a/packages/appkit/src/database/tooling/introspect/tests/catalog.test.ts b/packages/appkit/src/database/tooling/introspect/tests/catalog.test.ts new file mode 100644 index 000000000..ca7f64170 --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/tests/catalog.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, test } from "vitest"; + +import { buildTables, readCatalog } from "../catalog"; +import { catalogResponder, catalogRows } from "./fixture"; + +describe("readCatalog", () => { + test("maps catalog rows to tables, columns, keys, foreign keys and enums", async () => { + const { respond } = catalogResponder(); + const [authors, posts] = await readCatalog(respond, "app"); + + expect(authors).toEqual({ + name: "authors", + partitioned: false, + primaryKey: ["id"], + unique: [{ name: "authors_email_key", columns: ["email"] }], + foreignKeys: [], + checks: [], + indexes: [], + columns: [ + { + name: "id", + type: "integer", + notNull: true, + identity: "", + generated: false, + defaultExpr: "nextval('app.authors_id_seq'::regclass)", + }, + { + name: "email", + type: "character varying(255)", + notNull: true, + identity: "", + generated: false, + defaultExpr: null, + }, + { + name: "created_at", + type: "timestamp with time zone", + notNull: true, + identity: "", + generated: false, + defaultExpr: "now()", + }, + ], + }); + expect(posts?.primaryKey).toEqual(["id"]); + expect(posts?.foreignKeys).toEqual([ + { + name: "posts_author_id_fkey", + columns: ["author_id"], + schema: "app", + table: "authors", + refColumns: ["id"], + onDelete: "cascade", + onUpdate: "no action", + deferrable: false, + matchFull: false, + }, + ]); + expect(posts?.indexes).toEqual(["posts_status_idx"]); + expect(posts?.columns.find((c) => c.name === "id")?.identity).toBe("d"); + expect(posts?.columns.find((c) => c.name === "total")?.generated).toBe( + true, + ); + expect(posts?.columns.find((c) => c.name === "status")?.enum).toEqual({ + name: "ticket_status", + schema: "app", + values: ["open", "closed"], + }); + }); + + test("partitioned tables, CHECK and EXCLUDE constraints, indexes, DEFERRABLE and MATCH FULL are kept for the report", () => { + const row = (kind: "c" | "x" | "f", name: string, extra = {}) => ({ + ...catalogRows.constraints[2], + kind, + name, + ...extra, + }); + const [table] = buildTables( + [{ name: "posts", partitioned: true }], + [], + [ + row("c", "posts_score_check"), + row("x", "posts_no_overlap"), + row("f", "posts_author_id_fkey", { + deferrable: true, + match_type: "f", + }), + ] as never, + [ + { table_name: "posts", name: "posts_status_idx" }, + { table_name: "authors", name: "authors_name_idx" }, + ], + ); + expect(table).toMatchObject({ + partitioned: true, + checks: ["posts_score_check", "posts_no_overlap"], + indexes: ["posts_status_idx"], + foreignKeys: [{ deferrable: true, matchFull: true }], + }); + }); + + test("stored and virtual (PG18) generated columns are both generated", () => { + const [table] = buildTables( + [{ name: "t", partitioned: false }], + ["s", "v", ""].map((generated, i) => ({ + ...catalogRows.columns[0], + table_name: "t", + name: `c${i}`, + generated, + })) as never, + [], + ); + expect(table?.columns.map((c) => c.generated)).toEqual([true, true, false]); + }); + + test("referential action letters map to builder actions", () => { + const fkRow = (on_delete: string, on_update: string) => ({ + ...catalogRows.constraints[2], + on_delete, + on_update, + }); + const [table] = buildTables( + [{ name: "posts", partitioned: false }], + [], + ["r", "c", "n", "d", "a"].map((letter) => fkRow(letter, letter)) as never, + ); + expect(table?.foreignKeys.map((fk) => [fk.onDelete, fk.onUpdate])).toEqual([ + ["restrict", "restrict"], + ["cascade", "cascade"], + ["set null", "set null"], + ["set default", "set default"], + ["no action", "no action"], + ]); + }); +}); diff --git a/packages/appkit/src/database/tooling/introspect/tests/fixture.ts b/packages/appkit/src/database/tooling/introspect/tests/fixture.ts new file mode 100644 index 000000000..2f8b2f9ff --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/tests/fixture.ts @@ -0,0 +1,293 @@ +import path from "node:path"; + +import type { + CatalogQuery, + ColumnRow, + ConstraintRow, + IndexRow, + IntrospectedColumn, + IntrospectedForeignKey, + IntrospectedTable, + IntrospectedUnique, + TableRow, +} from "../catalog"; + +/** The builder by absolute path, so rendered sources load from a temp root. */ +export const BUILDER = path.resolve( + import.meta.dirname, + "../../../schema-builder/index.ts", +); + +export const col = ( + name: string, + type: string, + extra: Partial = {}, +): IntrospectedColumn => ({ + name, + type, + notNull: false, + identity: "", + generated: false, + defaultExpr: null, + ...extra, +}); + +/** A catalog table; what a test does not set is empty. */ +export const tableOf = ( + table: Pick & + Partial, +): IntrospectedTable => ({ + partitioned: false, + primaryKey: [], + unique: [], + foreignKeys: [], + checks: [], + indexes: [], + ...table, +}); + +export const foreignKey = ( + key: Omit & + Partial, +): IntrospectedForeignKey => ({ deferrable: false, matchFull: false, ...key }); + +/** A unique constraint with the name Postgres gives it. */ +export const uniqueKey = ( + table: string, + ...columns: string[] +): IntrospectedUnique => ({ + name: `${table}_${columns.join("_")}_key`, + columns, +}); + +const status = { + name: "ticket_status", + schema: "app", + values: ["open", "closed"], +}; + +/** Supported types, defaults, FKs and enums, plus one of each skip. Listed out of dependency order. */ +export const richTables: IntrospectedTable[] = [ + tableOf({ + name: "posts", + primaryKey: ["id"], + unique: [], + foreignKeys: [ + foreignKey({ + name: "posts_author_id_fkey", + columns: ["author_id"], + schema: "app", + table: "authors", + refColumns: ["id"], + onDelete: "cascade", + onUpdate: "no action", + }), + foreignKey({ + name: "posts_editor_id_fkey", + columns: ["editor_id"], + schema: "app", + table: "authors", + refColumns: ["id"], + onDelete: "set null", + onUpdate: "no action", + }), + ], + columns: [ + col("id", "uuid", { notNull: true, defaultExpr: "gen_random_uuid()" }), + col("author_id", "integer", { notNull: true }), + col("editor_id", "integer"), + col("status", "app.ticket_status", { + notNull: true, + defaultExpr: "'open'::app.ticket_status", + enum: status, + }), + col("body", "jsonb", { defaultExpr: "'{}'::jsonb" }), + col("views", "bigint", { notNull: true, defaultExpr: "0" }), + col("my col", "character varying"), + ], + }), + tableOf({ + name: "authors", + primaryKey: ["id"], + unique: [uniqueKey("authors", "email")], + foreignKeys: [], + columns: [ + col("id", "integer", { + notNull: true, + defaultExpr: "nextval('app.authors_id_seq'::regclass)", + }), + col("email", "character varying(255)", { notNull: true }), + col("bio", "text", { defaultExpr: "'it''s me'::text" }), + col("active", "boolean", { notNull: true, defaultExpr: "true" }), + col("score", "numeric(10,2)", { notNull: true }), + col("rank", "integer", { defaultExpr: "'-1'::integer" }), + col("created_at", "timestamp with time zone", { + notNull: true, + defaultExpr: "now()", + }), + col("seen_at", "timestamp(3) without time zone"), + ], + }), + tableOf({ + name: "events", + primaryKey: ["a", "b"], + unique: [], + foreignKeys: [], + columns: [ + col("a", "integer", { notNull: true }), + col("b", "integer", { notNull: true }), + ], + }), + tableOf({ + name: "logs", + primaryKey: [], + unique: [], + foreignKeys: [], + columns: [col("line", "text")], + }), + tableOf({ + name: "default", + primaryKey: ["id"], + unique: [], + foreignKeys: [], + columns: [col("id", "bigint", { notNull: true, identity: "d" })], + }), +]; + +export const columnRow = ( + table_name: string, + name: string, + type: string, + extra: Partial = {}, +): ColumnRow => ({ + table_name, + name, + type, + not_null: false, + identity: "", + generated: "", + default_expr: null, + type_kind: "b", + type_schema: "pg_catalog", + type_name: type, + enum_values: null, + ...extra, +}); + +/** What the three pg_catalog queries return for a small hand-made schema. */ +export const catalogRows = { + tables: [ + { name: "authors", partitioned: false }, + { name: "posts", partitioned: false }, + ] satisfies TableRow[], + columns: [ + columnRow("authors", "id", "integer", { + not_null: true, + default_expr: "nextval('app.authors_id_seq'::regclass)", + }), + columnRow("authors", "email", "character varying(255)", { + not_null: true, + }), + columnRow("authors", "created_at", "timestamp with time zone", { + not_null: true, + default_expr: "now()", + }), + columnRow("posts", "id", "integer", { not_null: true, identity: "d" }), + columnRow("posts", "author_id", "integer", { not_null: true }), + columnRow("posts", "status", "app.ticket_status", { + not_null: true, + default_expr: "'open'::app.ticket_status", + type_kind: "e", + type_schema: "app", + type_name: "ticket_status", + enum_values: ["open", "closed"], + }), + columnRow("posts", "score", "numeric(10,2)", { not_null: true }), + columnRow("posts", "total", "integer", { + generated: "s", + default_expr: "1", + }), + ] satisfies ColumnRow[], + constraints: [ + { + table_name: "authors", + name: "authors_email_key", + kind: "u", + columns: ["email"], + ref_schema: null, + ref_table: null, + ref_columns: [], + on_delete: " ", + on_update: " ", + }, + { + table_name: "authors", + name: "authors_pkey", + kind: "p", + columns: ["id"], + ref_schema: null, + ref_table: null, + ref_columns: [], + on_delete: " ", + on_update: " ", + }, + { + table_name: "posts", + name: "posts_author_id_fkey", + kind: "f", + columns: ["author_id"], + ref_schema: "app", + ref_table: "authors", + ref_columns: ["id"], + on_delete: "c", + on_update: "a", + }, + { + table_name: "posts", + name: "posts_pkey", + kind: "p", + columns: ["id"], + ref_schema: null, + ref_table: null, + ref_columns: [], + on_delete: " ", + on_update: " ", + }, + ] satisfies ConstraintRow[], + indexes: [ + { table_name: "posts", name: "posts_status_idx" }, + ] satisfies IndexRow[], +}; + +/** A CatalogQuery answering the catalog queries from rows, and the unread-objects query from labels. */ +export function catalogResponder( + rows: { + readonly tables: readonly TableRow[]; + readonly columns: readonly ColumnRow[]; + readonly constraints: readonly ConstraintRow[]; + readonly indexes: readonly IndexRow[]; + } = catalogRows, + unread: readonly string[] = [], +) { + const calls: Array<{ text: string; values: readonly unknown[] }> = []; + const respond: CatalogQuery = async ( + text: string, + values: readonly unknown[], + ) => { + calls.push({ text, values }); + // The unread and index queries mention what the others match on, so they go first. + const result = text.includes("pg_catalog.pg_depend") + ? unread.map((label) => ({ label })) + : text.includes("pg_catalog.pg_index") + ? rows.indexes + : text.includes("pg_catalog.pg_constraint") + ? rows.constraints + : text.includes("format_type") + ? rows.columns + : text.includes("select c.relname::text as name") + ? rows.tables + : []; + return result as unknown as T[]; + }; + return { calls, respond }; +} diff --git a/packages/appkit/src/database/tooling/introspect/tests/golden/rich.schema.ts.txt b/packages/appkit/src/database/tooling/introspect/tests/golden/rich.schema.ts.txt new file mode 100644 index 000000000..2fe3e54d5 --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/tests/golden/rich.schema.ts.txt @@ -0,0 +1,51 @@ +// Generated by `appkit db introspect` from schema "app". Edit tables here, +// then run `npx appkit db migrate` on your dev branch. +import { + bigid, + bigint, + boolean, + defineSchema, + fk, + id, + integer, + jsonb, + text, + timestamp, + uuid, + varchar, +} from "@databricks/appkit/beta"; + +export const schema = defineSchema(({ table, enum: enumType }) => { + const authors = table("authors", { + id: id(), + email: varchar(255).notNull().unique(), + bio: text().default("it's me"), + active: boolean().notNull().default(true), + // appkit: skipped column "score": type "numeric(10,2)" is not supported + rank: integer().default(-1), + created_at: timestamp({ withTimezone: true }).notNull().defaultNow(), + seen_at: timestamp(), + }); + + const posts = table("posts", { + id: uuid().primaryKey().defaultRandom(), + author_id: fk(() => authors.id).notNull().onDelete("cascade"), + // appkit: skipped foreign key to authors(id) is not kept: a second foreign key to "authors" would make the relation ambiguous + editor_id: integer(), + status: enumType("ticket_status", ["open", "closed"]).notNull().default("open"), + // appkit: skipped default '{}'::jsonb is not supported + body: jsonb(), + // appkit: skipped default 0 is not supported + views: bigint().notNull(), + "my col": text(), + }); + + const defaultTable = table("default", { + id: bigid(), + }); + + // appkit: skipped table "events": composite primary key (a, b) is not supported + // appkit: skipped table "logs": it has no primary key + + return { authors, posts, default: defaultTable }; +}); diff --git a/packages/appkit/src/database/tooling/introspect/tests/introspect.test.ts b/packages/appkit/src/database/tooling/introspect/tests/introspect.test.ts new file mode 100644 index 000000000..dc3fe1bb6 --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/tests/introspect.test.ts @@ -0,0 +1,741 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +const { markMigrationsApplied, resolveEnvBranch, withBranchPool, writeTypes } = + vi.hoisted(() => ({ + markMigrationsApplied: vi.fn(), + resolveEnvBranch: vi.fn(), + withBranchPool: vi.fn(), + writeTypes: vi.fn(), + })); + +vi.mock("../../context", async (importOriginal) => ({ + ...(await importOriginal()), + withBranchPool, +})); +vi.mock("../../guard", async (importOriginal) => ({ + ...(await importOriginal()), + resolveEnvBranch, +})); +vi.mock("../../../migrations/apply", async (importOriginal) => ({ + ...(await importOriginal()), + markMigrationsApplied, +})); +vi.mock("../../../../type-generator/database", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("../../../../type-generator/database") + >()), + writeDatabaseTypes: writeTypes, +})); + +import { loadDefaultDatabaseSchema } from "../../../../plugins/database/load-schema"; +import { readJournal } from "../../../migrations/paths"; +import { generateMigrationFiles } from "../../migrations"; +import { cleanupTempApps, fakeContext, tempApp } from "../../tests/fakes"; +import type { ColumnRow, ConstraintRow, TableRow } from "../catalog"; +import { IntrospectRefusedError, runIntrospect } from "../index"; +import { BUILDER, catalogResponder, columnRow } from "./fixture"; + +const { writeDatabaseTypes } = await vi.importActual< + typeof import("../../../../type-generator/database") +>("../../../../type-generator/database"); + +const SCHEMA = "config/database/schema.ts"; +const MIGRATIONS = "config/database/migrations"; +const TYPES = "shared/appkit-types/database.d.ts"; +const DEV = "projects/p/branches/dev-alice-3f9a2c"; +const PRODUCTION = "projects/p/branches/production"; + +const previousSchema = `import { defineSchema, id, text } from ${JSON.stringify(BUILDER)}; +export const schema = defineSchema(({ table }) => ({ + authors: table("authors", { id: id(), email: text() }), +})); +`; + +const onDevBranch = { branch: { name: DEV }, kind: "dev" }; +const onProduction = { + branch: { name: PRODUCTION, status: { default: true } }, + kind: "default", + reason: "it is the project's default branch", +}; + +let catalog: ReturnType; +/** `when` of each migration the branch recorded; empty: no journal table. */ +let recorded: number[]; +const lastWhen = (root: string) => + readJournal(path.join(root, MIGRATIONS))?.entries.at(-1)?.when ?? 0; +const pool = { + query: vi.fn(async (text: string, values: unknown[]) => { + // readMigrationDrift: whether the journal table exists, then its rows. + if (text.includes("c.relname = $2")) { + return { rows: recorded.length > 0 ? [{ "?column?": 1 }] : [] }; + } + if (text.includes("select hash, created_at")) { + return { + rows: recorded.map((when) => ({ hash: "h", created_at: String(when) })), + }; + } + return { rows: await catalog.respond(text, values) }; + }), + end: vi.fn(async () => undefined), +}; + +beforeEach(() => { + catalog = catalogResponder(); + recorded = []; + resolveEnvBranch.mockReset().mockResolvedValue(onDevBranch); + withBranchPool + .mockReset() + .mockImplementation( + async (_ctx: unknown, fn: (p: typeof pool) => Promise) => { + try { + return await fn(pool); + } finally { + await pool.end(); + } + }, + ); + markMigrationsApplied + .mockReset() + .mockImplementation( + async ( + _pool: unknown, + options: { migrationsFolder: string }, + tags: readonly string[], + ) => { + const entries = readJournal(options.migrationsFolder)?.entries ?? []; + for (const tag of tags) { + recorded.push(entries.find((entry) => entry.tag === tag)?.when ?? 0); + } + }, + ); + writeTypes.mockReset().mockImplementation(writeDatabaseTypes); + pool.end.mockClear(); +}); +afterEach(cleanupTempApps); + +const authorsPkey: ConstraintRow = { + table_name: "authors", + name: "authors_pkey", + kind: "p", + columns: ["id"], + ref_schema: null, + ref_table: null, + ref_columns: [], + on_delete: " ", + on_update: " ", +}; + +/** A catalog with authors(id) plus the given authors columns, and any extra tables. */ +const authorsOnly = ( + columns: ColumnRow[] = [], + extra: { tables: TableRow[]; columns: ColumnRow[] } = { + tables: [], + columns: [], + }, +) => + catalogResponder({ + tables: [{ name: "authors", partitioned: false }, ...extra.tables], + columns: [ + columnRow("authors", "id", "integer", { not_null: true, identity: "d" }), + ...columns, + ...extra.columns, + ], + constraints: [authorsPkey], + indexes: [], + }); + +/** Commit a migration of the current schema.ts; `applied`: the branch has recorded it. */ +async function commitMigration(root: string, applied = true): Promise { + await generateMigrationFiles({ + folder: path.join(root, MIGRATIONS), + schema: await loadDefaultDatabaseSchema(root), + interactive: false, + }); + if (applied) recorded.push(lastWhen(root)); +} + +/** Every file under config/, with its contents. */ +function tree(root: string): Record { + const files: Record = {}; + const walk = (dir: string) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else files[path.relative(root, full)] = fs.readFileSync(full, "utf-8"); + } + }; + if (fs.existsSync(path.join(root, "config"))) walk(path.join(root, "config")); + return files; +} + +const ctxFor = (root: string) => fakeContext({ root }); +const introspect = (root: string) => + runIntrospect(ctxFor(root), { interactive: false, importSource: BUILDER }); +/** The error a run fails with; a run that succeeds fails the test. */ +const rejection = (run: Promise) => + run.then( + () => { + throw new Error("expected introspect to fail"); + }, + (caught: unknown) => caught as Error, + ); +const refusal = (root: string) => rejection(introspect(root)); + +describe("runIntrospect", () => { + test("without history: schema.ts (+ .bak), 0000_baseline recorded as applied, database.d.ts", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + + const report = await introspect(root); + + expect(catalog.calls.every((call) => call.values[0] === "app")).toBe(true); + expect(fs.readFileSync(path.join(root, `${SCHEMA}.bak`), "utf-8")).toBe( + previousSchema, + ); + const written = fs.readFileSync(path.join(root, SCHEMA), "utf-8"); + expect(written).toContain('const authors = table("authors", {'); + expect(written).toContain( + 'author_id: fk(() => authors.id).notNull().onDelete("cascade"),', + ); + expect(report).toMatchObject({ + branch: DEV, + schemaName: "app", + tables: ["authors", "posts"], + schemaFile: path.join(root, SCHEMA), + backupFile: path.join(root, `${SCHEMA}.bak`), + recorded: ["0000_baseline"], + migration: { tag: "0000_baseline" }, + }); + expect(report.skipped.map((skip) => skip.column)).toEqual([ + "score", + "total", + ]); + expect(report.warnings).toEqual([ + expect.stringMatching(/^app\.posts\.score is NOT NULL with no default/), + expect.stringMatching( + /^Constraint names differ from the ones migrations use: authors_email_key \(authors_email_unique\), posts_author_id_fkey \(posts_author_id_authors_id_fk\)\./, + ), + ]); + expect(report.approximated).toEqual([ + "app.authors.id: a serial key is declared as id(), an identity generated BY DEFAULT", + "app.posts: index posts_status_idx is not declared", + ]); + expect( + fs.existsSync(path.join(root, MIGRATIONS, "0000_baseline.sql")), + ).toBe(true); + expect(markMigrationsApplied).toHaveBeenCalledWith( + pool, + { migrationsFolder: path.join(root, MIGRATIONS), schemaName: "app" }, + ["0000_baseline"], + ); + expect(fs.readFileSync(report.typesFile, "utf-8")).toContain("authors"); + expect(pool.end).toHaveBeenCalledOnce(); + // The written schema is what the plugin loads. + await expect(loadDefaultDatabaseSchema(root)).resolves.toMatchObject({ + $schemaName: "app", + }); + }); + + test.each([ + ["without history", false], + ["with history", true], + ])( + "on a branch that is not yours (%s): refuses before connecting and points to appkit db init", + async (_label, history) => { + resolveEnvBranch.mockResolvedValue(onProduction); + const root = tempApp({ [SCHEMA]: previousSchema }); + if (history) await commitMigration(root); + const before = tree(root); + + const error = await refusal(root); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + "Refusing to introspect: production is not your dev or session branch (it is the project's default branch). Introspect records the migration it writes as applied, so it runs only on branches `appkit db init` created for you. " + + "Run `appkit db init`: it creates your dev branch from the project's default branch, and introspects it when there is no config/database/schema.ts.", + ); + expect(withBranchPool).not.toHaveBeenCalled(); + expect(tree(root)).toEqual(before); + }, + ); + + test("with history: writes and records the next NNNN_introspect", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await commitMigration(root); + + const report = await introspect(root); + + expect(report.migration?.tag).toBe("0001_introspect"); + expect(report.migration?.statements.join("\n")).toContain( + 'CREATE TABLE "app"."posts"', + ); + expect(report.recorded).toEqual(["0001_introspect"]); + expect(markMigrationsApplied).toHaveBeenCalledWith( + pool, + expect.anything(), + ["0001_introspect"], + ); + }); + + test("a second run with no catalog change writes no migration and records nothing", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await introspect(root); + markMigrationsApplied.mockClear(); + + const report = await introspect(root); + + expect(report.migration).toBeNull(); + expect(report.recorded).toEqual([]); + expect(markMigrationsApplied).not.toHaveBeenCalled(); + }); + + test("a migration an interrupted run wrote but did not record is recorded when the branch matches it", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await introspect(root); + // The run stopped after writing 0000_baseline, before recording it. + recorded = []; + markMigrationsApplied.mockClear(); + + const report = await introspect(root); + + expect(report.migration).toBeNull(); + expect(report.recorded).toEqual(["0000_baseline"]); + expect(markMigrationsApplied).toHaveBeenCalledWith( + pool, + expect.anything(), + ["0000_baseline"], + ); + }); + + test("an unrecorded introspect migration the branch no longer matches is refused, and nothing changes", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await introspect(root); + recorded = []; + catalog = authorsOnly(); + markMigrationsApplied.mockClear(); + const before = tree(root); + + const error = await refusal(root); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + 'Refusing to introspect dev-alice-3f9a2c: it has not applied 0000_baseline in schema "app". Its tables differ from what it describes. Run `appkit db migrate` first.', + ); + expect(tree(root)).toEqual(before); + expect(markMigrationsApplied).not.toHaveBeenCalled(); + }); + + test("without schema.ts it reads the app schema and creates the file", async () => { + const root = tempApp(); + + const report = await introspect(root); + + expect(catalog.calls.every((call) => call.values[0] === "app")).toBe(true); + expect(report).toMatchObject({ + schemaName: "app", + backupFile: null, + tables: ["authors", "posts"], + migration: { tag: "0000_baseline" }, + recorded: ["0000_baseline"], + }); + expect(fs.readFileSync(path.join(root, SCHEMA), "utf-8")).toMatch( + /from schema "app"\.[^]*export const schema = defineSchema\(\(\{ table[^}]*\}\) => \{/, + ); + expect(fs.existsSync(path.join(root, `${SCHEMA}.bak`))).toBe(false); + }); + + test("--schema-name picks another schema", async () => { + const root = tempApp(); + + const report = await runIntrospect(ctxFor(root), { + interactive: false, + importSource: BUILDER, + schemaName: "sales", + }); + + expect(catalog.calls.every((call) => call.values[0] === "sales")).toBe( + true, + ); + expect(report.schemaName).toBe("sales"); + }); + + test.each([ + ["a syntax error", "export const schema = ;\n"], + ["a single line", "// todo\n"], + [ + "a missing import", + `import { defineSchema } from ${JSON.stringify(BUILDER)};\nexport const schema = defineSchema(({ table }) => ({ x: table("x", { id: id() }) }));\n`, + ], + ])( + "a schema.ts with %s is never read: it goes to .bak and is replaced from the app schema", + async (_label, broken) => { + const root = tempApp({ [SCHEMA]: broken }); + + const report = await introspect(root); + + expect(catalog.calls.every((call) => call.values[0] === "app")).toBe( + true, + ); + expect(report).toMatchObject({ + schemaName: "app", + tables: ["authors", "posts"], + backupFile: path.join(root, `${SCHEMA}.bak`), + migration: { tag: "0000_baseline" }, + }); + expect(fs.readFileSync(path.join(root, `${SCHEMA}.bak`), "utf-8")).toBe( + broken, + ); + expect(fs.readFileSync(path.join(root, SCHEMA), "utf-8")).toMatch( + /from schema "app"\.[^]*export const schema = defineSchema\(\(\{ table[^}]*\}\) => \{/, + ); + }, + ); + + test("--schema-name public is written explicitly, so the app never looks for the tables in the default app schema", async () => { + const root = tempApp(); + + const report = await runIntrospect(ctxFor(root), { + interactive: false, + importSource: BUILDER, + schemaName: "public", + }); + + expect(report.schemaName).toBe("public"); + expect(fs.readFileSync(path.join(root, SCHEMA), "utf-8")).toContain( + '{ schemaName: "public" }', + ); + expect((await loadDefaultDatabaseSchema(root)).$schemaName).toBe("public"); + }); + + test("the schema schema.ts declares does not change which schema is read", async () => { + const root = tempApp({ + [SCHEMA]: previousSchema.replace("}));", '}), { schemaName: "sales" });'), + }); + + const report = await introspect(root); + + expect(catalog.calls.every((call) => call.values[0] === "app")).toBe(true); + expect(report.schemaName).toBe("app"); + }); + + test("an empty schema next to an existing schema.ts refuses before writing anything", async () => { + catalog = catalogResponder({ + tables: [], + columns: [], + constraints: [], + indexes: [], + }); + const root = tempApp({ [SCHEMA]: previousSchema }); + + const error = await refusal(root); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + 'Schema "app" on dev-alice-3f9a2c has no tables, so introspect would replace config/database/schema.ts with an empty one. Pass --schema-name to read another schema.', + ); + expect(fs.readFileSync(path.join(root, SCHEMA), "utf-8")).toBe( + previousSchema, + ); + expect(fs.existsSync(path.join(root, `${SCHEMA}.bak`))).toBe(false); + expect(fs.existsSync(path.join(root, MIGRATIONS))).toBe(false); + }); + + test("an empty schema without schema.ts still writes an empty one", async () => { + catalog = catalogResponder({ + tables: [], + columns: [], + constraints: [], + indexes: [], + }); + const root = tempApp(); + + const report = await introspect(root); + + expect(report.tables).toEqual([]); + expect(report.migration).toBeNull(); + expect(report.recorded).toEqual([]); + expect(fs.readFileSync(path.join(root, SCHEMA), "utf-8")).toContain( + "export const schema = defineSchema(() => ({}));", + ); + }); + + test("without a configured branch it fails before writing", async () => { + resolveEnvBranch.mockResolvedValue(null); + const root = tempApp({ [SCHEMA]: previousSchema }); + + await expect(introspect(root)).rejects.toThrow( + "Did not introspect: no branch is configured in .env (PGHOST, PGDATABASE and LAKEBASE_ENDPOINT); run `appkit db init`.", + ); + expect(fs.readFileSync(path.join(root, SCHEMA), "utf-8")).toBe( + previousSchema, + ); + }); + + test("with history, a branch that has not applied every committed migration refuses before reading the catalog", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await commitMigration(root); + // A teammate's migration, pulled but not applied on this branch yet. + fs.writeFileSync( + path.join(root, SCHEMA), + previousSchema.replace( + "email: text() }),", + 'email: text() }),\n tags: table("tags", { id: id() }),', + ), + ); + await commitMigration(root, false); + const before = tree(root); + + const error = await refusal(root); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + 'Refusing to introspect dev-alice-3f9a2c: it has not applied 0001_migration in schema "app". The migration would undo it. Run `appkit db migrate` first.', + ); + expect(catalog.calls).toEqual([]); + expect(tree(root)).toEqual(before); + }); + + test.each([ + ["without history", false], + ["with history", true], + ])( + "a branch that records a migration the checkout lacks (%s) is refused before reading the catalog", + async (_label, history) => { + const root = tempApp({ [SCHEMA]: previousSchema }); + if (history) await commitMigration(root); + recorded.push(Date.now() + 60_000); + const before = tree(root); + + const error = await refusal(root); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + 'Refusing to introspect dev-alice-3f9a2c: it records 1 migration this checkout does not have in schema "app". Pull the latest config/database/migrations, then rerun.', + ); + expect(catalog.calls).toEqual([]); + expect(tree(root)).toEqual(before); + }, + ); + + test("a branch that skipped a committed migration is refused", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await commitMigration(root, false); + fs.writeFileSync( + path.join(root, SCHEMA), + previousSchema.replace("email: text() }", "email: text(), bio: text() }"), + ); + await commitMigration(root); + + const error = await refusal(root); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + 'Refusing to introspect dev-alice-3f9a2c: it skipped 0000_migration in schema "app". Run `appkit db migrate`, which says how to fix it.', + ); + }); + + test("with history and no schema.ts, an empty schema refuses instead of dropping every table", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await commitMigration(root); + fs.rmSync(path.join(root, SCHEMA)); + catalog = catalogResponder({ + tables: [], + columns: [], + constraints: [], + indexes: [], + }); + const before = tree(root); + + const error = await refusal(root); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + 'Schema "app" on dev-alice-3f9a2c has no tables, so introspect would write a migration that drops every table. Pass --schema-name to read another schema.', + ); + expect(tree(root)).toEqual(before); + }); + + test.each([ + [ + "a column it cannot declare", + previousSchema, + authorsOnly([columnRow("authors", "email", "numeric")]), + 'ALTER TABLE "app"."authors" DROP COLUMN "email"; (type "numeric" is not supported)', + ], + [ + "a default it cannot declare", + previousSchema.replace("email: text()", 'email: text().default("x")'), + authorsOnly([ + columnRow("authors", "email", "text", { default_expr: "lower('X')" }), + ]), + `ALTER TABLE "app"."authors" ALTER COLUMN "email" DROP DEFAULT; (default lower('X') is not supported)`, + ], + [ + "a table it cannot key", + previousSchema.replace( + "email: text() }),", + 'email: text() }),\n tags: table("tags", { id: id() }),', + ), + authorsOnly([columnRow("authors", "email", "text")], { + tables: [{ name: "tags", partitioned: false }], + columns: [columnRow("tags", "id", "integer", { not_null: true })], + }), + 'DROP TABLE "app"."tags" CASCADE; (it has no primary key)', + ], + [ + "a column in the public schema", + previousSchema.replace("}));", '}), { schemaName: "public" });'), + authorsOnly([columnRow("authors", "email", "numeric")]), + 'ALTER TABLE "authors" DROP COLUMN "email"; (type "numeric" is not supported)', + ], + ])( + "with history, a migration that drops %s the branch still has is refused and nothing changes", + async (_label, committed, branch, refused) => { + const root = tempApp({ [SCHEMA]: committed }); + await commitMigration(root); + catalog = branch; + const before = tree(root); + + const error = await rejection( + runIntrospect(ctxFor(root), { + interactive: false, + importSource: BUILDER, + schemaName: (await loadDefaultDatabaseSchema(root)).$schemaName, + }), + ); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + `Refusing to write a migration that drops what the branch still has but schema.ts cannot express: ${refused}. The deployed app would run it on start.`, + ); + expect(tree(root)).toEqual(before); + expect(markMigrationsApplied).not.toHaveBeenCalled(); + }, + ); + + test("with history, a skip elsewhere does not block dropping what the branch no longer has, which is recorded and warned about", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await commitMigration(root); + catalog = authorsOnly([], { + tables: [{ name: "events", partitioned: false }], + columns: [columnRow("events", "line", "text")], + }); + + const report = await introspect(root); + + expect(report.skipped).toEqual([ + { table: "events", reason: "it has no primary key" }, + ]); + expect(report.migration?.tag).toBe("0001_introspect"); + expect(report.migration?.statements).toEqual([ + 'ALTER TABLE "app"."authors" DROP COLUMN "email";', + ]); + expect(report.recorded).toEqual(["0001_introspect"]); + expect(report.warnings).toEqual([ + '0001_introspect breaks instances still running the previous release, which keep serving while the new one migrates on start: ALTER TABLE "app"."authors" DROP COLUMN "email";', + ]); + }); + + test("objects introspect does not read are named in a warning", async () => { + catalog = catalogResponder(undefined, [ + "trigger audit on authors", + "view active_authors", + ]); + const root = tempApp(); + + const report = await introspect(root); + + expect(report.warnings.at(-1)).toBe( + "Not read: trigger audit on authors, view active_authors. schema.ts and its migrations leave them out, so a database the migrations create will not have them.", + ); + }); + + test.each([ + [ + "recording", + () => + markMigrationsApplied.mockRejectedValueOnce(new Error("lock timeout")), + "lock timeout", + ], + [ + "writing database.d.ts", + () => writeTypes.mockRejectedValueOnce(new Error("disk full")), + "disk full", + ], + ])( + "when %s fails, schema.ts, its backup, database.d.ts and the migrations are put back, so a rerun records", + async (_label, failOnce, message) => { + const root = tempApp({ + [SCHEMA]: previousSchema, + [`${SCHEMA}.bak`]: "// an older backup\n", + }); + const before = tree(root); + failOnce(); + + const error = await refusal(root); + + expect(error.message).toBe(`Introspect changed nothing: ${message}`); + expect(tree(root)).toEqual(before); + expect(fs.existsSync(path.join(root, MIGRATIONS))).toBe(false); + expect(fs.existsSync(path.join(root, TYPES))).toBe(false); + expect(recorded).toEqual([]); + + const report = await introspect(root); + expect(report).toMatchObject({ + migration: { tag: "0000_baseline" }, + recorded: ["0000_baseline"], + }); + }, + ); + + test("a second run with the same catalog keeps the original .bak", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await introspect(root); + + const second = await introspect(root); + + expect(second.backupFile).toBeNull(); + expect(fs.readFileSync(path.join(root, `${SCHEMA}.bak`), "utf-8")).toBe( + previousSchema, + ); + }); + + test("when every table is skipped, an existing schema.ts is kept", async () => { + catalog = catalogResponder({ + tables: [{ name: "logs", partitioned: false }], + columns: [columnRow("logs", "line", "text")], + constraints: [], + indexes: [], + }); + const root = tempApp({ [SCHEMA]: previousSchema }); + const before = tree(root); + + const error = await refusal(root); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + 'Schema "app" on dev-alice-3f9a2c has no tables introspect can write (logs: it has no primary key), so introspect would replace config/database/schema.ts with an empty one. Pass --schema-name to read another schema.', + ); + expect(tree(root)).toEqual(before); + }); + + test("a generated schema.ts that does not load is reported as a bug, and schema.ts is put back", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + const before = tree(root); + + const error = await rejection( + runIntrospect(ctxFor(root), { + interactive: false, + importSource: path.join(root, "missing-builder.ts"), + }), + ); + + expect(error.message).toBe( + 'Introspect changed nothing: The schema.ts introspect generated does not load. This is a bug in `appkit db introspect`; please report it with the catalog of schema "app".', + ); + expect(tree(root)).toEqual(before); + expect(markMigrationsApplied).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/appkit/src/database/tooling/introspect/tests/render.test.ts b/packages/appkit/src/database/tooling/introspect/tests/render.test.ts new file mode 100644 index 000000000..1211a18a9 --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/tests/render.test.ts @@ -0,0 +1,623 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { describe, expect, test } from "vitest"; + +import type { IntrospectedForeignKey, IntrospectedTable } from "../catalog"; +import { renderSchemaSource } from "../render"; +import { col, richTables, tableOf, uniqueKey } from "./fixture"; + +const golden = (name: string) => + fs.readFileSync(path.join(import.meta.dirname, "golden", name), "utf-8"); + +const fk = ( + column: string, + table: string, + extra: Partial = {}, +): IntrospectedForeignKey => ({ + name: `${column}_fkey`, + columns: [column], + schema: "app", + table, + refColumns: ["id"], + onDelete: "no action", + onUpdate: "no action", + deferrable: false, + matchFull: false, + ...extra, +}); + +/** A table with an integer identity key plus the given columns and FKs. */ +const keyed = ( + name: string, + columns: IntrospectedTable["columns"] = [], + foreignKeys: IntrospectedForeignKey[] = [], + unique: string[][] = [], +): IntrospectedTable => + tableOf({ + name, + primaryKey: ["id"], + unique: unique.map((columns) => uniqueKey(name, ...columns)), + foreignKeys, + columns: [ + col("id", "integer", { notNull: true, identity: "d" }), + ...columns, + ], + }); + +const body = (source: string) => + source.slice(source.indexOf("export const schema")); + +describe("renderSchemaSource", () => { + test("golden: supported types, defaults, FKs and enums, with skip comments", () => { + const out = renderSchemaSource(richTables, "app"); + + expect(out.source).toBe(golden("rich.schema.ts.txt")); + expect(out.skipped).toEqual( + expect.arrayContaining([ + { + table: "events", + reason: "composite primary key (a, b) is not supported", + }, + { table: "logs", reason: "it has no primary key" }, + { + table: "authors", + column: "score", + reason: 'type "numeric(10,2)" is not supported', + }, + { + table: "posts", + column: "body", + reason: "default '{}'::jsonb is not supported", + }, + { + table: "posts", + column: "views", + reason: "default 0 is not supported", + }, + { + table: "posts", + column: "editor_id", + reason: + 'foreign key to authors(id) is not kept: a second foreign key to "authors" would make the relation ambiguous', + }, + ]), + ); + expect(out.skipped).toHaveLength(6); + }); + + test("a skipped NOT NULL column without a default is a loud warning", () => { + const { warnings } = renderSchemaSource(richTables, "app"); + expect(warnings).toContain( + 'app.authors.score is NOT NULL with no default but is not in schema.ts (type "numeric(10,2)" is not supported). Inserts through the API will fail until it gets a default, becomes nullable, or is dropped.', + ); + expect(warnings).toContain( + "app.posts.views: its default 0 is not in schema.ts, so the API requires a value for it on insert.", + ); + expect( + warnings.filter((warning) => !warning.startsWith("Constraint names")), + ).toHaveLength(2); + }); + + test.each([ + ["app", "export const schema = defineSchema(() => ({}));\n"], + [ + "public", + 'export const schema = defineSchema(() => ({}), { schemaName: "public" });\n', + ], + ])( + "no tables in %s renders an empty schema, naming the schema only when it is not app", + (schemaName, expected) => { + expect(body(renderSchemaSource([], schemaName).source)).toBe(expected); + }, + ); + + test("tables in a schema other than app pass it to defineSchema", () => { + expect(body(renderSchemaSource([keyed("notes")], "public").source)).toBe( + [ + "export const schema = defineSchema(", + " ({ table }) => {", + ' const notes = table("notes", {', + " id: id(),", + " });", + "", + " return { notes };", + " },", + ' { schemaName: "public" },', + ");", + "", + ].join("\n"), + ); + }); + + test("declares FK targets first, whatever the catalog order", () => { + const { source } = renderSchemaSource( + [ + keyed("c", [col("b_id", "integer")], [fk("b_id", "b")]), + keyed("b", [col("a_id", "integer")], [fk("a_id", "a")]), + keyed("a"), + ], + "app", + ); + const order = ["a", "b", "c"].map((name) => + source.indexOf(`const ${name} = table`), + ); + expect(order).toEqual([...order].sort((x, y) => x - y)); + expect(source).toContain("return { a, b, c };"); + }); + + test("FKs the builder cannot express stay plain columns, each with its reason", () => { + const cases: Array<[IntrospectedTable[], string]> = [ + [ + [keyed("t", [col("parent_id", "integer")], [fk("parent_id", "t")])], + "introspect does not write self-references", + ], + [ + [ + keyed("a", [col("b_id", "integer")], [fk("b_id", "b")]), + keyed("b", [col("a_id", "integer")], [fk("a_id", "a")]), + ], + "it would close a cycle of foreign keys", + ], + [ + [ + keyed( + "t", + [col("x", "integer"), col("y", "integer")], + [{ ...fk("x", "u"), columns: ["x", "y"], refColumns: ["id", "k"] }], + ), + keyed("u"), + ], + "foreign keys over several columns are not supported", + ], + [ + [ + keyed( + "t", + [col("u_id", "integer")], + [fk("u_id", "u", { schema: "public" })], + ), + keyed("u"), + ], + 'it references schema "public"', + ], + [ + [ + keyed( + "t", + [col("u_code", "integer")], + [fk("u_code", "u", { refColumns: ["code"] })], + ), + keyed("u", [col("code", "integer")]), + ], + "u.code is not a primary key or single-column unique key", + ], + [ + [keyed("t", [col("u_id", "bigint")], [fk("u_id", "u")]), keyed("u")], + "its type bigint() differs from u.id", + ], + [ + [ + keyed( + "t", + [col("u_id", "integer", { notNull: true })], + [fk("u_id", "u", { onDelete: "set null" })], + ), + keyed("u"), + ], + "SET NULL on a NOT NULL column", + ], + [ + [ + keyed( + "t", + [col("u_id", "integer")], + [fk("u_id", "u", { onDelete: "set default" })], + ), + keyed("u"), + ], + "SET DEFAULT without a default", + ], + [ + [ + keyed( + "t", + [col("u_id", "integer"), col("u", "text")], + [fk("u_id", "u")], + ), + keyed("u"), + ], + 'the relation "u" would clash with the column of the same name', + ], + [ + [ + keyed("t", [col("u_id", "integer")], [fk("u_id", "u")]), + keyed("u", [col("t", "text")]), + ], + 'the reverse relation "t" would clash with column u.t', + ], + [ + [ + keyed("t", [col("u_id", "integer")], [fk("u_id", "u")]), + { ...keyed("u"), primaryKey: ["id", "x"] }, + ], + 'table "u" is skipped', + ], + ]; + for (const [tables, reason] of cases) { + const out = renderSchemaSource(tables, "app"); + expect(out.source, reason).toContain(reason); + const fkSkips = out.skipped.filter((skip) => + skip.reason.startsWith("foreign key"), + ); + expect(fkSkips, reason).toHaveLength(1); + // Exactly one fk() survives only in the cycle case (the first edge). + expect(out.source.includes("fk(() =>"), reason).toBe( + reason.includes("cycle"), + ); + } + }); + + test("an FK keeps NOT NULL, unique, a default and its actions", () => { + const { source } = renderSchemaSource( + [ + keyed( + "t", + [col("u_id", "integer", { notNull: true, defaultExpr: "1" })], + [fk("u_id", "u", { onDelete: "set default", onUpdate: "cascade" })], + [["u_id"]], + ), + keyed("u"), + ], + "app", + ); + expect(source).toContain( + 'u_id: fk(() => u.id).notNull().unique().default(1).onDelete("set default").onUpdate("cascade"),', + ); + }); + + test("a table whose primary key cannot be expressed is skipped whole", () => { + const out = renderSchemaSource( + [ + tableOf({ + name: "t", + primaryKey: ["code"], + unique: [], + foreignKeys: [], + columns: [ + col("code", "smallint", { notNull: true }), + col("x", "numeric", { notNull: true }), + ], + }), + ], + "app", + ); + expect(out.skipped).toEqual([ + { + table: "t", + reason: + 'its primary key column "code" cannot be expressed (type "smallint" is not supported)', + }, + ]); + // The table is gone, so its columns produce no warnings. + expect(out.warnings).toEqual([]); + }); + + test("names AppKit reserves are skipped; unusual names are quoted or renamed", () => { + const out = renderSchemaSource( + [ + keyed("sql"), + keyed("my-table", [ + col("$meta", "text"), + col("and", "text"), + col("select", "text"), + ]), + keyed("users"), + keyed("usersRelations"), + ], + "app", + ); + expect( + out.skipped.map((skip) => [skip.table, skip.column ?? null, skip.reason]), + ).toEqual([ + ["sql", null, 'the name "sql" is reserved by AppKit'], + [ + "usersRelations", + null, + 'the name collides with the relation metadata AppKit generates for "users"', + ], + ["my-table", "$meta", 'the name "$meta" is reserved by AppKit'], + ["my-table", "and", 'the name "and" is reserved by AppKit'], + ]); + expect(out.source).toContain('const my_table = table("my-table", {'); + expect(out.source).toContain(" select: text(),"); + expect(out.source).toContain('return { "my-table": my_table, users };'); + }); + + test("unsupported defaults are dropped with a comment; nextval on a key becomes id()", () => { + const { source, skipped } = renderSchemaSource( + [ + tableOf({ + name: "t", + primaryKey: ["id"], + unique: [], + foreignKeys: [], + columns: [ + col("id", "bigint", { + notNull: true, + defaultExpr: "nextval('t_id_seq'::regclass)", + }), + col("code", "character varying(2)", { + defaultExpr: "'toolong'::character varying", + }), + col("at", "timestamp without time zone", { + defaultExpr: "'2020-01-01 00:00:00'::timestamp without time zone", + }), + col("n", "integer", { defaultExpr: "nextval('n_seq'::regclass)" }), + ], + }), + ], + "app", + ); + expect(source).toContain(" id: bigid(),"); + expect(skipped.map((skip) => skip.column)).toEqual(["code", "at", "n"]); + expect(source).toContain(" code: varchar(2),"); + }); + + test("a catalog name with a line break stays inside its comment", () => { + const evil = "x\nrequire('node:child_process').execSync('touch /tmp/p')//"; + const { source, skipped, warnings } = renderSchemaSource( + [ + keyed("t", [ + col(evil, "numeric", { notNull: true }), + col("y\u2028z", "numeric"), + ]), + ], + "app", + ); + for (const line of source.split("\n")) { + expect(line.trimStart().startsWith("require(")).toBe(false); + } + expect(source).not.toMatch(/[\u2028\u2029]/); + expect(source).toContain( + `// appkit: skipped column "x\\u000arequire('node:child_process').execSync('touch /tmp/p')//": type "numeric" is not supported`, + ); + expect(source).toContain( + '// appkit: skipped column "y\\u2028z": type "numeric" is not supported', + ); + expect(skipped.map((skip) => skip.column)).toEqual([ + "x\\u000arequire('node:child_process').execSync('touch /tmp/p')//", + "y\\u2028z", + ]); + expect(warnings[0]).toMatch(/^app\.t\.x\\u000arequire/); + }); + + test("the schema name in the header comment is escaped too", () => { + const { source } = renderSchemaSource([keyed("t")], "a\nb"); + expect(source.split("\n")[0]).toBe( + '// Generated by `appkit db introspect` from schema "a\\u000ab". Edit tables here,', + ); + expect(source).toContain('{ schemaName: "a\\nb" }'); + }); + + test("returns the tables it wrote", () => { + expect(renderSchemaSource(richTables, "app").tables).toEqual([ + "authors", + "posts", + "default", + ]); + }); + + test("what schema.ts declares differently from the database is listed as approximated", () => { + const { approximated } = renderSchemaSource( + [ + tableOf({ + name: "a", + partitioned: true, + primaryKey: ["id"], + checks: ["a_n_check"], + indexes: ["a_n_idx", "a_m_idx"], + columns: [ + col("id", "integer", { notNull: true, identity: "a" }), + col("n", "character varying"), + col("at", "timestamp(3) with time zone"), + ], + }), + keyed( + "b", + [col("a_id", "integer")], + [ + fk("a_id", "a", { + name: "b_a_id_a_id_fk", + deferrable: true, + matchFull: true, + }), + ], + ), + ], + "app", + ); + expect(approximated).toEqual([ + "app.a.id: GENERATED ALWAYS AS IDENTITY is declared as id(), an identity generated BY DEFAULT", + "app.a.n: character varying without a length is declared as text()", + "app.a.at: timestamp(3) with time zone is declared without its precision", + "app.a: a partitioned table is declared as a plain table", + "app.a: CHECK or EXCLUDE constraint a_n_check is not declared", + "app.a: indexes a_n_idx, a_m_idx are not declared", + "app.b.a_id: the foreign key is declared without DEFERRABLE", + "app.b.a_id: the foreign key is declared without MATCH FULL", + ]); + }); + + test("constraint names that differ from the ones migrations use are one warning", () => { + const { warnings } = renderSchemaSource(richTables, "app"); + expect(warnings).toContain( + "Constraint names differ from the ones migrations use: authors_email_key (authors_email_unique), posts_author_id_fkey (posts_author_id_authors_id_fk). A later migration that drops or changes one of them fails until it is renamed in the database (ALTER TABLE … RENAME CONSTRAINT).", + ); + }); + + test("constraint names migrations would use raise no warning", () => { + const { warnings } = renderSchemaSource( + [ + tableOf({ + ...keyed("u", [col("email", "text")]), + unique: [{ name: "u_email_unique", columns: ["email"] }], + }), + keyed( + "t", + [col("u_id", "integer")], + [fk("u_id", "u", { name: "t_u_id_u_id_fk" })], + ), + ], + "app", + ); + expect(warnings).toEqual([]); + }); + + test("migrations' constraint names are cut at 63 bytes, as Postgres cuts them", () => { + const column = "ç".repeat(40); + const { warnings } = renderSchemaSource( + [ + tableOf({ + ...keyed("t", [col(column, "text")]), + // "t_" plus 30 two-byte characters is 62 bytes; one more would pass 63. + unique: [{ name: `t_${"ç".repeat(30)}`, columns: [column] }], + }), + ], + "app", + ); + expect(warnings).toEqual([]); + }); + + test("bidi and line-separator characters are escaped in string literals", () => { + const name = "t\u202e\u2066\u2028x"; + const { source } = renderSchemaSource([keyed(name)], "app"); + expect(source).not.toMatch(/[\u2028\u202e\u2066]/); + expect(source).toContain('table("t\\u202e\\u2066\\u2028x", {'); + }); + + test("what schema.ts leaves out is listed by raw name, for the drop check", () => { + const status = { name: "status", schema: "app", values: ["a"] }; + const { unexpressed } = renderSchemaSource( + [ + keyed( + "t", + [ + col("n", "numeric"), + col("s", "app.status", { generated: true, enum: status }), + col("d", "text", { defaultExpr: "lower('X')" }), + col("x", "integer"), + col("y", "integer"), + col("p", "integer"), + ], + [fk("p", "t")], + [["x", "y"]], + ), + tableOf({ name: "logs", columns: [col("line", "text")] }), + ], + "app", + ); + expect(unexpressed).toEqual([ + { kind: "table", name: "logs", reason: "it has no primary key" }, + { + kind: "constraint", + name: "t", + member: "t_x_y_key", + reason: + "unique constraint (x, y) spans several columns, which is not supported", + }, + { + kind: "column", + name: "t", + member: "n", + reason: 'type "numeric" is not supported', + }, + { + kind: "column", + name: "t", + member: "s", + reason: "generated columns are not supported", + }, + { + kind: "default", + name: "t", + member: "d", + reason: "default lower('X') is not supported", + }, + ...["p_fkey", "t_p_t_id_fk"].map((member) => ({ + kind: "constraint", + name: "t", + member, + reason: + "foreign key to t(id) is not kept: introspect does not write self-references", + })), + { + kind: "type", + name: "status", + reason: "only what schema.ts leaves out uses it", + }, + ]); + }); + + test("a uuid FK keeps the key and skips gen_random_uuid(), which fk() does not take", () => { + const { source, skipped, warnings } = renderSchemaSource( + [ + tableOf({ + name: "u", + primaryKey: ["id"], + columns: [ + col("id", "uuid", { + notNull: true, + defaultExpr: "gen_random_uuid()", + }), + ], + }), + keyed( + "t", + [ + col("u_id", "uuid", { + notNull: true, + defaultExpr: "gen_random_uuid()", + }), + ], + [fk("u_id", "u", { name: "t_u_id_u_id_fk" })], + ), + ], + "app", + ); + expect(source).toContain(" u_id: fk(() => u.id).notNull(),"); + expect(source).not.toContain("fk(() => u.id).notNull().defaultRandom()"); + expect(skipped).toEqual([ + { + table: "t", + column: "u_id", + reason: + "default gen_random_uuid() is not supported on a foreign key column", + }, + ]); + expect(warnings).toEqual([ + "app.t.u_id: its default gen_random_uuid() is not in schema.ts, so the API requires a value for it on insert.", + ]); + }); + + test("SET DEFAULT needs a default fk() can take", () => { + const { skipped } = renderSchemaSource( + [ + tableOf({ + name: "u", + primaryKey: ["id"], + columns: [col("id", "uuid", { notNull: true })], + }), + keyed( + "t", + [col("u_id", "uuid", { defaultExpr: "gen_random_uuid()" })], + [fk("u_id", "u", { onDelete: "set default" })], + ), + ], + "app", + ); + expect(skipped.map((skip) => skip.reason)).toEqual([ + "foreign key to u(id) is not kept: SET DEFAULT without a default the builder can express", + ]); + }); +}); diff --git a/packages/appkit/src/database/tooling/introspect/tests/roundtrip.test.ts b/packages/appkit/src/database/tooling/introspect/tests/roundtrip.test.ts new file mode 100644 index 000000000..f3d52d3c2 --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/tests/roundtrip.test.ts @@ -0,0 +1,368 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { getTableConfig, type PgTable } from "drizzle-orm/pg-core"; +import { afterEach, describe, expect, test } from "vitest"; + +import { loadDefaultDatabaseSchema } from "../../../../plugins/database/load-schema"; +import { drizzleKitImports } from "../../drizzle-exports"; +import { generateMigrationFiles } from "../../migrations"; +import type { IntrospectedTable } from "../catalog"; +import { renderSchemaSource } from "../render"; +import { + BUILDER, + col, + foreignKey, + richTables, + tableOf, + uniqueKey, +} from "./fixture"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) + fs.rmSync(root, { recursive: true, force: true }); +}); + +/** Write the rendered source as an app's schema.ts and load it the way the plugin does. */ +async function load(tables: IntrospectedTable[]) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "appkit-introspect-")); + roots.push(root); + const file = path.join(root, "config/database/schema.ts"); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync( + file, + renderSchemaSource(tables, "app", { importSource: BUILDER }).source, + ); + return { root, schema: await loadDefaultDatabaseSchema(root) }; +} + +interface KitColumn { + name: string; + type: string; + notNull: boolean; + primaryKey: boolean; + default?: unknown; +} +interface KitTable { + name: string; + schema: string; + columns: Record; + foreignKeys: Record< + string, + { + tableTo: string; + columnsFrom: string[]; + columnsTo: string[]; + onDelete?: string; + } + >; +} + +describe("introspect round trip", () => { + test("the rendered source loads through loadDefaultDatabaseSchema, and drizzle-kit sees the same tables", async () => { + const { schema } = await load(richTables); + expect(schema.$schemaName).toBe("app"); + expect(Object.keys(schema.$tables).sort()).toEqual([ + "authors", + "default", + "posts", + ]); + + const { generateDrizzleJson } = await import("drizzle-kit/api"); + const snapshot = generateDrizzleJson(drizzleKitImports(schema), undefined, [ + "app", + ]) as unknown as { + tables: Record; + enums: Record; + }; + const columns = (table: string) => + Object.fromEntries( + Object.values(snapshot.tables[`app.${table}`]?.columns ?? {}).map( + (c) => [ + c.name, + `${c.type}${c.notNull ? " not null" : ""}${c.primaryKey ? " pk" : ""}`, + ], + ), + ); + + expect(columns("authors")).toEqual({ + id: "integer not null pk", + email: "varchar(255) not null", + bio: "text", + active: "boolean not null", + rank: "integer", + created_at: "timestamp with time zone not null", + seen_at: "timestamp", + }); + expect(columns("posts")).toEqual({ + id: "uuid not null pk", + author_id: "integer not null", + editor_id: "integer", + status: "ticket_status not null", + body: "jsonb", + views: "bigint not null", + "my col": "text", + }); + expect(columns("default")).toEqual({ id: "bigint not null pk" }); + expect( + Object.values(snapshot.tables["app.posts"]?.foreignKeys ?? {}), + ).toEqual([ + expect.objectContaining({ + tableTo: "authors", + columnsFrom: ["author_id"], + columnsTo: ["id"], + onDelete: "cascade", + }), + ]); + expect(snapshot.enums["app.ticket_status"]).toMatchObject({ + values: ["open", "closed"], + }); + }); + + test("the baseline is generated from the empty snapshot: it creates every rendered table", async () => { + const { root, schema } = await load(richTables); + const folder = path.join(root, "config/database/migrations"); + + const migration = await generateMigrationFiles({ + folder, + schema, + tag: "baseline", + interactive: false, + }); + + expect(migration?.tag).toBe("0000_baseline"); + const sql = migration?.statements.join("\n") ?? ""; + for (const table of ["authors", "posts", "default"]) { + expect(sql).toContain(`CREATE TABLE "app"."${table}"`); + } + expect(sql).toContain( + `CREATE TYPE "app"."ticket_status" AS ENUM('open', 'closed')`, + ); + expect(sql).not.toContain("CREATE SCHEMA"); + }); + + test("every skip path still yields a loadable schema", async () => { + const tables: IntrospectedTable[] = [ + tableOf({ + name: "a", + primaryKey: ["id"], + unique: [uniqueKey("a", "x", "y")], + foreignKeys: [ + foreignKey({ + name: "self", + columns: ["parent_id"], + schema: "app", + table: "a", + refColumns: ["id"], + onDelete: "no action", + onUpdate: "no action", + }), + foreignKey({ + name: "to_b", + columns: ["b_id"], + schema: "app", + table: "b", + refColumns: ["id"], + onDelete: "set null", + onUpdate: "no action", + }), + ], + columns: [ + col("id", "integer", { notNull: true, identity: "a" }), + col("parent_id", "integer"), + col("b_id", "integer", { notNull: true }), + col("x", "integer"), + col("y", "integer"), + col("g", "integer", { generated: true, defaultExpr: "(x + y)" }), + col("seq", "integer", { identity: "d" }), + col("tags", "text[]"), + ], + }), + tableOf({ + name: "b", + primaryKey: ["id"], + unique: [], + foreignKeys: [ + foreignKey({ + name: "to_a", + columns: ["a_id"], + schema: "app", + table: "a", + refColumns: ["id"], + onDelete: "no action", + onUpdate: "no action", + }), + ], + columns: [ + col("id", "integer", { notNull: true, identity: "d" }), + col("a_id", "integer"), + ], + }), + ]; + const { schema } = await load(tables); + expect(Object.keys(schema.$tables).sort()).toEqual(["a", "b"]); + expect(Object.keys(schema.$tables.a?.$columns ?? {})).toEqual([ + "id", + "parent_id", + "b_id", + "x", + "y", + ]); + }); + + test("a uuid FK with gen_random_uuid() and names with line breaks still load, with no extra code", async () => { + const marker = "__introspect_injected__"; + const { schema } = await load([ + tableOf({ + name: "u", + primaryKey: ["id"], + columns: [ + col("id", "uuid", { + notNull: true, + defaultExpr: "gen_random_uuid()", + }), + ], + }), + tableOf({ + name: "t", + primaryKey: ["id"], + foreignKeys: [ + foreignKey({ + name: "t_u_id_fkey", + columns: ["u_id"], + schema: "app", + table: "u", + refColumns: ["id"], + onDelete: "no action", + onUpdate: "no action", + }), + ], + columns: [ + col("id", "integer", { notNull: true, identity: "d" }), + col("u_id", "uuid", { + notNull: true, + defaultExpr: "gen_random_uuid()", + }), + col(`x\nglobalThis.${marker} = true; //`, "numeric"), + col(`y\u2028globalThis.${marker} = true; //`, "numeric"), + ], + }), + ]); + + expect((globalThis as Record)[marker]).toBeUndefined(); + expect(Object.keys(schema.$tables).sort()).toEqual(["t", "u"]); + expect(Object.keys(schema.$tables.t?.$columns ?? {})).toEqual([ + "id", + "u_id", + ]); + }); + + test("hostile names reach every place render writes them and load back unchanged, with no code run", async () => { + const marker = "__introspect_hostile__"; + const payload = `globalThis.${marker} = true; //`; + const [ + lf, + cr, + ls, + ps, + quote, + apostrophe, + backslash, + backtick, + bidi, + close, + ] = [ + `lf\n${payload}`, + `cr\r${payload}`, + `ls\u2028${payload}`, + `ps\u2029${payload}`, + `q"]; ${payload}`, + `s'); ${payload}`, + `b\\"; ${payload}`, + `t\`\${globalThis.${marker} = true}\``, + "bidi\u202e\u2066x", + `*/ ${payload}`, + ]; + const texts = [lf, cr, ls, ps, apostrophe, backslash, backtick, bidi]; + const literal = (value: string) => `'${value.replace(/'/g, "''")}'::text`; + const kind = { name: close, schema: "app", values: [quote, cr, bidi] }; + const level = { name: "level", schema: "app", values: [ls, bidi] }; + const tables = [ + tableOf({ + name: "target", + primaryKey: [quote], + columns: [col(quote, "integer", { notNull: true, identity: "d" })], + }), + tableOf({ + name: "source", + primaryKey: ["id"], + foreignKeys: [ + foreignKey({ + name: apostrophe, + columns: [backslash], + schema: "app", + table: "target", + refColumns: [quote], + onDelete: "no action", + onUpdate: "no action", + }), + ], + columns: [ + col("id", "integer", { notNull: true, identity: "d" }), + col(backslash, "integer", { notNull: true }), + ...texts.map((name) => + col(`c ${name}`, "text", { defaultExpr: literal(name) }), + ), + col("kind", `app."${close}"`, { enum: kind }), + col("level", "app.level", { + defaultExpr: `'${bidi}'::app.level`, + enum: level, + }), + ...[lf, cr, ls, ps].map((name) => col(`skipped ${name}`, "numeric")), + ], + }), + tableOf({ + name: close, + primaryKey: ["id"], + columns: [col("id", "integer", { notNull: true, identity: "d" })], + }), + tableOf({ name: ps, columns: [col(cr, "text")] }), + ]; + + const { schema } = await load(tables); + + expect((globalThis as Record)[marker]).toBeUndefined(); + expect(Object.keys(schema.$tables).sort()).toEqual( + ["source", "target", close].sort(), + ); + const config = (name: string) => + getTableConfig(schema.$engine[name] as unknown as PgTable); + expect(config(close).name).toBe(close); + expect(config("target").columns.map((c) => c.name)).toEqual([quote]); + const source = config("source"); + expect(source.columns.map((c) => c.name)).toEqual([ + "id", + backslash, + ...texts.map((name) => `c ${name}`), + "kind", + "level", + ]); + const column = (name: string) => + source.columns.find((c) => c.name === name); + for (const name of texts) expect(column(`c ${name}`)?.default).toBe(name); + expect( + (column("kind") as unknown as { enum: { enumName: string } }).enum + .enumName, + ).toBe(close); + expect(column("kind")?.enumValues).toEqual(kind.values); + expect(column("level")?.enumValues).toEqual(level.values); + expect(column("level")?.default).toBe(bidi); + expect( + source.foreignKeys.map((fk) => + fk.reference().foreignColumns.map((c) => c.name), + ), + ).toEqual([[quote]]); + }); +}); diff --git a/packages/appkit/src/database/tooling/migrate.ts b/packages/appkit/src/database/tooling/migrate.ts index ec2343450..1613bb9d9 100644 --- a/packages/appkit/src/database/tooling/migrate.ts +++ b/packages/appkit/src/database/tooling/migrate.ts @@ -45,7 +45,7 @@ function readJournalText(folder: string): string | null { const file = journalFile(folder); return fs.existsSync(file) ? fs.readFileSync(file, "utf-8") : null; } -const DATABASE_TYPES_PATH = path.join( +export const DATABASE_TYPES_PATH = path.join( "shared/appkit-types", DATABASE_TYPES_FILE, ); diff --git a/packages/appkit/src/database/tooling/migrations.ts b/packages/appkit/src/database/tooling/migrations.ts index 81786a253..0983b5b74 100644 --- a/packages/appkit/src/database/tooling/migrations.ts +++ b/packages/appkit/src/database/tooling/migrations.ts @@ -27,8 +27,12 @@ interface GenerateOptions { /** Absolute migrations folder. */ readonly folder: string; readonly schema: Schema; + /** Default "migration" → "NNNN_migration". */ + readonly tag?: string; /** false: an ambiguous rename throws instead of prompting. */ readonly interactive: boolean; + /** Sees the SQL before anything is written; throwing writes nothing. */ + readonly beforeWrite?: (statements: readonly string[]) => void; } export interface GeneratedMigration { @@ -60,7 +64,7 @@ export class MigrationRenameAmbiguityError extends Error { kind === "column" ? `columns of table "${table}"` : `${kind}s`; super( `Cannot tell whether ${scope} were renamed: removed ${deleted.join(", ")}; added ${created.join(", ")}. ` + - "Run `appkit db migrate` in an interactive terminal to choose between a rename and a drop plus create.", + "Run the command in an interactive terminal to choose between a rename and a drop plus create.", ); this.name = "MigrationRenameAmbiguityError"; this.kind = kind; @@ -73,6 +77,7 @@ export class MigrationRenameAmbiguityError extends Error { /** drizzle-kit's statement separator in v7 migration files. */ const BREAKPOINT = "--> statement-breakpoint\n"; const ORIGIN_ID = "00000000-0000-0000-0000-000000000000"; +const TAG_NAME = /^[a-z0-9_]+$/; /** The parts of a drizzle-kit PostgreSQL snapshot this module reads. */ interface Snapshot { @@ -290,12 +295,18 @@ function writeFileAtomic(file: string, content: string): void { /** * Diff the schema against the latest committed snapshot with drizzle-kit and - * write the next v7 migration: `NNNN_migration.sql`, `meta/NNNN_snapshot.json`, + * write the next v7 migration: `NNNN_.sql`, `meta/NNNN_snapshot.json`, * and the appended `meta/_journal.json`. null when nothing changed. */ export async function generateMigrationFiles( options: GenerateOptions, ): Promise { + const name = options.tag ?? "migration"; + if (!TAG_NAME.test(name)) { + throw new Error( + `Migration tag "${name}" must use lowercase letters, digits and underscores`, + ); + } const { generateDrizzleJson, generateMigration } = await loadDrizzleKit(); const journal = readJournal(options.folder); @@ -319,11 +330,12 @@ export async function generateMigrationFiles( cur as unknown as KitSnapshot, ); if (statements.length === 0) return null; + options.beforeWrite?.(statements); const entries = journal?.entries ?? []; const last = entries.at(-1); const idx = last ? last.idx + 1 : 0; - const tag = `${migrationPrefix(idx)}_migration`; + const tag = `${migrationPrefix(idx)}_${name}`; const entry: JournalEntry = { idx, version: "7", diff --git a/packages/appkit/src/database/tooling/tests/contract.test.ts b/packages/appkit/src/database/tooling/tests/contract.test.ts index f4e3765d3..192305dcf 100644 --- a/packages/appkit/src/database/tooling/tests/contract.test.ts +++ b/packages/appkit/src/database/tooling/tests/contract.test.ts @@ -10,6 +10,7 @@ const exports: DatabaseTooling = tooling; const contractKeys = { createDbContext: true, + runIntrospect: true, runMigrate: true, } satisfies Record; diff --git a/packages/appkit/src/database/tooling/tests/migrations.test.ts b/packages/appkit/src/database/tooling/tests/migrations.test.ts index edfdf0fb8..c187abf92 100644 --- a/packages/appkit/src/database/tooling/tests/migrations.test.ts +++ b/packages/appkit/src/database/tooling/tests/migrations.test.ts @@ -303,6 +303,17 @@ describe("generateMigrationFiles", () => { expect(listFiles(folder)).toEqual(before); }); + test("rejects a tag that is not a safe file name", async () => { + await expect( + generateMigrationFiles({ + folder: tempFolder(), + schema: blog(), + tag: "../escape", + interactive: false, + }), + ).rejects.toThrow(/lowercase letters/); + }); + test("refuses to diff when the latest snapshot is missing", async () => { const folder = tempFolder(); await generateMigrationFiles({ @@ -342,6 +353,65 @@ describe("generateMigrationFiles", () => { ), ); }); + + test("beforeWrite sees the SQL first, and throwing from it writes nothing", async () => { + const folder = tempFolder(); + const seen: string[][] = []; + + const error = await generateMigrationFiles({ + folder, + schema: blog(), + interactive: false, + beforeWrite: (statements) => { + seen.push([...statements]); + throw new Error("refused"); + }, + }).catch((caught: unknown) => caught); + + expect((error as Error).message).toBe("refused"); + expect(seen).toHaveLength(1); + expect(seen[0]?.join("\n")).toContain('CREATE TABLE "app"."authors"'); + expect(listFiles(folder)).toEqual([]); + }); +}); + +describe("discardMigrationFiles", () => { + test("a first migration goes with the folders it created", async () => { + const folder = tempFolder(); + const migration = await generateMigrationFiles({ + folder, + schema: blog(), + interactive: false, + }); + + discardMigrationFiles(folder, migration as never, null); + + expect(fs.existsSync(folder)).toBe(false); + }); + + test("a later migration goes, and the previous journal comes back byte for byte", async () => { + const folder = tempFolder(); + await generateMigrationFiles({ + folder, + schema: blog(), + interactive: false, + }); + const journalFile = path.join(folder, "meta/_journal.json"); + const journalBefore = `${fs.readFileSync(journalFile, "utf-8")}\n`; + fs.writeFileSync(journalFile, journalBefore); + const filesBefore = listFiles(folder); + const migration = await generateMigrationFiles({ + folder, + schema: blog({ summary: text() }), + interactive: false, + }); + expect(listFiles(folder)).not.toEqual(filesBefore); + + discardMigrationFiles(folder, migration as never, journalBefore); + + expect(listFiles(folder)).toEqual(filesBefore); + expect(fs.readFileSync(journalFile, "utf-8")).toBe(journalBefore); + }); }); describe("breakingStatements", () => { diff --git a/packages/shared/src/cli/commands/db/index.ts b/packages/shared/src/cli/commands/db/index.ts index aca41b888..35ff2f721 100644 --- a/packages/shared/src/cli/commands/db/index.ts +++ b/packages/shared/src/cli/commands/db/index.ts @@ -1,5 +1,6 @@ import { Command } from "commander"; +import { dbIntrospectCommand } from "./introspect"; import { dbMigrateCommand } from "./migrate"; /** @@ -8,4 +9,5 @@ import { dbMigrateCommand } from "./migrate"; */ export const dbCommand = new Command("db") .description("Lakebase database lifecycle") - .addCommand(dbMigrateCommand); + .addCommand(dbMigrateCommand) + .addCommand(dbIntrospectCommand); diff --git a/packages/shared/src/cli/commands/db/introspect.test.ts b/packages/shared/src/cli/commands/db/introspect.test.ts new file mode 100644 index 000000000..8c9e2e6c1 --- /dev/null +++ b/packages/shared/src/cli/commands/db/introspect.test.ts @@ -0,0 +1,171 @@ +import { + afterEach, + beforeEach, + describe, + expect, + type MockInstance, + test, + vi, +} from "vitest"; + +const tooling = vi.hoisted(() => ({ + createDbContext: vi.fn(() => ({ root: "/app", env: {} })), + runIntrospect: vi.fn(), +})); + +vi.mock("@databricks/appkit/database-tooling", () => tooling); + +import { dbCommand } from "./index"; + +let log: MockInstance; +let error: MockInstance; +const output = (spy: MockInstance) => + spy.mock.calls.map((call) => call.join(" ")).join("\n"); + +const report = { + branch: "projects/p/branches/dev-alice-3f9a2c", + schemaName: "app", + tables: ["authors", "posts"], + schemaFile: `${process.cwd()}/config/database/schema.ts`, + backupFile: `${process.cwd()}/config/database/schema.ts.bak`, + skipped: [ + { + table: "events", + reason: "composite primary key (a, b) is not supported", + }, + { + table: "posts", + column: "score", + reason: 'type "numeric(10,2)" is not supported', + }, + ], + warnings: [ + "app.posts.score is NOT NULL with no default but is not in schema.ts.", + ], + approximated: [ + "app.authors.id: a serial key is declared as id(), an identity generated BY DEFAULT", + ], + migration: { + tag: "0000_baseline", + file: `${process.cwd()}/config/database/migrations/0000_baseline.sql`, + statements: ["CREATE TABLE a", "CREATE TABLE b"], + }, + recorded: ["0000_baseline"], + typesFile: `${process.cwd()}/shared/appkit-types/database.d.ts`, +}; + +beforeEach(() => { + for (const option of dbCommand.commands.flatMap((c) => + c.options.map((o) => [c, o] as const), + )) { + option[0].setOptionValueWithSource( + option[1].attributeName(), + undefined, + "default", + ); + } + tooling.runIntrospect.mockReset().mockResolvedValue(report); + log = vi.spyOn(console, "log").mockImplementation(() => undefined); + error = vi.spyOn(console, "error").mockImplementation(() => undefined); +}); + +afterEach(() => { + vi.restoreAllMocks(); + process.exitCode = undefined; +}); + +describe("appkit db introspect", () => { + test("prints the written schema, each skip, the loud warnings, and the recorded baseline", async () => { + await dbCommand.parseAsync(["introspect"], { from: "user" }); + + expect(tooling.runIntrospect).toHaveBeenCalledWith( + { root: "/app", env: {} }, + { interactive: false }, + ); + const printed = output(log); + expect(printed).toContain( + "Wrote config/database/schema.ts from schema app on dev-alice-3f9a2c (2 tables); the previous file is config/database/schema.ts.bak", + ); + expect(printed).toContain( + "Skipped table events: composite primary key (a, b) is not supported", + ); + expect(printed).toContain( + 'Skipped posts.score: type "numeric(10,2)" is not supported', + ); + expect(printed).toContain("app.posts.score is NOT NULL with no default"); + expect(printed).toContain( + "Approximated app.authors.id: a serial key is declared as id(), an identity generated BY DEFAULT", + ); + expect(printed).toContain( + "This branch keeps the approximated objects as they are; a database the migrations create gets what schema.ts declares.", + ); + expect(printed).toContain( + "Generated config/database/migrations/0000_baseline.sql (2 statements)", + ); + expect(printed).toContain( + "Recorded 0000_baseline as applied on dev-alice-3f9a2c: the branch already has what it describes", + ); + expect(printed).toContain("Generated shared/appkit-types/database.d.ts"); + expect(process.exitCode).toBeUndefined(); + }); + + test("--schema-name is passed through", async () => { + await dbCommand.parseAsync(["introspect", "--schema-name", "app"], { + from: "user", + }); + + expect(tooling.runIntrospect).toHaveBeenCalledWith(expect.anything(), { + interactive: false, + schemaName: "app", + }); + }); + + test("migrations an interrupted run left unrecorded are listed with the one it recorded", async () => { + tooling.runIntrospect.mockResolvedValue({ + ...report, + migration: null, + recorded: ["0002_introspect"], + }); + + await dbCommand.parseAsync(["introspect"], { from: "user" }); + + expect(output(log)).toContain( + "Recorded 0002_introspect as applied on dev-alice-3f9a2c: the branch already has what it describes", + ); + }); + + test.each([ + [[], "No migration needed: schema.ts has no tables"], + [ + ["authors"], + "No migration needed: the migrations already describe this schema", + ], + ])("without a migration, %j tables print %j", async (tables, line) => { + tooling.runIntrospect.mockResolvedValue({ + ...report, + tables, + migration: null, + recorded: [], + }); + + await dbCommand.parseAsync(["introspect"], { from: "user" }); + + expect(output(log)).toContain(line); + expect(output(log)).not.toContain("Recorded"); + }); + + test("a refusal is printed and exits 1", async () => { + tooling.runIntrospect.mockRejectedValue( + new Error( + "Refusing to introspect: production is not your dev or session branch (it is the project's default branch).", + ), + ); + + await dbCommand.parseAsync(["introspect"], { from: "user" }); + + expect(output(error)).toContain( + "Refusing to introspect: production is not your dev or session branch", + ); + expect(process.exitCode).toBe(1); + }); +}); diff --git a/packages/shared/src/cli/commands/db/introspect.ts b/packages/shared/src/cli/commands/db/introspect.ts new file mode 100644 index 000000000..1d2b37031 --- /dev/null +++ b/packages/shared/src/cli/commands/db/introspect.ts @@ -0,0 +1,43 @@ +import { Command } from "commander"; + +import { loadDatabaseTooling } from "./load-tooling"; +import { fail, isInteractive, printIntrospectReport } from "./output"; + +async function runDbIntrospect(flags: { schemaName?: string }): Promise { + try { + const tooling = await loadDatabaseTooling(); + const report = await tooling.runIntrospect(tooling.createDbContext(), { + interactive: isInteractive(), + ...(flags.schemaName ? { schemaName: flags.schemaName } : {}), + }); + printIntrospectReport(report); + } catch (error) { + fail(error); + } +} + +export const dbIntrospectCommand = new Command("introspect") + .description( + "Write config/database/schema.ts from the .env branch's tables, plus the migration that brings history up to them", + ) + .option("--schema-name ", "Postgres schema to read (default: app)") + .addHelpText( + "after", + ` +The previous schema.ts is kept as schema.ts.bak. What the schema builder cannot +express (composite keys, unsupported types or defaults, some foreign keys) is left +out with an "// appkit: skipped …" comment. + +Without migration history this writes 0000_baseline; with history, the next +NNNN_introspect, once the branch has applied every committed migration. The +migration is recorded as applied, so introspect runs only on your dev or session +branch; \`appkit db init\` creates one from the project's default branch. The +deployed app adopts 0000_baseline on start when its tables already exist. If +anything fails, schema.ts, database.d.ts and the migrations are left as they +were. + +Examples: + $ appkit db introspect + $ appkit db introspect --schema-name sales`, + ) + .action(runDbIntrospect); diff --git a/packages/shared/src/cli/commands/db/output.ts b/packages/shared/src/cli/commands/db/output.ts index f3dc540a4..b73bbe163 100644 --- a/packages/shared/src/cli/commands/db/output.ts +++ b/packages/shared/src/cli/commands/db/output.ts @@ -3,7 +3,7 @@ import process from "node:process"; import pc from "picocolors"; -import type { MigrateReport } from "./tooling-contract"; +import type { IntrospectReport, MigrateReport } from "./tooling-contract"; /** Prompts only when a human can answer them. */ export function isInteractive(): boolean { @@ -14,6 +14,11 @@ export function relative(file: string): string { return path.relative(process.cwd(), file) || file; } +/** The last segment of a Lakebase resource name: projects/p/branches/dev-a → dev-a. */ +function shortName(name: string): string { + return name.slice(name.lastIndexOf("/") + 1); +} + export const ok = (message: string) => console.log(`${pc.green("✔")} ${message}`); export const info = (message: string) => @@ -73,3 +78,44 @@ export function printMigrateReport(report: MigrateReport): void { ok(`Generated ${relative(report.typesFile)}`); } + +export function printIntrospectReport(report: IntrospectReport): void { + const count = report.tables.length; + ok( + `Wrote ${relative(report.schemaFile)} from schema ${report.schemaName} on ${shortName(report.branch)} (${count} table${count === 1 ? "" : "s"})${report.backupFile ? `; the previous file is ${relative(report.backupFile)}` : ""}`, + ); + for (const skip of report.skipped) { + warn( + `Skipped ${skip.column ? `${skip.table}.${skip.column}` : `table ${skip.table}`}: ${skip.reason}`, + ); + } + for (const warning of report.warnings) { + console.log(`${pc.bold(pc.red("!"))} ${pc.bold(warning)}`); + } + for (const difference of report.approximated) { + info(`Approximated ${difference}`); + } + if (report.approximated.length > 0) { + info( + "This branch keeps the approximated objects as they are; a database the migrations create gets what schema.ts declares.", + ); + } + if (report.migration) { + const statements = report.migration.statements.length; + ok( + `Generated ${relative(report.migration.file)} (${statements} statement${statements === 1 ? "" : "s"})`, + ); + } else { + info( + report.tables.length === 0 + ? "No migration needed: schema.ts has no tables" + : "No migration needed: the migrations already describe this schema", + ); + } + if (report.recorded.length > 0) { + ok( + `Recorded ${report.recorded.join(", ")} as applied on ${shortName(report.branch)}: the branch already has ${report.recorded.length === 1 ? "what it describes" : "what they describe"}`, + ); + } + ok(`Generated ${relative(report.typesFile)}`); +} diff --git a/packages/shared/src/cli/commands/db/tooling-contract.ts b/packages/shared/src/cli/commands/db/tooling-contract.ts index 06e73e359..cb39f69b2 100644 --- a/packages/shared/src/cli/commands/db/tooling-contract.ts +++ b/packages/shared/src/cli/commands/db/tooling-contract.ts @@ -19,6 +19,26 @@ export interface MigrateReport { readonly typesFile: string; } +export interface Skip { + readonly table: string; + readonly column?: string; + readonly reason: string; +} + +export interface IntrospectReport { + readonly branch: string; + readonly schemaName: string; + readonly tables: readonly string[]; + readonly schemaFile: string; + readonly backupFile: string | null; + readonly skipped: readonly Skip[]; + readonly warnings: readonly string[]; + readonly approximated: readonly string[]; + readonly migration: GeneratedMigration | null; + readonly recorded: readonly string[]; + readonly typesFile: string; +} + /** * The exports of `@databricks/appkit/database-tooling` the CLI calls. `shared` * must not depend on appkit, so the CLI imports the tooling dynamically and @@ -35,4 +55,8 @@ export interface DatabaseTooling { ctx: Ctx, options: { interactive: boolean }, ) => Promise; + readonly runIntrospect: ( + ctx: Ctx, + options: { interactive: boolean; schemaName?: string }, + ) => Promise; }