diff --git a/packages/appkit/src/database/migrations/apply.ts b/packages/appkit/src/database/migrations/apply.ts index c8e5f9c3f..b9f528640 100644 --- a/packages/appkit/src/database/migrations/apply.ts +++ b/packages/appkit/src/database/migrations/apply.ts @@ -147,7 +147,7 @@ export class MigrationBaselineMismatchError extends Error { constructor(schemaName: string, problems: readonly string[]) { super( `Cannot adopt ${BASELINE_TAG}: schema "${schemaName}" has no recorded migrations and only part of the baseline: ${problems.join("; ")}. ` + - "Create the missing tables and columns (or drop the partial ones), or regenerate the baseline against this database.", + "Create the missing tables and columns (or drop the partial ones), or regenerate the baseline with `appkit db introspect` against this database.", ); this.name = "MigrationBaselineMismatchError"; this.problems = Object.freeze([...problems]); @@ -575,6 +575,124 @@ function duration(ms: number): string { return `${ms}ms`; } +/** How the branch's recorded migrations differ from the committed ones. */ +interface MigrationDrift { + /** Committed, not recorded, newer than every recorded one. */ + readonly pending: readonly string[]; + /** Committed, not recorded, older than the last recorded one. */ + readonly stranded: readonly string[]; + /** Recorded rows no committed migration matches: the branch is ahead of the checkout. */ + readonly unknown: number; +} + +function driftOf( + history: ReturnType, +): MigrationDrift { + return { + pending: tags(history.pending), + stranded: tags(history.stranded), + unknown: history.unknown.length, + }; +} + +function committedMigrations(folder: string): MigrationFile[] { + const journal = readJournal(folder); + return journal ? readMigrations(folder, journal) : []; +} + +/** Read without the lock; markMigrationsApplied checks again under it. */ +export async function readMigrationDrift( + client: QueryClient, + options: ApplyMigrationsOptions, +): Promise { + return driftOf( + reconcileMigrations( + committedMigrations(options.migrationsFolder), + await readRecordedMigrations(client, options.schemaName), + ), + ); +} + +/** Why the branch's history is not the committed one plus toRecord; undefined when it is. */ +function driftReason( + drift: MigrationDrift, + toRecord: readonly string[], +): string | undefined { + const notApplied = drift.pending.filter((tag) => !toRecord.includes(tag)); + const recorded = toRecord.filter((tag) => !drift.pending.includes(tag)); + const reasons = [ + drift.unknown > 0 + ? `it records ${drift.unknown} migration${drift.unknown === 1 ? "" : "s"} this checkout does not have` + : "", + drift.stranded.length > 0 ? `it skipped ${drift.stranded.join(", ")}` : "", + notApplied.length > 0 ? `it has not applied ${notApplied.join(", ")}` : "", + recorded.length > 0 ? `it already records ${recorded.join(", ")}` : "", + ].filter(Boolean); + return reasons.length > 0 ? reasons.join("; ") : undefined; +} + +/** + * Record migrations as applied without running them: the branch already holds + * what they describe. Under the migration lock, and only when they are exactly + * the branch's pending migrations, so DDL the branch never ran is not recorded. + */ +export async function markMigrationsApplied( + pool: Pick, + options: ApplyMigrationsOptions, + toRecord: readonly string[], +): Promise { + const files = committedMigrations(options.migrationsFolder); + const missing = toRecord.filter( + (tag) => !files.some((file) => file.tag === tag), + ); + if (missing.length > 0) { + throw new MigrationFilesError( + `Cannot record ${missing.join(", ")} as applied: not in ${options.migrationsFolder}/meta/_journal.json.`, + ); + } + const { schemaName } = options; + const client = await pool.connect(); + const reset: string[] = []; + let locked = false; + try { + await prepareMigrationSession(client, reset); + await acquireMigrationLock( + client, + schemaName, + options.lockWaitMs ?? LOCK_WAIT_MS, + ); + locked = true; + const history = reconcileMigrations( + files, + await readRecordedMigrations(client, schemaName), + ); + const reason = driftReason(driftOf(history), toRecord); + if (reason) { + throw new Error( + `Schema "${schemaName}" changed while this ran (${reason}), so nothing was recorded.`, + ); + } + await inTransaction(client, async () => { + await ensureJournalTable(client, schemaName); + for (const file of history.pending) { + await recordMigration(client, schemaName, file); + } + }); + } catch (error) { + if (!(error instanceof MigrationLockWaitError)) throw error; + throw new Error( + `${error.holder ? `Session ${error.holder}` : "Another session"} held the migration lock of schema "${schemaName}" for over ${duration(options.lockWaitMs ?? LOCK_WAIT_MS)}, so nothing was recorded. Rerun once it finishes.`, + { cause: error }, + ); + } finally { + await releaseMigrationClient( + client, + locked ? migrationLockKey(schemaName) : undefined, + reset, + ); + } +} + /** * Apply every committed migration the database has not recorded yet. * diff --git a/packages/appkit/src/database/migrations/tests/apply.pg.test.ts b/packages/appkit/src/database/migrations/tests/apply.pg.test.ts index 7e4875576..c3b2d7b99 100644 --- a/packages/appkit/src/database/migrations/tests/apply.pg.test.ts +++ b/packages/appkit/src/database/migrations/tests/apply.pg.test.ts @@ -16,8 +16,11 @@ import { applyPendingMigrations, MigrationApplyError, MigrationHistoryError, + markMigrationsApplied, migrationLockKey, + readMigrationDrift, } from "../apply"; +import { MigrationFilesError } from "../paths"; import { hashOf, removeMigrationFolders, writeMigrations } from "./fixtures"; // A real server, for what a fake cannot show: transactions, enum rules, locks @@ -405,4 +408,130 @@ describe.skipIf(!url)("applyPendingMigrations against Postgres", () => { }); expect(await columns("notes")).toEqual(["id", "body"]); }); + + describe("recording migrations the branch already has", () => { + const withTags = () => + writeMigrations([ + { + tag: "0000_notes", + when: 1000, + sql: [`CREATE TABLE ${q("notes")} ("id" integer PRIMARY KEY);`], + }, + { + tag: "0001_introspect", + when: 3000, + sql: [`ALTER TABLE ${q("notes")} ADD COLUMN "body" text;`], + }, + ]); + /** The branch applied 0000_notes and recorded a 2000 migration this checkout lacks. */ + const aheadOfCheckout = async (folder: string) => { + await applyPendingMigrations(newPool(), { + ...options(folder), + migrationsFolder: notes(), + }); + await admin.query( + `delete from ${q("__drizzle_migrations")} where created_at <> 1000`, + ); + await admin.query( + `insert into ${q("__drizzle_migrations")} (hash, created_at) values ('teammate', 2000)`, + ); + }; + + test("the drift lists unrecorded committed migrations and recorded ones the checkout lacks", async () => { + const folder = withTags(); + expect(await readMigrationDrift(newPool(), options(folder))).toEqual({ + pending: ["0000_notes", "0001_introspect"], + stranded: [], + unknown: 0, + }); + + await aheadOfCheckout(folder); + + expect(await readMigrationDrift(newPool(), options(folder))).toEqual({ + pending: ["0001_introspect"], + stranded: [], + unknown: 1, + }); + }); + + test("records exactly the pending migrations, without running them", async () => { + await admin.query(`create schema "${schema}"`); + const folder = withTags(); + + await markMigrationsApplied(newPool(), options(folder), [ + "0000_notes", + "0001_introspect", + ]); + + expect(await recorded()).toEqual([ + { hash: hashOf(folder, "0000_notes"), createdAt: 1000 }, + { hash: hashOf(folder, "0001_introspect"), createdAt: 3000 }, + ]); + expect(await columns("notes")).toEqual([]); + }); + + test.each([ + [ + "records a migration the checkout lacks", + aheadOfCheckout, + ["0001_introspect"], + "it records 1 migration this checkout does not have", + ], + [ + "has not applied an earlier one", + async () => { + await admin.query(`create schema "${schema}"`); + }, + ["0001_introspect"], + "it has not applied 0000_notes", + ], + ])( + "refuses when the branch %s, and records nothing", + async (_label, arrange, toRecord, reason) => { + const folder = withTags(); + await arrange(folder); + const before = await recorded().catch(() => []); + + await expect( + markMigrationsApplied(newPool(), options(folder), toRecord), + ).rejects.toThrow( + `Schema "${schema}" changed while this ran (${reason}), so nothing was recorded.`, + ); + expect(await recorded().catch(() => [])).toEqual(before); + }, + ); + + test("a tag missing from the journal fails before connecting", async () => { + const pool = newPool(); + const connect = vi.spyOn(pool, "connect"); + + await expect( + markMigrationsApplied(pool, options(withTags()), ["0002_introspect"]), + ).rejects.toBeInstanceOf(MigrationFilesError); + expect(connect).not.toHaveBeenCalled(); + }); + + test("the wait for the migration lock has a limit, and nothing is recorded", async () => { + await admin.query(`create schema "${schema}"`); + const holder = await admin.connect(); + held.push(holder); + await holder.query("set application_name = 'another-deploy'"); + await holder.query("select pg_advisory_lock($1::bigint)", [ + migrationLockKey(schema), + ]); + + await expect( + markMigrationsApplied( + newPool(), + { ...options(withTags()), lockWaitMs: 300 }, + ["0000_notes", "0001_introspect"], + ), + ).rejects.toThrow( + new RegExp( + `^Session pid \\d+ \\(another-deploy, idle\\) held the migration lock of schema "${schema}" for over 300ms, so nothing was recorded\\. Rerun once it finishes\\.$`, + ), + ); + expect(await recorded().catch(() => [])).toEqual([]); + }); + }); }); diff --git a/packages/appkit/src/database/schema-builder/define-schema.ts b/packages/appkit/src/database/schema-builder/define-schema.ts index 5d8cf3d91..8d8038d08 100644 --- a/packages/appkit/src/database/schema-builder/define-schema.ts +++ b/packages/appkit/src/database/schema-builder/define-schema.ts @@ -72,9 +72,27 @@ function assertName(value: string, label: string): void { } } +/** Table names defineSchema() refuses. */ +export function isReservedTableName(name: string): boolean { + return ( + !name || + RESERVED_OBJECT_KEYS.has(name) || + RESERVED_DATABASE_EXPORT_KEYS.has(name) + ); +} + +/** Column keys that collide with the runtime metadata on a table handle. */ +function collidesWithMetadata(key: string): boolean { + return key.startsWith("$") || key === "and" || key === "or"; +} + +/** Column names defineSchema() refuses. */ +export function isReservedColumnName(name: string): boolean { + return !name || RESERVED_OBJECT_KEYS.has(name) || collidesWithMetadata(name); +} + function assertTableName(value: string): void { - assertName(value, "Table name"); - if (RESERVED_DATABASE_EXPORT_KEYS.has(value)) { + if (isReservedTableName(value)) { throw new SchemaBuildError(`Table name "${value}" is reserved`); } } @@ -120,7 +138,7 @@ function declareTable>( const handle = Object.create(null) as Record; for (const [key, column] of Object.entries(columns)) { assertName(key, `Column name on table "${name}"`); - if (key.startsWith("$") || key === "and" || key === "or") { + if (collidesWithMetadata(key)) { throw new SchemaBuildError( `Column "${name}.${key}" collides with AppKit runtime metadata`, ); @@ -344,7 +362,7 @@ export function assertFinalizedSchema(value: unknown): asserts value is Schema { * the deployed app's service principal can create and own a schema of its * own, but has no CREATE on `public`. */ -const DEFAULT_SCHEMA_NAME = "app"; +export const DEFAULT_SCHEMA_NAME = "app"; /** * Compile one declared schema. The returned type keeps the table names the diff --git a/packages/appkit/src/database/tooling/index.ts b/packages/appkit/src/database/tooling/index.ts index bbec5337a..29adba529 100644 --- a/packages/appkit/src/database/tooling/index.ts +++ b/packages/appkit/src/database/tooling/index.ts @@ -7,4 +7,9 @@ * the same AppKit release should import it. */ export { createDbContext, type DbContext } from "./context"; +export { + type IntrospectOptions, + type IntrospectReport, + runIntrospect, +} from "./introspect"; export { type MigrateOptions, type MigrateReport, runMigrate } from "./migrate"; diff --git a/packages/appkit/src/database/tooling/introspect/catalog.ts b/packages/appkit/src/database/tooling/introspect/catalog.ts new file mode 100644 index 000000000..07703b0aa --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/catalog.ts @@ -0,0 +1,322 @@ +import { MIGRATIONS_TABLE } from "../../migrations/paths"; +import type { ReferentialAction } from "../../schema-builder/types"; + +/** Run one parameterized query and return its rows (a pg Pool fits: `(t, v) => pool.query(t, v).then(r => r.rows)`). */ +export type CatalogQuery = ( + text: string, + values: readonly unknown[], +) => Promise; + +export interface IntrospectedEnum { + readonly name: string; + readonly schema: string; + readonly values: readonly string[]; +} + +export interface IntrospectedColumn { + readonly name: string; + /** format_type(), e.g. "character varying(120)". */ + readonly type: string; + readonly notNull: boolean; + /** attidentity: "" none, "a" always, "d" by default. */ + readonly identity: "" | "a" | "d"; + /** Generated column, stored or virtual (attgenerated 's' or 'v'). */ + readonly generated: boolean; + /** pg_get_expr() of the default (or generation expression). */ + readonly defaultExpr: string | null; + readonly enum?: IntrospectedEnum; +} + +export interface IntrospectedForeignKey { + readonly name: string; + readonly columns: readonly string[]; + readonly schema: string; + readonly table: string; + readonly refColumns: readonly string[]; + readonly onDelete: ReferentialAction; + readonly onUpdate: ReferentialAction; + /** DEFERRABLE or MATCH FULL, neither of which the builder declares. */ + readonly deferrable: boolean; + readonly matchFull: boolean; +} + +export interface IntrospectedUnique { + readonly name: string; + readonly columns: readonly string[]; +} + +export interface IntrospectedTable { + readonly name: string; + /** relkind 'p': partitioned in the database. */ + readonly partitioned: boolean; + readonly columns: readonly IntrospectedColumn[]; + readonly primaryKey: readonly string[]; + readonly unique: readonly IntrospectedUnique[]; + readonly foreignKeys: readonly IntrospectedForeignKey[]; + /** CHECK and EXCLUDE constraint names. */ + readonly checks: readonly string[]; + /** Index names, except the indexes behind primary-key, unique and exclusion constraints. */ + readonly indexes: readonly string[]; +} + +/** pg_constraint.confdeltype / confupdtype. */ +const ACTIONS: Record = { + a: "no action", + r: "restrict", + c: "cascade", + n: "set null", + d: "set default", +}; + +export interface ColumnRow { + table_name: string; + name: string; + type: string; + not_null: boolean; + identity: string; + generated: string; + default_expr: string | null; + type_kind: string; + type_schema: string; + type_name: string; + enum_values: string[] | null; +} + +export interface TableRow { + name: string; + partitioned: boolean; +} + +export interface ConstraintRow { + table_name: string; + name: string; + kind: "p" | "u" | "f" | "c" | "x"; + columns: string[]; + ref_schema: string | null; + ref_table: string | null; + ref_columns: string[] | null; + on_delete: string; + on_update: string; + deferrable?: boolean; + /** confmatchtype: "f" full, "s" simple. */ + match_type?: string; +} + +export interface IndexRow { + table_name: string; + name: string; +} + +// pg_catalog, never information_schema: it hides objects the role cannot access. +// Ordinary and partitioned tables only; partitions belong to their parent. +const TABLES_SQL = ` +select c.relname::text as name, c.relkind = 'p' as partitioned + from pg_catalog.pg_class c + join pg_catalog.pg_namespace n on n.oid = c.relnamespace + where n.nspname = $1 and c.relkind in ('r', 'p') and not c.relispartition + and c.relname <> $2 + order by c.relname`; + +const COLUMNS_SQL = ` +select c.relname::text as table_name, a.attname::text as name, + pg_catalog.format_type(a.atttypid, a.atttypmod) as type, + a.attnotnull as not_null, a.attidentity::text as identity, + a.attgenerated::text as generated, + pg_catalog.pg_get_expr(d.adbin, d.adrelid) as default_expr, + t.typtype::text as type_kind, tn.nspname::text as type_schema, + t.typname::text as type_name, + case when t.typtype = 'e' then array( + select e.enumlabel::text from pg_catalog.pg_enum e + where e.enumtypid = t.oid order by e.enumsortorder) + end as enum_values + from pg_catalog.pg_attribute a + join pg_catalog.pg_class c on c.oid = a.attrelid + join pg_catalog.pg_namespace n on n.oid = c.relnamespace + join pg_catalog.pg_type t on t.oid = a.atttypid + join pg_catalog.pg_namespace tn on tn.oid = t.typnamespace + left join pg_catalog.pg_attrdef d on d.adrelid = a.attrelid and d.adnum = a.attnum + where n.nspname = $1 and c.relkind in ('r', 'p') and not c.relispartition + and c.relname <> $2 and a.attnum > 0 and not a.attisdropped + order by c.relname, a.attnum`; + +const CONSTRAINTS_SQL = ` +select c.relname::text as table_name, k.conname::text as name, k.contype::text as kind, + array(select a.attname::text + from unnest(k.conkey) with ordinality as u(attnum, ord) + join pg_catalog.pg_attribute a on a.attrelid = k.conrelid and a.attnum = u.attnum + order by u.ord) as columns, + rn.nspname::text as ref_schema, rc.relname::text as ref_table, + array(select a.attname::text + from unnest(k.confkey) with ordinality as u(attnum, ord) + join pg_catalog.pg_attribute a on a.attrelid = k.confrelid and a.attnum = u.attnum + order by u.ord) as ref_columns, + k.confdeltype::text as on_delete, k.confupdtype::text as on_update, + k.condeferrable as deferrable, k.confmatchtype::text as match_type + from pg_catalog.pg_constraint k + join pg_catalog.pg_class c on c.oid = k.conrelid + join pg_catalog.pg_namespace n on n.oid = c.relnamespace + left join pg_catalog.pg_class rc on rc.oid = k.confrelid + left join pg_catalog.pg_namespace rn on rn.oid = rc.relnamespace + where n.nspname = $1 and k.contype in ('p', 'u', 'f', 'c', 'x') and c.relname <> $2 + -- A foreign key to a partitioned table has one copy per partition; keep the original. + and k.conparentid = 0 + order by c.relname, k.conname`; + +// Indexes no constraint stands for: schema.ts cannot declare them. +const INDEXES_SQL = ` +select c.relname::text as table_name, i.relname::text as name + from pg_catalog.pg_index x + join pg_catalog.pg_class c on c.oid = x.indrelid + join pg_catalog.pg_class i on i.oid = x.indexrelid + join pg_catalog.pg_namespace n on n.oid = c.relnamespace + where n.nspname = $1 and c.relname <> $2 + and not exists ( + select 1 from pg_catalog.pg_constraint k + where k.conrelid = x.indrelid and k.conindid = x.indexrelid + and k.contype in ('p', 'u', 'x')) + order by c.relname, i.relname`; + +// What readCatalog does not read, so the report can name it. Objects an +// extension owns are left out: the extension recreates them. +const UNREAD_SQL = ` +with ns as (select oid from pg_catalog.pg_namespace where nspname = $1), +objects (label, classid, objid) as ( + select case c.relkind when 'v' then 'view ' when 'm' then 'materialized view ' + when 'f' then 'foreign table ' else 'sequence ' end || c.relname, + 'pg_catalog.pg_class'::regclass, c.oid + from pg_catalog.pg_class c + where c.relnamespace = (select oid from ns) and c.relkind in ('v', 'm', 'f', 'S') + -- serial and identity sequences belong to their column + and not exists ( + select 1 from pg_catalog.pg_depend d + where d.classid = 'pg_catalog.pg_class'::regclass and d.objid = c.oid + and d.refclassid = 'pg_catalog.pg_class'::regclass and d.deptype in ('a', 'i')) + union all + select 'trigger ' || t.tgname || ' on ' || c.relname, 'pg_catalog.pg_trigger'::regclass, t.oid + from pg_catalog.pg_trigger t + join pg_catalog.pg_class c on c.oid = t.tgrelid + where c.relnamespace = (select oid from ns) and not t.tgisinternal + union all + select 'policy ' || p.polname || ' on ' || c.relname, 'pg_catalog.pg_policy'::regclass, p.oid + from pg_catalog.pg_policy p + join pg_catalog.pg_class c on c.oid = p.polrelid + where c.relnamespace = (select oid from ns) + union all + select 'row-level security on ' || c.relname, 'pg_catalog.pg_class'::regclass, c.oid + from pg_catalog.pg_class c + where c.relnamespace = (select oid from ns) and c.relrowsecurity + union all + select 'inheritance of ' || c.relname || ' from ' || p.relname, 'pg_catalog.pg_class'::regclass, c.oid + from pg_catalog.pg_inherits i + join pg_catalog.pg_class c on c.oid = i.inhrelid + join pg_catalog.pg_class p on p.oid = i.inhparent + where c.relnamespace = (select oid from ns) and not c.relispartition + union all + select case p.prokind when 'p' then 'procedure ' when 'a' then 'aggregate ' else 'function ' end + || p.proname || '(' || pg_catalog.pg_get_function_identity_arguments(p.oid) || ')', + 'pg_catalog.pg_proc'::regclass, p.oid + from pg_catalog.pg_proc p + where p.pronamespace = (select oid from ns) + union all + select case t.typtype when 'd' then 'domain ' else 'type ' end || t.typname, + 'pg_catalog.pg_type'::regclass, t.oid + from pg_catalog.pg_type t + where t.typnamespace = (select oid from ns) + and (t.typtype in ('d', 'r') + or (t.typtype = 'c' and exists ( + select 1 from pg_catalog.pg_class c where c.oid = t.typrelid and c.relkind = 'c')) + -- an enum is read through the columns that use it + or (t.typtype = 'e' and not exists ( + select 1 from pg_catalog.pg_attribute a + join pg_catalog.pg_class c on c.oid = a.attrelid + where a.atttypid = t.oid and c.relkind in ('r', 'p') and not a.attisdropped))) +) +select o.label from objects o + where not exists ( + select 1 from pg_catalog.pg_depend d + where d.classid = o.classid and d.objid = o.objid and d.deptype = 'e') + order by o.label`; + +function toColumn(row: ColumnRow): IntrospectedColumn { + const identity = + row.identity === "a" || row.identity === "d" ? row.identity : ""; + return { + name: row.name, + type: row.type, + notNull: row.not_null, + identity, + generated: row.generated !== "", + defaultExpr: row.default_expr, + ...(row.type_kind === "e" + ? { + enum: { + name: row.type_name, + schema: row.type_schema, + values: row.enum_values ?? [], + }, + } + : {}), + }; +} + +/** Assemble the model from the catalog result sets (exported for fixture tests). */ +export function buildTables( + tables: readonly TableRow[], + columns: readonly ColumnRow[], + constraints: readonly ConstraintRow[], + indexes: readonly IndexRow[] = [], +): IntrospectedTable[] { + return tables.map(({ name, partitioned }) => { + const own = constraints.filter((row) => row.table_name === name); + return { + name, + partitioned, + columns: columns.filter((row) => row.table_name === name).map(toColumn), + primaryKey: own.find((row) => row.kind === "p")?.columns ?? [], + unique: own + .filter((row) => row.kind === "u") + .map((row) => ({ name: row.name, columns: row.columns })), + foreignKeys: own + .filter((row) => row.kind === "f" && row.ref_table) + .map((row) => ({ + name: row.name, + columns: row.columns, + schema: row.ref_schema ?? "", + table: row.ref_table ?? "", + refColumns: row.ref_columns ?? [], + onDelete: ACTIONS[row.on_delete] ?? "no action", + onUpdate: ACTIONS[row.on_update] ?? "no action", + deferrable: row.deferrable === true, + matchFull: row.match_type === "f", + })), + checks: own + .filter((row) => row.kind === "c" || row.kind === "x") + .map((row) => row.name), + indexes: indexes + .filter((row) => row.table_name === name) + .map((row) => row.name), + }; + }); +} + +/** Tables of one schema from pg_catalog, excluding the migration journal. */ +export async function readCatalog( + query: CatalogQuery, + schemaName: string, +): Promise { + const params = [schemaName, MIGRATIONS_TABLE]; + const tables = await query(TABLES_SQL, params); + const columns = await query(COLUMNS_SQL, params); + const constraints = await query(CONSTRAINTS_SQL, params); + const indexes = await query(INDEXES_SQL, params); + return buildTables(tables, columns, constraints, indexes); +} + +/** Objects of the schema readCatalog does not read, e.g. "trigger audit on notes". */ +export async function readUnreadObjects( + query: CatalogQuery, + schemaName: string, +): Promise { + const rows = await query<{ label: string }>(UNREAD_SQL, [schemaName]); + return rows.map((row) => row.label); +} diff --git a/packages/appkit/src/database/tooling/introspect/index.ts b/packages/appkit/src/database/tooling/introspect/index.ts new file mode 100644 index 000000000..66ce9bd7d --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/index.ts @@ -0,0 +1,315 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { + DEFAULT_SCHEMA_FILE, + loadDefaultDatabaseSchema, +} from "../../../plugins/database/load-schema"; +import { writeDatabaseTypes } from "../../../type-generator/database"; +import { + markMigrationsApplied, + readMigrationDrift, +} from "../../migrations/apply"; +import { + journalFile, + migrationsFolder, + readJournal, +} from "../../migrations/paths"; +import { DEFAULT_SCHEMA_NAME } from "../../schema-builder/define-schema"; +import { type DbContext, NO_ENV_BRANCH, withBranchPool } from "../context"; +import { notYourBranch, resolveEnvBranch, takesWrites } from "../guard"; +import { DATABASE_TYPES_PATH } from "../migrate"; +import { + breakingStatements, + discardMigrationFiles, + type GeneratedMigration, + generateMigrationFiles, +} from "../migrations"; +import { resourceId } from "../targets"; +import { readCatalog, readUnreadObjects } from "./catalog"; +import { + renderSchemaSource, + type Skip, + shown, + type Unexpressed, +} from "./render"; + +export interface IntrospectOptions { + /** false: an ambiguous rename against existing history throws instead of prompting. */ + readonly interactive: boolean; + /** Schema to read; defaults to DEFAULT_SCHEMA_NAME. The current schema.ts is never read. */ + readonly schemaName?: string; + /** Module the written schema.ts imports the builder from. Default "@databricks/appkit/beta". */ + readonly importSource?: string; +} + +export interface IntrospectReport { + readonly branch: string; + readonly schemaName: string; + /** Tables written to schema.ts. */ + readonly tables: readonly string[]; + readonly schemaFile: string; + /** Where the previous schema.ts went, when this run replaced a different one. */ + readonly backupFile: string | null; + readonly skipped: readonly Skip[]; + readonly warnings: readonly string[]; + /** What schema.ts declares differently from the database, which keeps it as it is. */ + readonly approximated: readonly string[]; + /** 0000_baseline without history, NNNN_introspect with it; null when nothing changed. */ + readonly migration: GeneratedMigration | null; + /** Tags recorded as applied: the migration, and any an interrupted run wrote but did not record. */ + readonly recorded: readonly string[]; + readonly typesFile: string; +} + +/** Introspect would record history the branch or the deployed app contradicts. */ +export class IntrospectRefusedError extends Error { + constructor(message: string) { + super(message); + this.name = "IntrospectRefusedError"; + } +} + +/** Tags introspect writes. Pending ones match the branch when a fresh render adds nothing. */ +const INTROSPECT_TAG = /^\d{4}_(introspect|baseline)$/; + +/** File contents, or null when it does not exist. */ +function readIfExists(file: string): string | null { + return fs.existsSync(file) ? fs.readFileSync(file, "utf-8") : null; +} + +function writeOrRemove(file: string, contents: string | null): void { + if (contents === null) fs.rmSync(file, { force: true }); + else fs.writeFileSync(file, contents); +} + +const plural = (count: number, word: string) => + `${count} ${word}${count === 1 ? "" : "s"}`; + +/** The statement drizzle-kit 0.31 writes to remove it. */ +function dropStatement(item: Unexpressed, schemaName: string): string { + // drizzle-kit leaves the public schema out of the name. + const named = + schemaName === "public" + ? `"${item.name}"` + : `"${schemaName}"."${item.name}"`; + switch (item.kind) { + case "table": + return `DROP TABLE ${named} CASCADE;`; + case "column": + return `ALTER TABLE ${named} DROP COLUMN "${item.member}";`; + case "default": + return `ALTER TABLE ${named} ALTER COLUMN "${item.member}" DROP DEFAULT;`; + case "constraint": + return `ALTER TABLE ${named} DROP CONSTRAINT "${item.member}";`; + case "type": + return `DROP TYPE ${named};`; + } +} + +/** + * With history, the migration is recorded here as applied and runs on deploy. + * A drop of what the branch still has but schema.ts cannot express would be + * recorded falsely here, and would remove it from every deployed database. + */ +function refuseDropsOfUnexpressed( + statements: readonly string[], + unexpressed: readonly Unexpressed[], + schemaName: string, +): void { + const reasons = new Map( + unexpressed.map((item) => [dropStatement(item, schemaName), item.reason]), + ); + const refused = statements + .map((statement) => statement.trim()) + .filter((statement) => reasons.has(statement)); + if (refused.length === 0) return; + throw new IntrospectRefusedError( + `Refusing to write a migration that drops what the branch still has but schema.ts cannot express: ${refused + .map( + (statement) => + `${shown(statement)} (${shown(reasons.get(statement) ?? "")})`, + ) + .join( + " ", + )}. The deployed app would run ${refused.length === 1 ? "it" : "them"} on start.`, + ); +} + +/** + * `appkit db introspect`: read the `.env` branch's app schema from pg_catalog, + * write schema.ts (the previous file goes to schema.ts.bak), write the + * migration that brings history up to the database, and record it as applied. + * Runs only on your dev or session branch, and only when the branch records + * exactly the committed migrations. A failure puts back schema.ts, the backup, + * database.d.ts and the migrations folder. + */ +export async function runIntrospect( + ctx: DbContext, + options: IntrospectOptions, +): Promise { + const schemaFile = path.join(ctx.root, DEFAULT_SCHEMA_FILE); + // Introspect replaces schema.ts, so the current file (missing, empty or broken) is never read. + const schemaName = options.schemaName || DEFAULT_SCHEMA_NAME; + const folder = migrationsFolder(ctx.root); + const migrationOptions = { migrationsFolder: folder, schemaName }; + const hasHistory = (readJournal(folder)?.entries.length ?? 0) > 0; + + const env = await resolveEnvBranch(ctx); + if (!env) throw new Error(`Did not introspect: ${NO_ENV_BRANCH}.`); + if (!takesWrites(env.kind)) { + throw new IntrospectRefusedError( + `Refusing to introspect: ${notYourBranch(env)}. Introspect records the migration it writes as applied, so it runs only on branches \`appkit db init\` created for you. ` + + `Run \`appkit db init\`: it creates your dev branch from the project's default branch, and introspects it when there is no ${DEFAULT_SCHEMA_FILE}.`, + ); + } + const branchId = resourceId(env.branch.name); + + return withBranchPool(ctx, async (pool) => { + // The diff starts from the latest committed snapshot, so the branch must + // record exactly the committed migrations: one it lacks would come out + // reversed, and one the checkout lacks would come out again. + const drift = await readMigrationDrift(pool, migrationOptions); + const refuse = (reason: string, advice: string) => + new IntrospectRefusedError( + `Refusing to introspect ${branchId}: ${reason} in schema "${schemaName}". ${advice}`, + ); + if (drift.unknown > 0) { + throw refuse( + `it records ${plural(drift.unknown, "migration")} this checkout does not have`, + "Pull the latest config/database/migrations, then rerun.", + ); + } + if (drift.stranded.length > 0) { + throw refuse( + `it skipped ${drift.stranded.join(", ")}`, + "Run `appkit db migrate`, which says how to fix it.", + ); + } + // An interrupted run leaves its migration written but not recorded. + const unrecorded = drift.pending.every((tag) => INTROSPECT_TAG.test(tag)) + ? drift.pending + : []; + const notApplied = (them: string) => + refuse( + `it has not applied ${drift.pending.join(", ")}`, + `${them} Run \`appkit db migrate\` first.`, + ); + if (drift.pending.length > unrecorded.length) { + throw notApplied( + `The migration would undo ${drift.pending.length === 1 ? "it" : "them"}.`, + ); + } + + const query = async (text: string, values: readonly unknown[]) => + (await pool.query(text, [...values])).rows as T[]; + const tables = await readCatalog(query, schemaName); + const unread = await readUnreadObjects(query, schemaName); + const rendered = renderSchemaSource(tables, schemaName, { + importSource: options.importSource, + }); + // An app on another schema that forgot --schema-name must not lose its + // schema.ts to an empty one; with history, an empty schema drops every table. + if ( + rendered.tables.length === 0 && + (hasHistory || fs.existsSync(schemaFile)) + ) { + const reasons = rendered.skipped + .filter((skip) => !skip.column) + .map((skip) => `${skip.table}: ${skip.reason}`); + throw new IntrospectRefusedError( + `Schema "${schemaName}" on ${branchId} has no tables${tables.length > 0 ? ` introspect can write (${reasons.join("; ")})` : ""}, so introspect would ${hasHistory ? "write a migration that drops every table" : `replace ${DEFAULT_SCHEMA_FILE} with an empty one`}. Pass --schema-name to read another schema.`, + ); + } + + const backupFile = `${schemaFile}.bak`; + const typesFile = path.join(ctx.root, DATABASE_TYPES_PATH); + const previous = readIfExists(schemaFile); + const previousBackup = readIfExists(backupFile); + const previousTypes = readIfExists(typesFile); + const backsUp = previous !== null && previous !== rendered.source; + const journalBefore = readIfExists(journalFile(folder)); + let migration: GeneratedMigration | null = null; + let recorded: string[] = []; + try { + if (backsUp) fs.writeFileSync(backupFile, previous); + fs.mkdirSync(path.dirname(schemaFile), { recursive: true }); + fs.writeFileSync(schemaFile, rendered.source); + let schema: Awaited>; + try { + schema = await loadDefaultDatabaseSchema(ctx.root, { + showImportError: true, + }); + } catch (error) { + throw new Error( + `The schema.ts introspect generated does not load. This is a bug in \`appkit db introspect\`; please report it with the catalog of schema "${schemaName}".`, + { cause: error }, + ); + } + + migration = await generateMigrationFiles({ + folder, + schema, + tag: hasHistory ? "introspect" : "baseline", + interactive: options.interactive, + ...(hasHistory + ? { + beforeWrite: (statements: readonly string[]) => + refuseDropsOfUnexpressed( + statements, + rendered.unexpressed, + schemaName, + ), + } + : {}), + }); + if (unrecorded.length > 0 && migration) { + throw notApplied( + `Its tables differ from what ${unrecorded.length === 1 ? "it describes" : "they describe"}.`, + ); + } + await writeDatabaseTypes(schema, typesFile); + // The database already holds what the migrations describe; recording is the last step. + recorded = [...unrecorded, ...(migration ? [migration.tag] : [])]; + if (recorded.length > 0) { + await markMigrationsApplied(pool, migrationOptions, recorded); + } + } catch (error) { + writeOrRemove(schemaFile, previous); + if (backsUp) writeOrRemove(backupFile, previousBackup); + writeOrRemove(typesFile, previousTypes); + if (migration) discardMigrationFiles(folder, migration, journalBefore); + if (error instanceof IntrospectRefusedError) throw error; + throw new Error( + `Introspect changed nothing: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } + + const warnings = [...rendered.warnings]; + if (unread.length > 0) { + warnings.push( + `Not read: ${unread.map(shown).join(", ")}. schema.ts and its migrations leave ${unread.length === 1 ? "it" : "them"} out, so a database the migrations create will not have ${unread.length === 1 ? "it" : "them"}.`, + ); + } + const breaking = breakingStatements(migration?.statements ?? []); + if (migration && breaking.length > 0) { + warnings.push( + `${migration.tag} breaks instances still running the previous release, which keep serving while the new one migrates on start: ${breaking.join(" ")}`, + ); + } + return { + branch: env.branch.name, + schemaName, + tables: rendered.tables, + schemaFile, + backupFile: backsUp ? backupFile : null, + skipped: rendered.skipped, + warnings, + approximated: rendered.approximated, + migration, + recorded, + typesFile, + }; + }); +} diff --git a/packages/appkit/src/database/tooling/introspect/render.ts b/packages/appkit/src/database/tooling/introspect/render.ts new file mode 100644 index 000000000..b8378f5e5 --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/render.ts @@ -0,0 +1,864 @@ +import { + DEFAULT_SCHEMA_NAME, + isReservedColumnName, + isReservedTableName, +} from "../../schema-builder/define-schema"; +import type { + IntrospectedColumn, + IntrospectedForeignKey, + IntrospectedTable, +} from "./catalog"; + +/** + * Something introspect left out of schema.ts, with an `// appkit: skipped …` comment. + * Display text: catalog names come with control characters escaped (see shown()). + */ +export interface Skip { + readonly table: string; + readonly column?: string; + readonly reason: string; +} + +/** + * Something the branch has that schema.ts leaves out, by raw catalog name. A + * migration that drops it must not be recorded as applied: the branch keeps it. + */ +export interface Unexpressed { + readonly kind: "table" | "column" | "default" | "constraint" | "type"; + /** The table, or the enum type for "type". */ + readonly name: string; + /** The column for "column" and "default", the constraint for "constraint". */ + readonly member?: string; + readonly reason: string; +} + +interface SchemaSource { + readonly source: string; + /** Tables written to schema.ts. */ + readonly tables: readonly string[]; + readonly skipped: readonly Skip[]; + readonly unexpressed: readonly Unexpressed[]; + /** Omissions that change behavior, e.g. a NOT NULL column the API can no longer fill. */ + readonly warnings: readonly string[]; + /** What schema.ts declares differently from the database, which keeps it as it is. */ + readonly approximated: readonly string[]; +} + +interface RenderOptions { + /** Module the builder is imported from. Default "@databricks/appkit/beta". */ + readonly importSource?: string; +} + +/** Storage as the builder sees it; id/bigid are server-generated integer/bigint keys. */ +type Storage = + | "id" + | "bigid" + | "integer" + | "bigint" + | "text" + | "varchar" + | "boolean" + | "uuid" + | "timestamp" + | "jsonb" + | "enum"; + +interface BaseType { + /** Builder call, e.g. `varchar(120)`. */ + readonly call: string; + /** Import it needs ("" for the context's enum). */ + readonly importName: string; + readonly storage: Storage; + readonly varcharLength?: number; + /** How the call differs from the database type. */ + readonly approximation?: string; +} + +interface ColumnPlan { + readonly column: IntrospectedColumn; + base: BaseType; + /** Set when `fk(() => .)` replaces the base call. */ + fk?: { + readonly table: string; + readonly column: string; + readonly actions: string[]; + }; + readonly isPrimaryKey: boolean; + readonly isUnique: boolean; + defaultCall?: string; + readonly comments: string[]; +} + +interface TablePlan { + readonly table: IntrospectedTable; + readonly columns: ColumnPlan[]; + /** Table-level comments, rendered first. */ + readonly comments: string[]; + /** Columns and skipped-column comments, in catalog order. */ + readonly body: Array; + variable: string; +} + +const JS_RESERVED = new Set( + ( + "break case catch class const continue debugger default delete do else enum export extends false finally for " + + "function if import in instanceof new null return super switch this throw true try typeof var void while with " + + "yield let static implements interface package private protected public await arguments eval undefined" + ).split(" "), +); +const IDENTIFIER = /^[A-Za-z_$][A-Za-z0-9_$]*$/; +const INT32 = { min: -2147483648, max: 2147483647 }; + +/** + * Line separators end a `//` comment, and bidi controls reorder what an editor + * shows. JSON.stringify keeps both. + */ +const UNSAFE_IN_SOURCE = + /[\u2028\u2029\u200e\u200f\u202a-\u202e\u2066-\u2069]/g; +const escaped = (char: string) => + `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`; +const q = (value: string) => + JSON.stringify(value).replace(UNSAFE_IN_SOURCE, escaped); +/** + * Catalog text for a `//` comment or a terminal line. A quoted Postgres name can + * hold a newline, which would end the comment and turn the rest of the name + * into code. + */ +export const shown = (text: string) => + Array.from(text, (char) => { + const code = char.charCodeAt(0); + return code < 0x20 || code === 0x7f ? escaped(char) : char; + }) + .join("") + .replace(UNSAFE_IN_SOURCE, escaped); +/** Postgres keeps the first 63 bytes of a name, cut on a character boundary. */ +function pgName(name: string): string { + let bytes = 0; + let kept = ""; + for (const char of name) { + bytes += Buffer.byteLength(char); + if (bytes > 63) break; + kept += char; + } + return kept; +} +const key = (name: string) => (IDENTIFIER.test(name) ? name : q(name)); +const access = (variable: string, name: string) => + IDENTIFIER.test(name) ? `${variable}.${name}` : `${variable}[${q(name)}]`; +const list = (names: readonly string[]) => names.join(", "); + +function baseType( + column: IntrospectedColumn, + schemaName: string, +): BaseType | string { + const type = column.type; + if (column.enum) { + if (column.enum.schema !== schemaName) { + return `enum type "${column.enum.schema}.${column.enum.name}" lives outside schema "${schemaName}"`; + } + if (column.enum.values.length === 0) { + return `enum type "${column.enum.name}" has no values`; + } + return { + call: `enumType(${q(column.enum.name)}, [${column.enum.values.map(q).join(", ")}])`, + importName: "", + storage: "enum", + }; + } + switch (type) { + case "integer": + return { call: "integer()", importName: "integer", storage: "integer" }; + case "bigint": + return { call: "bigint()", importName: "bigint", storage: "bigint" }; + case "text": + return { call: "text()", importName: "text", storage: "text" }; + case "character varying": + return { + call: "text()", + importName: "text", + storage: "text", + approximation: + "character varying without a length is declared as text()", + }; + case "boolean": + return { call: "boolean()", importName: "boolean", storage: "boolean" }; + case "uuid": + return { call: "uuid()", importName: "uuid", storage: "uuid" }; + case "jsonb": + return { call: "jsonb()", importName: "jsonb", storage: "jsonb" }; + } + const varchar = /^character varying\((\d+)\)$/.exec(type); + if (varchar) { + const length = Number(varchar[1]); + return { + call: `varchar(${length})`, + importName: "varchar", + storage: "varchar", + varcharLength: length, + }; + } + const timestamp = /^timestamp(\(\d\))? (with|without) time zone$/.exec(type); + if (timestamp) { + return { + call: + timestamp[2] === "with" + ? "timestamp({ withTimezone: true })" + : "timestamp()", + importName: "timestamp", + storage: "timestamp", + ...(timestamp[1] + ? { approximation: `${type} is declared without its precision` } + : {}), + }; + } + return `type "${type}" is not supported`; +} + +const isSequenceDefault = (expr: string | null) => + !!expr && /^nextval\(/.test(expr); + +/** A builder default call, or why the database default cannot be expressed. */ +function defaultCall( + expr: string, + base: BaseType, + enumValues: readonly string[] | undefined, +): { call: string } | { skip: string } { + const unsupported = { skip: `default ${expr} is not supported` }; + switch (base.storage) { + case "timestamp": + return /^(now\(\)|CURRENT_TIMESTAMP)$/i.test(expr) + ? { call: ".defaultNow()" } + : unsupported; + case "uuid": + return expr === "gen_random_uuid()" + ? { call: ".defaultRandom()" } + : unsupported; + case "boolean": + return expr === "true" || expr === "false" + ? { call: `.default(${expr})` } + : unsupported; + case "integer": { + const match = /^\(?'?(-?\d+)'?(?:::integer)?\)?$/.exec(expr); + const value = match ? Number(match[1]) : Number.NaN; + return Number.isInteger(value) && value >= INT32.min && value <= INT32.max + ? { call: `.default(${value})` } + : unsupported; + } + case "text": + case "varchar": + case "enum": { + const match = /^'((?:[^']|'')*)'(?:::[\w." ]+)?$/.exec(expr); + if (!match) return unsupported; + const value = (match[1] ?? "").replace(/''/g, "'"); + if ( + base.storage === "varchar" && + value.length > (base.varcharLength ?? 0) + ) { + return unsupported; + } + if (base.storage === "enum" && !enumValues?.includes(value)) { + return unsupported; + } + return { call: `.default(${q(value)})` }; + } + default: + // The builder accepts no literal defaults for bigint and jsonb. + return unsupported; + } +} + +function importLine(names: readonly string[], source: string): string { + const line = `import { ${names.join(", ")} } from ${q(source)};`; + if (line.length <= 80) return line; + return `import {\n${names.map((name) => ` ${name},`).join("\n")}\n} from ${q(source)};`; +} + +function makeSkip( + table: string, + column: string | undefined, + reason: string, +): Skip { + return { + table: shown(table), + ...(column === undefined ? {} : { column: shown(column) }), + reason: shown(reason), + }; +} + +function skipComment(skip: Skip): string { + return `// appkit: skipped ${skip.column ? `column "${skip.column}"` : `table "${skip.table}"`}: ${skip.reason}`; +} + +/** Defaults the builder allows only on uuid / timestamp columns, never on fk(). */ +const TYPED_DEFAULTS = new Set([".defaultRandom()", ".defaultNow()"]); + +/** The storage an fk() column inherits from its target. */ +function mirrored(target: BaseType): string { + if (target.storage === "id") return "integer()"; + if (target.storage === "bigid") return "bigint()"; + return target.call; +} + +/** + * Render pg_catalog tables as a `defineSchema` source the builder accepts. + * What the builder cannot express is left out with an `// appkit: skipped …` + * comment; what it declares differently (a timestamp without its precision, + * say) is listed in `approximated`. + */ +export function renderSchemaSource( + tables: readonly IntrospectedTable[], + schemaName: string, + options: RenderOptions = {}, +): SchemaSource { + const skipped: Skip[] = []; + const unexpressed: Unexpressed[] = []; + const warnings: string[] = []; + const approximated: string[] = []; + /** Constraints whose database name is not the one migrations use: "db (migrations)". */ + const renamed: string[] = []; + const tableComments: string[] = []; + const qualified = (table: string, column?: string) => + `${schemaName}.${table}${column ? `.${column}` : ""}`; + const skipTable = (table: string, reason: string) => { + const skip = makeSkip(table, undefined, reason); + skipped.push(skip); + unexpressed.push({ kind: "table", name: table, reason }); + tableComments.push(skipComment(skip)); + }; + + // 1. Tables the builder can key: one primary-key column, a usable name. + const names = new Set(tables.map((table) => table.name)); + const plans = new Map(); + for (const table of tables) { + if (isReservedTableName(table.name)) { + skipTable(table.name, `the name "${table.name}" is reserved by AppKit`); + } else if ( + table.name.endsWith("Relations") && + names.has(table.name.slice(0, -"Relations".length)) + ) { + skipTable( + table.name, + `the name collides with the relation metadata AppKit generates for "${table.name.slice(0, -"Relations".length)}"`, + ); + } else if (table.primaryKey.length === 0) { + skipTable(table.name, "it has no primary key"); + } else if (table.primaryKey.length > 1) { + skipTable( + table.name, + `composite primary key (${list(table.primaryKey)}) is not supported`, + ); + } else { + plans.set(table.name, { + table, + columns: [], + comments: [], + body: [], + variable: "", + }); + } + } + + // 2. Columns. A table whose primary-key column cannot be expressed is skipped whole. + for (const plan of [...plans.values()]) { + const { table } = plan; + const pk = table.primaryKey[0]; + const tableSkips: Skip[] = []; + const tableWarnings: string[] = []; + const tableApproximated: string[] = []; + const tableRenamed: string[] = []; + const columnReasons = new Map(); + /** Single-column unique constraints: column → constraint name. */ + const singleUnique = new Map( + table.unique + .filter((unique) => unique.columns.length === 1) + .map((unique) => [unique.columns[0] as string, unique.name]), + ); + const tableUnexpressed: Unexpressed[] = []; + for (const unique of table.unique.filter((u) => u.columns.length > 1)) { + const reason = `unique constraint (${list(unique.columns)}) spans several columns, which is not supported`; + const skip = makeSkip(table.name, undefined, reason); + tableSkips.push(skip); + tableUnexpressed.push({ + kind: "constraint", + name: table.name, + member: unique.name, + reason, + }); + plan.comments.push(`// appkit: skipped ${skip.reason}`); + } + + for (const column of table.columns) { + const skipColumn = (reason: string) => { + const skip = makeSkip(table.name, column.name, reason); + tableSkips.push(skip); + tableUnexpressed.push({ + kind: "column", + name: table.name, + member: column.name, + reason, + }); + columnReasons.set(column.name, reason); + plan.body.push(skipComment(skip)); + const generatesValue = + column.identity !== "" || column.generated || !!column.defaultExpr; + if (column.notNull && !generatesValue) { + tableWarnings.push( + `${qualified(table.name, column.name)} is NOT NULL with no default but is not in schema.ts (${reason}). Inserts through the API will fail until it gets a default, becomes nullable, or is dropped.`, + ); + } + }; + + if (isReservedColumnName(column.name)) { + skipColumn(`the name "${column.name}" is reserved by AppKit`); + continue; + } + if (column.generated) { + skipColumn("generated columns are not supported"); + continue; + } + const base = baseType(column, schemaName); + if (typeof base === "string") { + skipColumn(base); + continue; + } + const isPrimaryKey = column.name === pk; + if (column.identity !== "" && !isPrimaryKey) { + skipColumn( + "identity columns other than the primary key are not supported", + ); + continue; + } + + const columnPlan: ColumnPlan = { + column, + base, + isPrimaryKey, + isUnique: singleUnique.has(column.name) && !isPrimaryKey, + comments: [], + }; + if (base.approximation) { + tableApproximated.push( + `${qualified(table.name, column.name)}: ${base.approximation}`, + ); + } + const uniqueName = singleUnique.get(column.name); + const migrationsUnique = pgName(`${table.name}_${column.name}_unique`); + if (columnPlan.isUnique && uniqueName !== migrationsUnique) { + tableRenamed.push(`${uniqueName} (${migrationsUnique})`); + } + const generatesKey = + column.identity !== "" || isSequenceDefault(column.defaultExpr); + if ( + isPrimaryKey && + generatesKey && + (base.storage === "integer" || base.storage === "bigint") + ) { + // A serial or identity key is the builder's server-generated id()/bigid(). + columnPlan.base = + base.storage === "integer" + ? { call: "id()", importName: "id", storage: "id" } + : { call: "bigid()", importName: "bigid", storage: "bigid" }; + if (column.identity !== "d") { + tableApproximated.push( + `${qualified(table.name, column.name)}: ${column.identity === "a" ? "GENERATED ALWAYS AS IDENTITY" : "a serial key"} is declared as ${columnPlan.base.call}, an identity generated BY DEFAULT`, + ); + } + } else if (column.defaultExpr) { + const parsed = defaultCall( + column.defaultExpr, + base, + column.enum?.values, + ); + if ("call" in parsed) { + columnPlan.defaultCall = parsed.call; + } else { + const skip = makeSkip(table.name, column.name, parsed.skip); + tableSkips.push(skip); + tableUnexpressed.push({ + kind: "default", + name: table.name, + member: column.name, + reason: parsed.skip, + }); + columnPlan.comments.push(`// appkit: skipped ${skip.reason}`); + if (column.notNull) { + tableWarnings.push( + `${qualified(table.name, column.name)}: its ${parsed.skip.replace(/ is not supported$/, "")} is not in schema.ts, so the API requires a value for it on insert.`, + ); + } + } + } + plan.columns.push(columnPlan); + plan.body.push(columnPlan); + } + + if (!plan.columns.some((c) => c.isPrimaryKey)) { + plans.delete(table.name); + skipTable( + table.name, + `its primary key column "${pk}" cannot be expressed (${columnReasons.get(pk as string) ?? "unsupported"})`, + ); + continue; + } + skipped.push(...tableSkips); + unexpressed.push(...tableUnexpressed); + warnings.push(...tableWarnings); + if (table.partitioned) { + tableApproximated.push( + `${qualified(table.name)}: a partitioned table is declared as a plain table`, + ); + } + for (const [one, many, names] of [ + [ + "CHECK or EXCLUDE constraint", + "CHECK or EXCLUDE constraints", + table.checks, + ], + ["index", "indexes", table.indexes], + ] as const) { + if (names.length > 0) { + tableApproximated.push( + `${qualified(table.name)}: ${names.length === 1 ? `${one} ${names[0]} is` : `${many} ${list(names)} are`} not declared`, + ); + } + } + approximated.push(...tableApproximated); + renamed.push(...tableRenamed); + } + + // 3. Foreign keys the builder can express; the rest stay plain columns. + const edges = new Map>(); + const reaches = (from: string, to: string): boolean => { + const seen = new Set(); + const stack = [from]; + while (stack.length > 0) { + const node = stack.pop() as string; + if (node === to) return true; + if (seen.has(node)) continue; + seen.add(node); + stack.push(...(edges.get(node) ?? [])); + } + return false; + }; + /** Why the builder cannot express this FK; null when it can. */ + const fkProblem = ( + plan: TablePlan, + foreignKey: IntrospectedForeignKey, + source: ColumnPlan | undefined, + seenTargets: ReadonlySet, + ): string | null => { + if (foreignKey.columns.length !== 1) { + return "foreign keys over several columns are not supported"; + } + if (!source) return "its column is skipped"; + if (foreignKey.schema !== schemaName) { + return `it references schema "${foreignKey.schema}"`; + } + if (foreignKey.table === plan.table.name) { + // The builder takes one with a TableHandle annotation, which introspect does not write. + return "introspect does not write self-references"; + } + const target = plans.get(foreignKey.table); + if (!target) return `table "${foreignKey.table}" is skipped`; + const targetColumn = target.columns.find( + (c) => c.column.name === foreignKey.refColumns[0], + ); + if (!targetColumn) { + return `column ${foreignKey.table}.${foreignKey.refColumns[0]} is skipped`; + } + if (!targetColumn.isPrimaryKey && !targetColumn.isUnique) { + return `${foreignKey.table}.${foreignKey.refColumns[0]} is not a primary key or single-column unique key`; + } + if (seenTargets.has(foreignKey.table)) { + return `a second foreign key to "${foreignKey.table}" would make the relation ambiguous`; + } + if (plan.table.columns.some((c) => c.name === foreignKey.table)) { + return `the relation "${foreignKey.table}" would clash with the column of the same name`; + } + if (target.table.columns.some((c) => c.name === plan.table.name)) { + return `the reverse relation "${plan.table.name}" would clash with column ${foreignKey.table}.${plan.table.name}`; + } + if (reaches(foreignKey.table, plan.table.name)) { + // TypeScript cannot infer a cycle of fk() thunks without TableHandle annotations (TS7022). + return "it would close a cycle of foreign keys"; + } + if (mirrored(targetColumn.base) !== source.base.call) { + return `its type ${source.base.call} differs from ${foreignKey.table}.${foreignKey.refColumns[0]}`; + } + const actions = [foreignKey.onDelete, foreignKey.onUpdate]; + if (actions.includes("set null") && source.column.notNull) { + return "SET NULL on a NOT NULL column"; + } + const fkDefault = + source.defaultCall && !TYPED_DEFAULTS.has(source.defaultCall); + if (actions.includes("set default") && !fkDefault) { + return "SET DEFAULT without a default the builder can express"; + } + return null; + }; + + for (const plan of plans.values()) { + const targets = new Set(); + for (const foreignKey of plan.table.foreignKeys) { + const source = plan.columns.find( + (c) => c.column.name === foreignKey.columns[0], + ); + const problem = fkProblem(plan, foreignKey, source, targets); + if (problem) { + const reason = `foreign key to ${foreignKey.table}(${list(foreignKey.refColumns)}) is not kept: ${problem}`; + // Migrations name it after its column; a schema.ts that declared it gave it that name. + const constraintNames = [foreignKey.name]; + if (foreignKey.columns.length === 1) { + constraintNames.push( + pgName( + `${plan.table.name}_${foreignKey.columns[0]}_${foreignKey.table}_${foreignKey.refColumns[0]}_fk`, + ), + ); + } + for (const member of new Set(constraintNames)) { + unexpressed.push({ + kind: "constraint", + name: plan.table.name, + member, + reason, + }); + } + // A skipped column already carries its own comment. + if (source || foreignKey.columns.length > 1) { + const skip = makeSkip( + plan.table.name, + foreignKey.columns.join(", "), + reason, + ); + skipped.push(skip); + (source?.comments ?? plan.comments).push( + `// appkit: skipped ${skip.reason}`, + ); + } + continue; + } + const sourceColumn = source as ColumnPlan; + const target = plans.get(foreignKey.table) as TablePlan; + const targetColumn = target.columns.find( + (c) => c.column.name === foreignKey.refColumns[0], + ) as ColumnPlan; + sourceColumn.base = { ...targetColumn.base }; + if ( + sourceColumn.defaultCall && + TYPED_DEFAULTS.has(sourceColumn.defaultCall) + ) { + // fk() takes no defaultRandom()/defaultNow(); the key matters more than the default. + const reason = `default ${sourceColumn.column.defaultExpr} is not supported on a foreign key column`; + const skip = makeSkip( + plan.table.name, + sourceColumn.column.name, + reason, + ); + skipped.push(skip); + unexpressed.push({ + kind: "default", + name: plan.table.name, + member: sourceColumn.column.name, + reason, + }); + sourceColumn.comments.push(`// appkit: skipped ${skip.reason}`); + delete sourceColumn.defaultCall; + if (sourceColumn.column.notNull) { + warnings.push( + `${qualified(plan.table.name, sourceColumn.column.name)}: its default ${sourceColumn.column.defaultExpr} is not in schema.ts, so the API requires a value for it on insert.`, + ); + } + } + for (const [flag, what] of [ + [foreignKey.deferrable, "DEFERRABLE"], + [foreignKey.matchFull, "MATCH FULL"], + ] as const) { + if (flag) { + approximated.push( + `${qualified(plan.table.name, sourceColumn.column.name)}: the foreign key is declared without ${what}`, + ); + } + } + const migrationsFk = pgName( + `${plan.table.name}_${sourceColumn.column.name}_${foreignKey.table}_${foreignKey.refColumns[0]}_fk`, + ); + if (foreignKey.name !== migrationsFk) { + renamed.push(`${foreignKey.name} (${migrationsFk})`); + } + sourceColumn.fk = { + table: foreignKey.table, + column: foreignKey.refColumns[0] as string, + actions: [ + ...(foreignKey.onDelete !== "no action" + ? [`.onDelete(${q(foreignKey.onDelete)})`] + : []), + ...(foreignKey.onUpdate !== "no action" + ? [`.onUpdate(${q(foreignKey.onUpdate)})`] + : []), + ], + }; + targets.add(foreignKey.table); + const out = edges.get(plan.table.name) ?? new Set(); + out.add(foreignKey.table); + edges.set(plan.table.name, out); + } + } + + const declaredEnums = new Set( + [...plans.values()].flatMap((plan) => + plan.columns.flatMap((c) => + c.base.storage === "enum" && c.column.enum ? [c.column.enum.name] : [], + ), + ), + ); + const undeclaredEnums = new Set( + tables.flatMap((table) => + table.columns.flatMap((column) => + column.enum?.schema === schemaName && + !declaredEnums.has(column.enum.name) + ? [column.enum.name] + : [], + ), + ), + ); + for (const name of undeclaredEnums) { + unexpressed.push({ + kind: "type", + name, + reason: "only what schema.ts leaves out uses it", + }); + } + + // 4. Declaration order: targets before the tables that reference them. + const ordered: TablePlan[] = []; + const placed = new Set(); + const place = (plan: TablePlan) => { + if (placed.has(plan.table.name)) return; + placed.add(plan.table.name); + for (const target of [...(edges.get(plan.table.name) ?? [])].sort()) { + const targetPlan = plans.get(target); + if (targetPlan) place(targetPlan); + } + ordered.push(plan); + }; + for (const plan of plans.values()) place(plan); + + // 5. Identifiers and imports. + const imports = new Set(["defineSchema"]); + for (const plan of ordered) { + for (const column of plan.columns) { + if (column.fk) imports.add("fk"); + else if (column.base.importName) imports.add(column.base.importName); + } + } + const usesEnum = ordered.some((plan) => + plan.columns.some((c) => !c.fk && c.base.storage === "enum"), + ); + const used = new Set([...imports, "table", "enumType", "schema"]); + for (const plan of ordered) { + let candidate = plan.table.name.replace(/[^A-Za-z0-9_$]/g, "_"); + if (!/^[A-Za-z_$]/.test(candidate)) candidate = `_${candidate}`; + if (used.has(candidate) || JS_RESERVED.has(candidate)) { + candidate = `${candidate}Table`; + } + let variable = candidate; + for (let n = 2; used.has(variable); n++) variable = `${candidate}${n}`; + used.add(variable); + plan.variable = variable; + } + + // 6. Source. + const lines: string[] = [ + `// Generated by \`appkit db introspect\` from schema "${shown(schemaName)}". Edit tables here,`, + "// then run `npx appkit db migrate` on your dev branch.", + importLine( + [...imports].sort(), + options.importSource ?? "@databricks/appkit/beta", + ), + "", + ]; + // Only a schema other than the default is passed to defineSchema(). + const schemaOption = + schemaName === DEFAULT_SCHEMA_NAME + ? undefined + : `{ schemaName: ${q(schemaName)} }`; + if (renamed.length > 0) { + warnings.push( + `Constraint names differ from the ones migrations use: ${list(renamed)}. A later migration that drops or changes one of them fails until it is renamed in the database (ALTER TABLE … RENAME CONSTRAINT).`, + ); + } + const result = (source: string): SchemaSource => ({ + source, + tables: ordered.map((plan) => plan.table.name), + skipped, + unexpressed, + warnings: warnings.map(shown), + approximated: approximated.map(shown), + }); + if (ordered.length === 0) { + lines.push(...tableComments); + lines.push( + `export const schema = defineSchema(() => ({})${schemaOption ? `, ${schemaOption}` : ""});`, + "", + ); + return result(lines.join("\n")); + } + + // The builder's body, indented one level. + const body: string[] = []; + for (const plan of ordered) { + body.push(` const ${plan.variable} = table(${q(plan.table.name)}, {`); + for (const comment of plan.comments) body.push(` ${comment}`); + for (const entry of plan.body) { + if (typeof entry === "string") { + body.push(` ${entry}`); + continue; + } + const column = entry; + for (const comment of column.comments) body.push(` ${comment}`); + const chain: string[] = []; + const head = column.fk + ? `fk(() => ${access((plans.get(column.fk.table) as TablePlan).variable, column.fk.column)})` + : column.base.call; + const serverKey = + !column.fk && + (column.base.storage === "id" || column.base.storage === "bigid"); + if (column.isPrimaryKey && !serverKey) chain.push(".primaryKey()"); + else if (column.column.notNull && !serverKey) chain.push(".notNull()"); + if (column.isUnique) chain.push(".unique()"); + if (column.defaultCall) chain.push(column.defaultCall); + chain.push(...(column.fk?.actions ?? [])); + body.push(` ${key(column.column.name)}: ${head}${chain.join("")},`); + } + body.push(" });", ""); + } + for (const comment of tableComments) body.push(` ${comment}`); + if (tableComments.length > 0) body.push(""); + const entries = ordered.map((plan) => + plan.variable === plan.table.name + ? plan.variable + : `${key(plan.table.name)}: ${plan.variable}`, + ); + body.push(` return { ${entries.join(", ")} };`); + + const builder = `({ table${usesEnum ? ", enum: enumType" : ""} }) =>`; + if (schemaOption) { + lines.push( + "export const schema = defineSchema(", + ` ${builder} {`, + ...body.map((line) => (line ? ` ${line}` : line)), + " },", + ` ${schemaOption},`, + ");", + "", + ); + } else { + lines.push( + `export const schema = defineSchema(${builder} {`, + ...body, + "});", + "", + ); + } + return result(lines.join("\n")); +} diff --git a/packages/appkit/src/database/tooling/introspect/tests/catalog.pg.test.ts b/packages/appkit/src/database/tooling/introspect/tests/catalog.pg.test.ts new file mode 100644 index 000000000..a7334f852 --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/tests/catalog.pg.test.ts @@ -0,0 +1,147 @@ +import { randomUUID } from "node:crypto"; + +import { Pool } from "pg"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; + +import { type CatalogQuery, readCatalog, readUnreadObjects } from "../catalog"; + +// The catalog SQL against a real server. Point it at a scratch database, e.g. +// APPKIT_TEST_PG_URL=postgres://localhost:5432/postgres +const url = process.env.APPKIT_TEST_PG_URL; + +describe.skipIf(!url)("readCatalog against Postgres", () => { + const schema = `appkit_catalog_${randomUUID().slice(0, 8)}`; + let pool: Pool; + const query: CatalogQuery = async ( + text: string, + values: readonly unknown[], + ) => (await pool.query(text, [...values])).rows as T[]; + + beforeAll(async () => { + pool = new Pool({ connectionString: url, max: 1 }); + const s = `"${schema}"`; + await pool.query(` + create schema ${s}; + create table ${s}.__drizzle_migrations (id serial primary key, hash text not null, created_at bigint); + create type ${s}.ticket_status as enum ('open', 'closed'); + create table ${s}.authors ( + id serial primary key, + email varchar(255) not null unique, + created_at timestamptz not null default now() + ); + create table ${s}.posts ( + id integer generated by default as identity primary key, + author_id integer not null references ${s}.authors (id) on delete cascade, + status ${s}.ticket_status not null default 'open', + total integer generated always as (id * 2) stored, + score numeric(10, 2) check (score > 0) + ); + create index posts_status_idx on ${s}.posts (status); + create table ${s}.events (a integer, b integer, primary key (a, b)); + create table ${s}.parts (id integer primary key) partition by range (id); + create table ${s}.parts_1 partition of ${s}.parts for values from (0) to (100); + create table ${s}.refs (id integer primary key, part_id integer references ${s}.parts (id)); + + create view ${s}.active_authors as select * from ${s}.authors; + create materialized view ${s}.mv as select 1 as x; + create sequence ${s}.standalone; + create function ${s}.touch() returns trigger language plpgsql as 'begin return new; end'; + create trigger audit before update on ${s}.authors for each row execute function ${s}.touch(); + alter table ${s}.authors enable row level security; + create policy own on ${s}.authors using (true); + create table ${s}.base (id integer primary key); + create table ${s}.child (extra integer) inherits (${s}.base); + create domain ${s}.positive as integer check (value > 0); + create type ${s}.pair as (a integer, b integer); + create type ${s}.unused_kind as enum ('x'); + `); + }); + afterAll(async () => { + await pool.query(`drop schema if exists "${schema}" cascade`); + await pool.end(); + }); + + test("reads tables, keys, foreign keys, enums, generated and identity columns, and what schema.ts cannot declare", async () => { + const tables = await readCatalog(query, schema); + const byName = Object.fromEntries(tables.map((t) => [t.name, t])); + + expect(tables.map((t) => t.name)).toEqual([ + "authors", + "base", + "child", + "events", + "parts", + "posts", + "refs", + ]); + expect(byName.authors).toMatchObject({ + primaryKey: ["id"], + unique: [{ name: "authors_email_key", columns: ["email"] }], + columns: [ + { + name: "id", + type: "integer", + notNull: true, + identity: "", + defaultExpr: `nextval('${schema}.authors_id_seq'::regclass)`, + }, + { name: "email", type: "character varying(255)", notNull: true }, + { + name: "created_at", + type: "timestamp with time zone", + defaultExpr: "now()", + }, + ], + }); + expect(byName.posts).toMatchObject({ + primaryKey: ["id"], + foreignKeys: [ + { + name: "posts_author_id_fkey", + columns: ["author_id"], + schema, + table: "authors", + refColumns: ["id"], + onDelete: "cascade", + onUpdate: "no action", + }, + ], + checks: ["posts_score_check"], + indexes: ["posts_status_idx"], + }); + const posts = Object.fromEntries( + (byName.posts?.columns ?? []).map((c) => [c.name, c]), + ); + expect(posts.id?.identity).toBe("d"); + expect(posts.total?.generated).toBe(true); + expect(posts.status?.enum).toEqual({ + name: "ticket_status", + schema, + values: ["open", "closed"], + }); + expect(byName.events?.primaryKey).toEqual(["a", "b"]); + expect(byName.parts?.partitioned).toBe(true); + // One foreign key, not one more per partition of its target. + expect(byName.refs?.foreignKeys.map((fk) => fk.table)).toEqual(["parts"]); + }); + + test("names what it does not read, leaving out sequences that belong to a column", async () => { + const unread = await readUnreadObjects(query, schema); + + expect([...unread].sort()).toEqual( + [ + "domain positive", + "function touch()", + "inheritance of child from base", + "materialized view mv", + "policy own on authors", + "row-level security on authors", + "sequence standalone", + "trigger audit on authors", + "type pair", + "type unused_kind", + "view active_authors", + ].sort(), + ); + }); +}); diff --git a/packages/appkit/src/database/tooling/introspect/tests/catalog.test.ts b/packages/appkit/src/database/tooling/introspect/tests/catalog.test.ts new file mode 100644 index 000000000..ca7f64170 --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/tests/catalog.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, test } from "vitest"; + +import { buildTables, readCatalog } from "../catalog"; +import { catalogResponder, catalogRows } from "./fixture"; + +describe("readCatalog", () => { + test("maps catalog rows to tables, columns, keys, foreign keys and enums", async () => { + const { respond } = catalogResponder(); + const [authors, posts] = await readCatalog(respond, "app"); + + expect(authors).toEqual({ + name: "authors", + partitioned: false, + primaryKey: ["id"], + unique: [{ name: "authors_email_key", columns: ["email"] }], + foreignKeys: [], + checks: [], + indexes: [], + columns: [ + { + name: "id", + type: "integer", + notNull: true, + identity: "", + generated: false, + defaultExpr: "nextval('app.authors_id_seq'::regclass)", + }, + { + name: "email", + type: "character varying(255)", + notNull: true, + identity: "", + generated: false, + defaultExpr: null, + }, + { + name: "created_at", + type: "timestamp with time zone", + notNull: true, + identity: "", + generated: false, + defaultExpr: "now()", + }, + ], + }); + expect(posts?.primaryKey).toEqual(["id"]); + expect(posts?.foreignKeys).toEqual([ + { + name: "posts_author_id_fkey", + columns: ["author_id"], + schema: "app", + table: "authors", + refColumns: ["id"], + onDelete: "cascade", + onUpdate: "no action", + deferrable: false, + matchFull: false, + }, + ]); + expect(posts?.indexes).toEqual(["posts_status_idx"]); + expect(posts?.columns.find((c) => c.name === "id")?.identity).toBe("d"); + expect(posts?.columns.find((c) => c.name === "total")?.generated).toBe( + true, + ); + expect(posts?.columns.find((c) => c.name === "status")?.enum).toEqual({ + name: "ticket_status", + schema: "app", + values: ["open", "closed"], + }); + }); + + test("partitioned tables, CHECK and EXCLUDE constraints, indexes, DEFERRABLE and MATCH FULL are kept for the report", () => { + const row = (kind: "c" | "x" | "f", name: string, extra = {}) => ({ + ...catalogRows.constraints[2], + kind, + name, + ...extra, + }); + const [table] = buildTables( + [{ name: "posts", partitioned: true }], + [], + [ + row("c", "posts_score_check"), + row("x", "posts_no_overlap"), + row("f", "posts_author_id_fkey", { + deferrable: true, + match_type: "f", + }), + ] as never, + [ + { table_name: "posts", name: "posts_status_idx" }, + { table_name: "authors", name: "authors_name_idx" }, + ], + ); + expect(table).toMatchObject({ + partitioned: true, + checks: ["posts_score_check", "posts_no_overlap"], + indexes: ["posts_status_idx"], + foreignKeys: [{ deferrable: true, matchFull: true }], + }); + }); + + test("stored and virtual (PG18) generated columns are both generated", () => { + const [table] = buildTables( + [{ name: "t", partitioned: false }], + ["s", "v", ""].map((generated, i) => ({ + ...catalogRows.columns[0], + table_name: "t", + name: `c${i}`, + generated, + })) as never, + [], + ); + expect(table?.columns.map((c) => c.generated)).toEqual([true, true, false]); + }); + + test("referential action letters map to builder actions", () => { + const fkRow = (on_delete: string, on_update: string) => ({ + ...catalogRows.constraints[2], + on_delete, + on_update, + }); + const [table] = buildTables( + [{ name: "posts", partitioned: false }], + [], + ["r", "c", "n", "d", "a"].map((letter) => fkRow(letter, letter)) as never, + ); + expect(table?.foreignKeys.map((fk) => [fk.onDelete, fk.onUpdate])).toEqual([ + ["restrict", "restrict"], + ["cascade", "cascade"], + ["set null", "set null"], + ["set default", "set default"], + ["no action", "no action"], + ]); + }); +}); diff --git a/packages/appkit/src/database/tooling/introspect/tests/fixture.ts b/packages/appkit/src/database/tooling/introspect/tests/fixture.ts new file mode 100644 index 000000000..2f8b2f9ff --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/tests/fixture.ts @@ -0,0 +1,293 @@ +import path from "node:path"; + +import type { + CatalogQuery, + ColumnRow, + ConstraintRow, + IndexRow, + IntrospectedColumn, + IntrospectedForeignKey, + IntrospectedTable, + IntrospectedUnique, + TableRow, +} from "../catalog"; + +/** The builder by absolute path, so rendered sources load from a temp root. */ +export const BUILDER = path.resolve( + import.meta.dirname, + "../../../schema-builder/index.ts", +); + +export const col = ( + name: string, + type: string, + extra: Partial = {}, +): IntrospectedColumn => ({ + name, + type, + notNull: false, + identity: "", + generated: false, + defaultExpr: null, + ...extra, +}); + +/** A catalog table; what a test does not set is empty. */ +export const tableOf = ( + table: Pick & + Partial, +): IntrospectedTable => ({ + partitioned: false, + primaryKey: [], + unique: [], + foreignKeys: [], + checks: [], + indexes: [], + ...table, +}); + +export const foreignKey = ( + key: Omit & + Partial, +): IntrospectedForeignKey => ({ deferrable: false, matchFull: false, ...key }); + +/** A unique constraint with the name Postgres gives it. */ +export const uniqueKey = ( + table: string, + ...columns: string[] +): IntrospectedUnique => ({ + name: `${table}_${columns.join("_")}_key`, + columns, +}); + +const status = { + name: "ticket_status", + schema: "app", + values: ["open", "closed"], +}; + +/** Supported types, defaults, FKs and enums, plus one of each skip. Listed out of dependency order. */ +export const richTables: IntrospectedTable[] = [ + tableOf({ + name: "posts", + primaryKey: ["id"], + unique: [], + foreignKeys: [ + foreignKey({ + name: "posts_author_id_fkey", + columns: ["author_id"], + schema: "app", + table: "authors", + refColumns: ["id"], + onDelete: "cascade", + onUpdate: "no action", + }), + foreignKey({ + name: "posts_editor_id_fkey", + columns: ["editor_id"], + schema: "app", + table: "authors", + refColumns: ["id"], + onDelete: "set null", + onUpdate: "no action", + }), + ], + columns: [ + col("id", "uuid", { notNull: true, defaultExpr: "gen_random_uuid()" }), + col("author_id", "integer", { notNull: true }), + col("editor_id", "integer"), + col("status", "app.ticket_status", { + notNull: true, + defaultExpr: "'open'::app.ticket_status", + enum: status, + }), + col("body", "jsonb", { defaultExpr: "'{}'::jsonb" }), + col("views", "bigint", { notNull: true, defaultExpr: "0" }), + col("my col", "character varying"), + ], + }), + tableOf({ + name: "authors", + primaryKey: ["id"], + unique: [uniqueKey("authors", "email")], + foreignKeys: [], + columns: [ + col("id", "integer", { + notNull: true, + defaultExpr: "nextval('app.authors_id_seq'::regclass)", + }), + col("email", "character varying(255)", { notNull: true }), + col("bio", "text", { defaultExpr: "'it''s me'::text" }), + col("active", "boolean", { notNull: true, defaultExpr: "true" }), + col("score", "numeric(10,2)", { notNull: true }), + col("rank", "integer", { defaultExpr: "'-1'::integer" }), + col("created_at", "timestamp with time zone", { + notNull: true, + defaultExpr: "now()", + }), + col("seen_at", "timestamp(3) without time zone"), + ], + }), + tableOf({ + name: "events", + primaryKey: ["a", "b"], + unique: [], + foreignKeys: [], + columns: [ + col("a", "integer", { notNull: true }), + col("b", "integer", { notNull: true }), + ], + }), + tableOf({ + name: "logs", + primaryKey: [], + unique: [], + foreignKeys: [], + columns: [col("line", "text")], + }), + tableOf({ + name: "default", + primaryKey: ["id"], + unique: [], + foreignKeys: [], + columns: [col("id", "bigint", { notNull: true, identity: "d" })], + }), +]; + +export const columnRow = ( + table_name: string, + name: string, + type: string, + extra: Partial = {}, +): ColumnRow => ({ + table_name, + name, + type, + not_null: false, + identity: "", + generated: "", + default_expr: null, + type_kind: "b", + type_schema: "pg_catalog", + type_name: type, + enum_values: null, + ...extra, +}); + +/** What the three pg_catalog queries return for a small hand-made schema. */ +export const catalogRows = { + tables: [ + { name: "authors", partitioned: false }, + { name: "posts", partitioned: false }, + ] satisfies TableRow[], + columns: [ + columnRow("authors", "id", "integer", { + not_null: true, + default_expr: "nextval('app.authors_id_seq'::regclass)", + }), + columnRow("authors", "email", "character varying(255)", { + not_null: true, + }), + columnRow("authors", "created_at", "timestamp with time zone", { + not_null: true, + default_expr: "now()", + }), + columnRow("posts", "id", "integer", { not_null: true, identity: "d" }), + columnRow("posts", "author_id", "integer", { not_null: true }), + columnRow("posts", "status", "app.ticket_status", { + not_null: true, + default_expr: "'open'::app.ticket_status", + type_kind: "e", + type_schema: "app", + type_name: "ticket_status", + enum_values: ["open", "closed"], + }), + columnRow("posts", "score", "numeric(10,2)", { not_null: true }), + columnRow("posts", "total", "integer", { + generated: "s", + default_expr: "1", + }), + ] satisfies ColumnRow[], + constraints: [ + { + table_name: "authors", + name: "authors_email_key", + kind: "u", + columns: ["email"], + ref_schema: null, + ref_table: null, + ref_columns: [], + on_delete: " ", + on_update: " ", + }, + { + table_name: "authors", + name: "authors_pkey", + kind: "p", + columns: ["id"], + ref_schema: null, + ref_table: null, + ref_columns: [], + on_delete: " ", + on_update: " ", + }, + { + table_name: "posts", + name: "posts_author_id_fkey", + kind: "f", + columns: ["author_id"], + ref_schema: "app", + ref_table: "authors", + ref_columns: ["id"], + on_delete: "c", + on_update: "a", + }, + { + table_name: "posts", + name: "posts_pkey", + kind: "p", + columns: ["id"], + ref_schema: null, + ref_table: null, + ref_columns: [], + on_delete: " ", + on_update: " ", + }, + ] satisfies ConstraintRow[], + indexes: [ + { table_name: "posts", name: "posts_status_idx" }, + ] satisfies IndexRow[], +}; + +/** A CatalogQuery answering the catalog queries from rows, and the unread-objects query from labels. */ +export function catalogResponder( + rows: { + readonly tables: readonly TableRow[]; + readonly columns: readonly ColumnRow[]; + readonly constraints: readonly ConstraintRow[]; + readonly indexes: readonly IndexRow[]; + } = catalogRows, + unread: readonly string[] = [], +) { + const calls: Array<{ text: string; values: readonly unknown[] }> = []; + const respond: CatalogQuery = async ( + text: string, + values: readonly unknown[], + ) => { + calls.push({ text, values }); + // The unread and index queries mention what the others match on, so they go first. + const result = text.includes("pg_catalog.pg_depend") + ? unread.map((label) => ({ label })) + : text.includes("pg_catalog.pg_index") + ? rows.indexes + : text.includes("pg_catalog.pg_constraint") + ? rows.constraints + : text.includes("format_type") + ? rows.columns + : text.includes("select c.relname::text as name") + ? rows.tables + : []; + return result as unknown as T[]; + }; + return { calls, respond }; +} diff --git a/packages/appkit/src/database/tooling/introspect/tests/golden/rich.schema.ts.txt b/packages/appkit/src/database/tooling/introspect/tests/golden/rich.schema.ts.txt new file mode 100644 index 000000000..2fe3e54d5 --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/tests/golden/rich.schema.ts.txt @@ -0,0 +1,51 @@ +// Generated by `appkit db introspect` from schema "app". Edit tables here, +// then run `npx appkit db migrate` on your dev branch. +import { + bigid, + bigint, + boolean, + defineSchema, + fk, + id, + integer, + jsonb, + text, + timestamp, + uuid, + varchar, +} from "@databricks/appkit/beta"; + +export const schema = defineSchema(({ table, enum: enumType }) => { + const authors = table("authors", { + id: id(), + email: varchar(255).notNull().unique(), + bio: text().default("it's me"), + active: boolean().notNull().default(true), + // appkit: skipped column "score": type "numeric(10,2)" is not supported + rank: integer().default(-1), + created_at: timestamp({ withTimezone: true }).notNull().defaultNow(), + seen_at: timestamp(), + }); + + const posts = table("posts", { + id: uuid().primaryKey().defaultRandom(), + author_id: fk(() => authors.id).notNull().onDelete("cascade"), + // appkit: skipped foreign key to authors(id) is not kept: a second foreign key to "authors" would make the relation ambiguous + editor_id: integer(), + status: enumType("ticket_status", ["open", "closed"]).notNull().default("open"), + // appkit: skipped default '{}'::jsonb is not supported + body: jsonb(), + // appkit: skipped default 0 is not supported + views: bigint().notNull(), + "my col": text(), + }); + + const defaultTable = table("default", { + id: bigid(), + }); + + // appkit: skipped table "events": composite primary key (a, b) is not supported + // appkit: skipped table "logs": it has no primary key + + return { authors, posts, default: defaultTable }; +}); diff --git a/packages/appkit/src/database/tooling/introspect/tests/introspect.test.ts b/packages/appkit/src/database/tooling/introspect/tests/introspect.test.ts new file mode 100644 index 000000000..dc3fe1bb6 --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/tests/introspect.test.ts @@ -0,0 +1,741 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +const { markMigrationsApplied, resolveEnvBranch, withBranchPool, writeTypes } = + vi.hoisted(() => ({ + markMigrationsApplied: vi.fn(), + resolveEnvBranch: vi.fn(), + withBranchPool: vi.fn(), + writeTypes: vi.fn(), + })); + +vi.mock("../../context", async (importOriginal) => ({ + ...(await importOriginal()), + withBranchPool, +})); +vi.mock("../../guard", async (importOriginal) => ({ + ...(await importOriginal()), + resolveEnvBranch, +})); +vi.mock("../../../migrations/apply", async (importOriginal) => ({ + ...(await importOriginal()), + markMigrationsApplied, +})); +vi.mock("../../../../type-generator/database", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("../../../../type-generator/database") + >()), + writeDatabaseTypes: writeTypes, +})); + +import { loadDefaultDatabaseSchema } from "../../../../plugins/database/load-schema"; +import { readJournal } from "../../../migrations/paths"; +import { generateMigrationFiles } from "../../migrations"; +import { cleanupTempApps, fakeContext, tempApp } from "../../tests/fakes"; +import type { ColumnRow, ConstraintRow, TableRow } from "../catalog"; +import { IntrospectRefusedError, runIntrospect } from "../index"; +import { BUILDER, catalogResponder, columnRow } from "./fixture"; + +const { writeDatabaseTypes } = await vi.importActual< + typeof import("../../../../type-generator/database") +>("../../../../type-generator/database"); + +const SCHEMA = "config/database/schema.ts"; +const MIGRATIONS = "config/database/migrations"; +const TYPES = "shared/appkit-types/database.d.ts"; +const DEV = "projects/p/branches/dev-alice-3f9a2c"; +const PRODUCTION = "projects/p/branches/production"; + +const previousSchema = `import { defineSchema, id, text } from ${JSON.stringify(BUILDER)}; +export const schema = defineSchema(({ table }) => ({ + authors: table("authors", { id: id(), email: text() }), +})); +`; + +const onDevBranch = { branch: { name: DEV }, kind: "dev" }; +const onProduction = { + branch: { name: PRODUCTION, status: { default: true } }, + kind: "default", + reason: "it is the project's default branch", +}; + +let catalog: ReturnType; +/** `when` of each migration the branch recorded; empty: no journal table. */ +let recorded: number[]; +const lastWhen = (root: string) => + readJournal(path.join(root, MIGRATIONS))?.entries.at(-1)?.when ?? 0; +const pool = { + query: vi.fn(async (text: string, values: unknown[]) => { + // readMigrationDrift: whether the journal table exists, then its rows. + if (text.includes("c.relname = $2")) { + return { rows: recorded.length > 0 ? [{ "?column?": 1 }] : [] }; + } + if (text.includes("select hash, created_at")) { + return { + rows: recorded.map((when) => ({ hash: "h", created_at: String(when) })), + }; + } + return { rows: await catalog.respond(text, values) }; + }), + end: vi.fn(async () => undefined), +}; + +beforeEach(() => { + catalog = catalogResponder(); + recorded = []; + resolveEnvBranch.mockReset().mockResolvedValue(onDevBranch); + withBranchPool + .mockReset() + .mockImplementation( + async (_ctx: unknown, fn: (p: typeof pool) => Promise) => { + try { + return await fn(pool); + } finally { + await pool.end(); + } + }, + ); + markMigrationsApplied + .mockReset() + .mockImplementation( + async ( + _pool: unknown, + options: { migrationsFolder: string }, + tags: readonly string[], + ) => { + const entries = readJournal(options.migrationsFolder)?.entries ?? []; + for (const tag of tags) { + recorded.push(entries.find((entry) => entry.tag === tag)?.when ?? 0); + } + }, + ); + writeTypes.mockReset().mockImplementation(writeDatabaseTypes); + pool.end.mockClear(); +}); +afterEach(cleanupTempApps); + +const authorsPkey: ConstraintRow = { + table_name: "authors", + name: "authors_pkey", + kind: "p", + columns: ["id"], + ref_schema: null, + ref_table: null, + ref_columns: [], + on_delete: " ", + on_update: " ", +}; + +/** A catalog with authors(id) plus the given authors columns, and any extra tables. */ +const authorsOnly = ( + columns: ColumnRow[] = [], + extra: { tables: TableRow[]; columns: ColumnRow[] } = { + tables: [], + columns: [], + }, +) => + catalogResponder({ + tables: [{ name: "authors", partitioned: false }, ...extra.tables], + columns: [ + columnRow("authors", "id", "integer", { not_null: true, identity: "d" }), + ...columns, + ...extra.columns, + ], + constraints: [authorsPkey], + indexes: [], + }); + +/** Commit a migration of the current schema.ts; `applied`: the branch has recorded it. */ +async function commitMigration(root: string, applied = true): Promise { + await generateMigrationFiles({ + folder: path.join(root, MIGRATIONS), + schema: await loadDefaultDatabaseSchema(root), + interactive: false, + }); + if (applied) recorded.push(lastWhen(root)); +} + +/** Every file under config/, with its contents. */ +function tree(root: string): Record { + const files: Record = {}; + const walk = (dir: string) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else files[path.relative(root, full)] = fs.readFileSync(full, "utf-8"); + } + }; + if (fs.existsSync(path.join(root, "config"))) walk(path.join(root, "config")); + return files; +} + +const ctxFor = (root: string) => fakeContext({ root }); +const introspect = (root: string) => + runIntrospect(ctxFor(root), { interactive: false, importSource: BUILDER }); +/** The error a run fails with; a run that succeeds fails the test. */ +const rejection = (run: Promise) => + run.then( + () => { + throw new Error("expected introspect to fail"); + }, + (caught: unknown) => caught as Error, + ); +const refusal = (root: string) => rejection(introspect(root)); + +describe("runIntrospect", () => { + test("without history: schema.ts (+ .bak), 0000_baseline recorded as applied, database.d.ts", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + + const report = await introspect(root); + + expect(catalog.calls.every((call) => call.values[0] === "app")).toBe(true); + expect(fs.readFileSync(path.join(root, `${SCHEMA}.bak`), "utf-8")).toBe( + previousSchema, + ); + const written = fs.readFileSync(path.join(root, SCHEMA), "utf-8"); + expect(written).toContain('const authors = table("authors", {'); + expect(written).toContain( + 'author_id: fk(() => authors.id).notNull().onDelete("cascade"),', + ); + expect(report).toMatchObject({ + branch: DEV, + schemaName: "app", + tables: ["authors", "posts"], + schemaFile: path.join(root, SCHEMA), + backupFile: path.join(root, `${SCHEMA}.bak`), + recorded: ["0000_baseline"], + migration: { tag: "0000_baseline" }, + }); + expect(report.skipped.map((skip) => skip.column)).toEqual([ + "score", + "total", + ]); + expect(report.warnings).toEqual([ + expect.stringMatching(/^app\.posts\.score is NOT NULL with no default/), + expect.stringMatching( + /^Constraint names differ from the ones migrations use: authors_email_key \(authors_email_unique\), posts_author_id_fkey \(posts_author_id_authors_id_fk\)\./, + ), + ]); + expect(report.approximated).toEqual([ + "app.authors.id: a serial key is declared as id(), an identity generated BY DEFAULT", + "app.posts: index posts_status_idx is not declared", + ]); + expect( + fs.existsSync(path.join(root, MIGRATIONS, "0000_baseline.sql")), + ).toBe(true); + expect(markMigrationsApplied).toHaveBeenCalledWith( + pool, + { migrationsFolder: path.join(root, MIGRATIONS), schemaName: "app" }, + ["0000_baseline"], + ); + expect(fs.readFileSync(report.typesFile, "utf-8")).toContain("authors"); + expect(pool.end).toHaveBeenCalledOnce(); + // The written schema is what the plugin loads. + await expect(loadDefaultDatabaseSchema(root)).resolves.toMatchObject({ + $schemaName: "app", + }); + }); + + test.each([ + ["without history", false], + ["with history", true], + ])( + "on a branch that is not yours (%s): refuses before connecting and points to appkit db init", + async (_label, history) => { + resolveEnvBranch.mockResolvedValue(onProduction); + const root = tempApp({ [SCHEMA]: previousSchema }); + if (history) await commitMigration(root); + const before = tree(root); + + const error = await refusal(root); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + "Refusing to introspect: production is not your dev or session branch (it is the project's default branch). Introspect records the migration it writes as applied, so it runs only on branches `appkit db init` created for you. " + + "Run `appkit db init`: it creates your dev branch from the project's default branch, and introspects it when there is no config/database/schema.ts.", + ); + expect(withBranchPool).not.toHaveBeenCalled(); + expect(tree(root)).toEqual(before); + }, + ); + + test("with history: writes and records the next NNNN_introspect", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await commitMigration(root); + + const report = await introspect(root); + + expect(report.migration?.tag).toBe("0001_introspect"); + expect(report.migration?.statements.join("\n")).toContain( + 'CREATE TABLE "app"."posts"', + ); + expect(report.recorded).toEqual(["0001_introspect"]); + expect(markMigrationsApplied).toHaveBeenCalledWith( + pool, + expect.anything(), + ["0001_introspect"], + ); + }); + + test("a second run with no catalog change writes no migration and records nothing", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await introspect(root); + markMigrationsApplied.mockClear(); + + const report = await introspect(root); + + expect(report.migration).toBeNull(); + expect(report.recorded).toEqual([]); + expect(markMigrationsApplied).not.toHaveBeenCalled(); + }); + + test("a migration an interrupted run wrote but did not record is recorded when the branch matches it", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await introspect(root); + // The run stopped after writing 0000_baseline, before recording it. + recorded = []; + markMigrationsApplied.mockClear(); + + const report = await introspect(root); + + expect(report.migration).toBeNull(); + expect(report.recorded).toEqual(["0000_baseline"]); + expect(markMigrationsApplied).toHaveBeenCalledWith( + pool, + expect.anything(), + ["0000_baseline"], + ); + }); + + test("an unrecorded introspect migration the branch no longer matches is refused, and nothing changes", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await introspect(root); + recorded = []; + catalog = authorsOnly(); + markMigrationsApplied.mockClear(); + const before = tree(root); + + const error = await refusal(root); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + 'Refusing to introspect dev-alice-3f9a2c: it has not applied 0000_baseline in schema "app". Its tables differ from what it describes. Run `appkit db migrate` first.', + ); + expect(tree(root)).toEqual(before); + expect(markMigrationsApplied).not.toHaveBeenCalled(); + }); + + test("without schema.ts it reads the app schema and creates the file", async () => { + const root = tempApp(); + + const report = await introspect(root); + + expect(catalog.calls.every((call) => call.values[0] === "app")).toBe(true); + expect(report).toMatchObject({ + schemaName: "app", + backupFile: null, + tables: ["authors", "posts"], + migration: { tag: "0000_baseline" }, + recorded: ["0000_baseline"], + }); + expect(fs.readFileSync(path.join(root, SCHEMA), "utf-8")).toMatch( + /from schema "app"\.[^]*export const schema = defineSchema\(\(\{ table[^}]*\}\) => \{/, + ); + expect(fs.existsSync(path.join(root, `${SCHEMA}.bak`))).toBe(false); + }); + + test("--schema-name picks another schema", async () => { + const root = tempApp(); + + const report = await runIntrospect(ctxFor(root), { + interactive: false, + importSource: BUILDER, + schemaName: "sales", + }); + + expect(catalog.calls.every((call) => call.values[0] === "sales")).toBe( + true, + ); + expect(report.schemaName).toBe("sales"); + }); + + test.each([ + ["a syntax error", "export const schema = ;\n"], + ["a single line", "// todo\n"], + [ + "a missing import", + `import { defineSchema } from ${JSON.stringify(BUILDER)};\nexport const schema = defineSchema(({ table }) => ({ x: table("x", { id: id() }) }));\n`, + ], + ])( + "a schema.ts with %s is never read: it goes to .bak and is replaced from the app schema", + async (_label, broken) => { + const root = tempApp({ [SCHEMA]: broken }); + + const report = await introspect(root); + + expect(catalog.calls.every((call) => call.values[0] === "app")).toBe( + true, + ); + expect(report).toMatchObject({ + schemaName: "app", + tables: ["authors", "posts"], + backupFile: path.join(root, `${SCHEMA}.bak`), + migration: { tag: "0000_baseline" }, + }); + expect(fs.readFileSync(path.join(root, `${SCHEMA}.bak`), "utf-8")).toBe( + broken, + ); + expect(fs.readFileSync(path.join(root, SCHEMA), "utf-8")).toMatch( + /from schema "app"\.[^]*export const schema = defineSchema\(\(\{ table[^}]*\}\) => \{/, + ); + }, + ); + + test("--schema-name public is written explicitly, so the app never looks for the tables in the default app schema", async () => { + const root = tempApp(); + + const report = await runIntrospect(ctxFor(root), { + interactive: false, + importSource: BUILDER, + schemaName: "public", + }); + + expect(report.schemaName).toBe("public"); + expect(fs.readFileSync(path.join(root, SCHEMA), "utf-8")).toContain( + '{ schemaName: "public" }', + ); + expect((await loadDefaultDatabaseSchema(root)).$schemaName).toBe("public"); + }); + + test("the schema schema.ts declares does not change which schema is read", async () => { + const root = tempApp({ + [SCHEMA]: previousSchema.replace("}));", '}), { schemaName: "sales" });'), + }); + + const report = await introspect(root); + + expect(catalog.calls.every((call) => call.values[0] === "app")).toBe(true); + expect(report.schemaName).toBe("app"); + }); + + test("an empty schema next to an existing schema.ts refuses before writing anything", async () => { + catalog = catalogResponder({ + tables: [], + columns: [], + constraints: [], + indexes: [], + }); + const root = tempApp({ [SCHEMA]: previousSchema }); + + const error = await refusal(root); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + 'Schema "app" on dev-alice-3f9a2c has no tables, so introspect would replace config/database/schema.ts with an empty one. Pass --schema-name to read another schema.', + ); + expect(fs.readFileSync(path.join(root, SCHEMA), "utf-8")).toBe( + previousSchema, + ); + expect(fs.existsSync(path.join(root, `${SCHEMA}.bak`))).toBe(false); + expect(fs.existsSync(path.join(root, MIGRATIONS))).toBe(false); + }); + + test("an empty schema without schema.ts still writes an empty one", async () => { + catalog = catalogResponder({ + tables: [], + columns: [], + constraints: [], + indexes: [], + }); + const root = tempApp(); + + const report = await introspect(root); + + expect(report.tables).toEqual([]); + expect(report.migration).toBeNull(); + expect(report.recorded).toEqual([]); + expect(fs.readFileSync(path.join(root, SCHEMA), "utf-8")).toContain( + "export const schema = defineSchema(() => ({}));", + ); + }); + + test("without a configured branch it fails before writing", async () => { + resolveEnvBranch.mockResolvedValue(null); + const root = tempApp({ [SCHEMA]: previousSchema }); + + await expect(introspect(root)).rejects.toThrow( + "Did not introspect: no branch is configured in .env (PGHOST, PGDATABASE and LAKEBASE_ENDPOINT); run `appkit db init`.", + ); + expect(fs.readFileSync(path.join(root, SCHEMA), "utf-8")).toBe( + previousSchema, + ); + }); + + test("with history, a branch that has not applied every committed migration refuses before reading the catalog", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await commitMigration(root); + // A teammate's migration, pulled but not applied on this branch yet. + fs.writeFileSync( + path.join(root, SCHEMA), + previousSchema.replace( + "email: text() }),", + 'email: text() }),\n tags: table("tags", { id: id() }),', + ), + ); + await commitMigration(root, false); + const before = tree(root); + + const error = await refusal(root); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + 'Refusing to introspect dev-alice-3f9a2c: it has not applied 0001_migration in schema "app". The migration would undo it. Run `appkit db migrate` first.', + ); + expect(catalog.calls).toEqual([]); + expect(tree(root)).toEqual(before); + }); + + test.each([ + ["without history", false], + ["with history", true], + ])( + "a branch that records a migration the checkout lacks (%s) is refused before reading the catalog", + async (_label, history) => { + const root = tempApp({ [SCHEMA]: previousSchema }); + if (history) await commitMigration(root); + recorded.push(Date.now() + 60_000); + const before = tree(root); + + const error = await refusal(root); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + 'Refusing to introspect dev-alice-3f9a2c: it records 1 migration this checkout does not have in schema "app". Pull the latest config/database/migrations, then rerun.', + ); + expect(catalog.calls).toEqual([]); + expect(tree(root)).toEqual(before); + }, + ); + + test("a branch that skipped a committed migration is refused", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await commitMigration(root, false); + fs.writeFileSync( + path.join(root, SCHEMA), + previousSchema.replace("email: text() }", "email: text(), bio: text() }"), + ); + await commitMigration(root); + + const error = await refusal(root); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + 'Refusing to introspect dev-alice-3f9a2c: it skipped 0000_migration in schema "app". Run `appkit db migrate`, which says how to fix it.', + ); + }); + + test("with history and no schema.ts, an empty schema refuses instead of dropping every table", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await commitMigration(root); + fs.rmSync(path.join(root, SCHEMA)); + catalog = catalogResponder({ + tables: [], + columns: [], + constraints: [], + indexes: [], + }); + const before = tree(root); + + const error = await refusal(root); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + 'Schema "app" on dev-alice-3f9a2c has no tables, so introspect would write a migration that drops every table. Pass --schema-name to read another schema.', + ); + expect(tree(root)).toEqual(before); + }); + + test.each([ + [ + "a column it cannot declare", + previousSchema, + authorsOnly([columnRow("authors", "email", "numeric")]), + 'ALTER TABLE "app"."authors" DROP COLUMN "email"; (type "numeric" is not supported)', + ], + [ + "a default it cannot declare", + previousSchema.replace("email: text()", 'email: text().default("x")'), + authorsOnly([ + columnRow("authors", "email", "text", { default_expr: "lower('X')" }), + ]), + `ALTER TABLE "app"."authors" ALTER COLUMN "email" DROP DEFAULT; (default lower('X') is not supported)`, + ], + [ + "a table it cannot key", + previousSchema.replace( + "email: text() }),", + 'email: text() }),\n tags: table("tags", { id: id() }),', + ), + authorsOnly([columnRow("authors", "email", "text")], { + tables: [{ name: "tags", partitioned: false }], + columns: [columnRow("tags", "id", "integer", { not_null: true })], + }), + 'DROP TABLE "app"."tags" CASCADE; (it has no primary key)', + ], + [ + "a column in the public schema", + previousSchema.replace("}));", '}), { schemaName: "public" });'), + authorsOnly([columnRow("authors", "email", "numeric")]), + 'ALTER TABLE "authors" DROP COLUMN "email"; (type "numeric" is not supported)', + ], + ])( + "with history, a migration that drops %s the branch still has is refused and nothing changes", + async (_label, committed, branch, refused) => { + const root = tempApp({ [SCHEMA]: committed }); + await commitMigration(root); + catalog = branch; + const before = tree(root); + + const error = await rejection( + runIntrospect(ctxFor(root), { + interactive: false, + importSource: BUILDER, + schemaName: (await loadDefaultDatabaseSchema(root)).$schemaName, + }), + ); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + `Refusing to write a migration that drops what the branch still has but schema.ts cannot express: ${refused}. The deployed app would run it on start.`, + ); + expect(tree(root)).toEqual(before); + expect(markMigrationsApplied).not.toHaveBeenCalled(); + }, + ); + + test("with history, a skip elsewhere does not block dropping what the branch no longer has, which is recorded and warned about", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await commitMigration(root); + catalog = authorsOnly([], { + tables: [{ name: "events", partitioned: false }], + columns: [columnRow("events", "line", "text")], + }); + + const report = await introspect(root); + + expect(report.skipped).toEqual([ + { table: "events", reason: "it has no primary key" }, + ]); + expect(report.migration?.tag).toBe("0001_introspect"); + expect(report.migration?.statements).toEqual([ + 'ALTER TABLE "app"."authors" DROP COLUMN "email";', + ]); + expect(report.recorded).toEqual(["0001_introspect"]); + expect(report.warnings).toEqual([ + '0001_introspect breaks instances still running the previous release, which keep serving while the new one migrates on start: ALTER TABLE "app"."authors" DROP COLUMN "email";', + ]); + }); + + test("objects introspect does not read are named in a warning", async () => { + catalog = catalogResponder(undefined, [ + "trigger audit on authors", + "view active_authors", + ]); + const root = tempApp(); + + const report = await introspect(root); + + expect(report.warnings.at(-1)).toBe( + "Not read: trigger audit on authors, view active_authors. schema.ts and its migrations leave them out, so a database the migrations create will not have them.", + ); + }); + + test.each([ + [ + "recording", + () => + markMigrationsApplied.mockRejectedValueOnce(new Error("lock timeout")), + "lock timeout", + ], + [ + "writing database.d.ts", + () => writeTypes.mockRejectedValueOnce(new Error("disk full")), + "disk full", + ], + ])( + "when %s fails, schema.ts, its backup, database.d.ts and the migrations are put back, so a rerun records", + async (_label, failOnce, message) => { + const root = tempApp({ + [SCHEMA]: previousSchema, + [`${SCHEMA}.bak`]: "// an older backup\n", + }); + const before = tree(root); + failOnce(); + + const error = await refusal(root); + + expect(error.message).toBe(`Introspect changed nothing: ${message}`); + expect(tree(root)).toEqual(before); + expect(fs.existsSync(path.join(root, MIGRATIONS))).toBe(false); + expect(fs.existsSync(path.join(root, TYPES))).toBe(false); + expect(recorded).toEqual([]); + + const report = await introspect(root); + expect(report).toMatchObject({ + migration: { tag: "0000_baseline" }, + recorded: ["0000_baseline"], + }); + }, + ); + + test("a second run with the same catalog keeps the original .bak", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + await introspect(root); + + const second = await introspect(root); + + expect(second.backupFile).toBeNull(); + expect(fs.readFileSync(path.join(root, `${SCHEMA}.bak`), "utf-8")).toBe( + previousSchema, + ); + }); + + test("when every table is skipped, an existing schema.ts is kept", async () => { + catalog = catalogResponder({ + tables: [{ name: "logs", partitioned: false }], + columns: [columnRow("logs", "line", "text")], + constraints: [], + indexes: [], + }); + const root = tempApp({ [SCHEMA]: previousSchema }); + const before = tree(root); + + const error = await refusal(root); + + expect(error).toBeInstanceOf(IntrospectRefusedError); + expect(error.message).toBe( + 'Schema "app" on dev-alice-3f9a2c has no tables introspect can write (logs: it has no primary key), so introspect would replace config/database/schema.ts with an empty one. Pass --schema-name to read another schema.', + ); + expect(tree(root)).toEqual(before); + }); + + test("a generated schema.ts that does not load is reported as a bug, and schema.ts is put back", async () => { + const root = tempApp({ [SCHEMA]: previousSchema }); + const before = tree(root); + + const error = await rejection( + runIntrospect(ctxFor(root), { + interactive: false, + importSource: path.join(root, "missing-builder.ts"), + }), + ); + + expect(error.message).toBe( + 'Introspect changed nothing: The schema.ts introspect generated does not load. This is a bug in `appkit db introspect`; please report it with the catalog of schema "app".', + ); + expect(tree(root)).toEqual(before); + expect(markMigrationsApplied).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/appkit/src/database/tooling/introspect/tests/render.test.ts b/packages/appkit/src/database/tooling/introspect/tests/render.test.ts new file mode 100644 index 000000000..1211a18a9 --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/tests/render.test.ts @@ -0,0 +1,623 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { describe, expect, test } from "vitest"; + +import type { IntrospectedForeignKey, IntrospectedTable } from "../catalog"; +import { renderSchemaSource } from "../render"; +import { col, richTables, tableOf, uniqueKey } from "./fixture"; + +const golden = (name: string) => + fs.readFileSync(path.join(import.meta.dirname, "golden", name), "utf-8"); + +const fk = ( + column: string, + table: string, + extra: Partial = {}, +): IntrospectedForeignKey => ({ + name: `${column}_fkey`, + columns: [column], + schema: "app", + table, + refColumns: ["id"], + onDelete: "no action", + onUpdate: "no action", + deferrable: false, + matchFull: false, + ...extra, +}); + +/** A table with an integer identity key plus the given columns and FKs. */ +const keyed = ( + name: string, + columns: IntrospectedTable["columns"] = [], + foreignKeys: IntrospectedForeignKey[] = [], + unique: string[][] = [], +): IntrospectedTable => + tableOf({ + name, + primaryKey: ["id"], + unique: unique.map((columns) => uniqueKey(name, ...columns)), + foreignKeys, + columns: [ + col("id", "integer", { notNull: true, identity: "d" }), + ...columns, + ], + }); + +const body = (source: string) => + source.slice(source.indexOf("export const schema")); + +describe("renderSchemaSource", () => { + test("golden: supported types, defaults, FKs and enums, with skip comments", () => { + const out = renderSchemaSource(richTables, "app"); + + expect(out.source).toBe(golden("rich.schema.ts.txt")); + expect(out.skipped).toEqual( + expect.arrayContaining([ + { + table: "events", + reason: "composite primary key (a, b) is not supported", + }, + { table: "logs", reason: "it has no primary key" }, + { + table: "authors", + column: "score", + reason: 'type "numeric(10,2)" is not supported', + }, + { + table: "posts", + column: "body", + reason: "default '{}'::jsonb is not supported", + }, + { + table: "posts", + column: "views", + reason: "default 0 is not supported", + }, + { + table: "posts", + column: "editor_id", + reason: + 'foreign key to authors(id) is not kept: a second foreign key to "authors" would make the relation ambiguous', + }, + ]), + ); + expect(out.skipped).toHaveLength(6); + }); + + test("a skipped NOT NULL column without a default is a loud warning", () => { + const { warnings } = renderSchemaSource(richTables, "app"); + expect(warnings).toContain( + 'app.authors.score is NOT NULL with no default but is not in schema.ts (type "numeric(10,2)" is not supported). Inserts through the API will fail until it gets a default, becomes nullable, or is dropped.', + ); + expect(warnings).toContain( + "app.posts.views: its default 0 is not in schema.ts, so the API requires a value for it on insert.", + ); + expect( + warnings.filter((warning) => !warning.startsWith("Constraint names")), + ).toHaveLength(2); + }); + + test.each([ + ["app", "export const schema = defineSchema(() => ({}));\n"], + [ + "public", + 'export const schema = defineSchema(() => ({}), { schemaName: "public" });\n', + ], + ])( + "no tables in %s renders an empty schema, naming the schema only when it is not app", + (schemaName, expected) => { + expect(body(renderSchemaSource([], schemaName).source)).toBe(expected); + }, + ); + + test("tables in a schema other than app pass it to defineSchema", () => { + expect(body(renderSchemaSource([keyed("notes")], "public").source)).toBe( + [ + "export const schema = defineSchema(", + " ({ table }) => {", + ' const notes = table("notes", {', + " id: id(),", + " });", + "", + " return { notes };", + " },", + ' { schemaName: "public" },', + ");", + "", + ].join("\n"), + ); + }); + + test("declares FK targets first, whatever the catalog order", () => { + const { source } = renderSchemaSource( + [ + keyed("c", [col("b_id", "integer")], [fk("b_id", "b")]), + keyed("b", [col("a_id", "integer")], [fk("a_id", "a")]), + keyed("a"), + ], + "app", + ); + const order = ["a", "b", "c"].map((name) => + source.indexOf(`const ${name} = table`), + ); + expect(order).toEqual([...order].sort((x, y) => x - y)); + expect(source).toContain("return { a, b, c };"); + }); + + test("FKs the builder cannot express stay plain columns, each with its reason", () => { + const cases: Array<[IntrospectedTable[], string]> = [ + [ + [keyed("t", [col("parent_id", "integer")], [fk("parent_id", "t")])], + "introspect does not write self-references", + ], + [ + [ + keyed("a", [col("b_id", "integer")], [fk("b_id", "b")]), + keyed("b", [col("a_id", "integer")], [fk("a_id", "a")]), + ], + "it would close a cycle of foreign keys", + ], + [ + [ + keyed( + "t", + [col("x", "integer"), col("y", "integer")], + [{ ...fk("x", "u"), columns: ["x", "y"], refColumns: ["id", "k"] }], + ), + keyed("u"), + ], + "foreign keys over several columns are not supported", + ], + [ + [ + keyed( + "t", + [col("u_id", "integer")], + [fk("u_id", "u", { schema: "public" })], + ), + keyed("u"), + ], + 'it references schema "public"', + ], + [ + [ + keyed( + "t", + [col("u_code", "integer")], + [fk("u_code", "u", { refColumns: ["code"] })], + ), + keyed("u", [col("code", "integer")]), + ], + "u.code is not a primary key or single-column unique key", + ], + [ + [keyed("t", [col("u_id", "bigint")], [fk("u_id", "u")]), keyed("u")], + "its type bigint() differs from u.id", + ], + [ + [ + keyed( + "t", + [col("u_id", "integer", { notNull: true })], + [fk("u_id", "u", { onDelete: "set null" })], + ), + keyed("u"), + ], + "SET NULL on a NOT NULL column", + ], + [ + [ + keyed( + "t", + [col("u_id", "integer")], + [fk("u_id", "u", { onDelete: "set default" })], + ), + keyed("u"), + ], + "SET DEFAULT without a default", + ], + [ + [ + keyed( + "t", + [col("u_id", "integer"), col("u", "text")], + [fk("u_id", "u")], + ), + keyed("u"), + ], + 'the relation "u" would clash with the column of the same name', + ], + [ + [ + keyed("t", [col("u_id", "integer")], [fk("u_id", "u")]), + keyed("u", [col("t", "text")]), + ], + 'the reverse relation "t" would clash with column u.t', + ], + [ + [ + keyed("t", [col("u_id", "integer")], [fk("u_id", "u")]), + { ...keyed("u"), primaryKey: ["id", "x"] }, + ], + 'table "u" is skipped', + ], + ]; + for (const [tables, reason] of cases) { + const out = renderSchemaSource(tables, "app"); + expect(out.source, reason).toContain(reason); + const fkSkips = out.skipped.filter((skip) => + skip.reason.startsWith("foreign key"), + ); + expect(fkSkips, reason).toHaveLength(1); + // Exactly one fk() survives only in the cycle case (the first edge). + expect(out.source.includes("fk(() =>"), reason).toBe( + reason.includes("cycle"), + ); + } + }); + + test("an FK keeps NOT NULL, unique, a default and its actions", () => { + const { source } = renderSchemaSource( + [ + keyed( + "t", + [col("u_id", "integer", { notNull: true, defaultExpr: "1" })], + [fk("u_id", "u", { onDelete: "set default", onUpdate: "cascade" })], + [["u_id"]], + ), + keyed("u"), + ], + "app", + ); + expect(source).toContain( + 'u_id: fk(() => u.id).notNull().unique().default(1).onDelete("set default").onUpdate("cascade"),', + ); + }); + + test("a table whose primary key cannot be expressed is skipped whole", () => { + const out = renderSchemaSource( + [ + tableOf({ + name: "t", + primaryKey: ["code"], + unique: [], + foreignKeys: [], + columns: [ + col("code", "smallint", { notNull: true }), + col("x", "numeric", { notNull: true }), + ], + }), + ], + "app", + ); + expect(out.skipped).toEqual([ + { + table: "t", + reason: + 'its primary key column "code" cannot be expressed (type "smallint" is not supported)', + }, + ]); + // The table is gone, so its columns produce no warnings. + expect(out.warnings).toEqual([]); + }); + + test("names AppKit reserves are skipped; unusual names are quoted or renamed", () => { + const out = renderSchemaSource( + [ + keyed("sql"), + keyed("my-table", [ + col("$meta", "text"), + col("and", "text"), + col("select", "text"), + ]), + keyed("users"), + keyed("usersRelations"), + ], + "app", + ); + expect( + out.skipped.map((skip) => [skip.table, skip.column ?? null, skip.reason]), + ).toEqual([ + ["sql", null, 'the name "sql" is reserved by AppKit'], + [ + "usersRelations", + null, + 'the name collides with the relation metadata AppKit generates for "users"', + ], + ["my-table", "$meta", 'the name "$meta" is reserved by AppKit'], + ["my-table", "and", 'the name "and" is reserved by AppKit'], + ]); + expect(out.source).toContain('const my_table = table("my-table", {'); + expect(out.source).toContain(" select: text(),"); + expect(out.source).toContain('return { "my-table": my_table, users };'); + }); + + test("unsupported defaults are dropped with a comment; nextval on a key becomes id()", () => { + const { source, skipped } = renderSchemaSource( + [ + tableOf({ + name: "t", + primaryKey: ["id"], + unique: [], + foreignKeys: [], + columns: [ + col("id", "bigint", { + notNull: true, + defaultExpr: "nextval('t_id_seq'::regclass)", + }), + col("code", "character varying(2)", { + defaultExpr: "'toolong'::character varying", + }), + col("at", "timestamp without time zone", { + defaultExpr: "'2020-01-01 00:00:00'::timestamp without time zone", + }), + col("n", "integer", { defaultExpr: "nextval('n_seq'::regclass)" }), + ], + }), + ], + "app", + ); + expect(source).toContain(" id: bigid(),"); + expect(skipped.map((skip) => skip.column)).toEqual(["code", "at", "n"]); + expect(source).toContain(" code: varchar(2),"); + }); + + test("a catalog name with a line break stays inside its comment", () => { + const evil = "x\nrequire('node:child_process').execSync('touch /tmp/p')//"; + const { source, skipped, warnings } = renderSchemaSource( + [ + keyed("t", [ + col(evil, "numeric", { notNull: true }), + col("y\u2028z", "numeric"), + ]), + ], + "app", + ); + for (const line of source.split("\n")) { + expect(line.trimStart().startsWith("require(")).toBe(false); + } + expect(source).not.toMatch(/[\u2028\u2029]/); + expect(source).toContain( + `// appkit: skipped column "x\\u000arequire('node:child_process').execSync('touch /tmp/p')//": type "numeric" is not supported`, + ); + expect(source).toContain( + '// appkit: skipped column "y\\u2028z": type "numeric" is not supported', + ); + expect(skipped.map((skip) => skip.column)).toEqual([ + "x\\u000arequire('node:child_process').execSync('touch /tmp/p')//", + "y\\u2028z", + ]); + expect(warnings[0]).toMatch(/^app\.t\.x\\u000arequire/); + }); + + test("the schema name in the header comment is escaped too", () => { + const { source } = renderSchemaSource([keyed("t")], "a\nb"); + expect(source.split("\n")[0]).toBe( + '// Generated by `appkit db introspect` from schema "a\\u000ab". Edit tables here,', + ); + expect(source).toContain('{ schemaName: "a\\nb" }'); + }); + + test("returns the tables it wrote", () => { + expect(renderSchemaSource(richTables, "app").tables).toEqual([ + "authors", + "posts", + "default", + ]); + }); + + test("what schema.ts declares differently from the database is listed as approximated", () => { + const { approximated } = renderSchemaSource( + [ + tableOf({ + name: "a", + partitioned: true, + primaryKey: ["id"], + checks: ["a_n_check"], + indexes: ["a_n_idx", "a_m_idx"], + columns: [ + col("id", "integer", { notNull: true, identity: "a" }), + col("n", "character varying"), + col("at", "timestamp(3) with time zone"), + ], + }), + keyed( + "b", + [col("a_id", "integer")], + [ + fk("a_id", "a", { + name: "b_a_id_a_id_fk", + deferrable: true, + matchFull: true, + }), + ], + ), + ], + "app", + ); + expect(approximated).toEqual([ + "app.a.id: GENERATED ALWAYS AS IDENTITY is declared as id(), an identity generated BY DEFAULT", + "app.a.n: character varying without a length is declared as text()", + "app.a.at: timestamp(3) with time zone is declared without its precision", + "app.a: a partitioned table is declared as a plain table", + "app.a: CHECK or EXCLUDE constraint a_n_check is not declared", + "app.a: indexes a_n_idx, a_m_idx are not declared", + "app.b.a_id: the foreign key is declared without DEFERRABLE", + "app.b.a_id: the foreign key is declared without MATCH FULL", + ]); + }); + + test("constraint names that differ from the ones migrations use are one warning", () => { + const { warnings } = renderSchemaSource(richTables, "app"); + expect(warnings).toContain( + "Constraint names differ from the ones migrations use: authors_email_key (authors_email_unique), posts_author_id_fkey (posts_author_id_authors_id_fk). A later migration that drops or changes one of them fails until it is renamed in the database (ALTER TABLE … RENAME CONSTRAINT).", + ); + }); + + test("constraint names migrations would use raise no warning", () => { + const { warnings } = renderSchemaSource( + [ + tableOf({ + ...keyed("u", [col("email", "text")]), + unique: [{ name: "u_email_unique", columns: ["email"] }], + }), + keyed( + "t", + [col("u_id", "integer")], + [fk("u_id", "u", { name: "t_u_id_u_id_fk" })], + ), + ], + "app", + ); + expect(warnings).toEqual([]); + }); + + test("migrations' constraint names are cut at 63 bytes, as Postgres cuts them", () => { + const column = "ç".repeat(40); + const { warnings } = renderSchemaSource( + [ + tableOf({ + ...keyed("t", [col(column, "text")]), + // "t_" plus 30 two-byte characters is 62 bytes; one more would pass 63. + unique: [{ name: `t_${"ç".repeat(30)}`, columns: [column] }], + }), + ], + "app", + ); + expect(warnings).toEqual([]); + }); + + test("bidi and line-separator characters are escaped in string literals", () => { + const name = "t\u202e\u2066\u2028x"; + const { source } = renderSchemaSource([keyed(name)], "app"); + expect(source).not.toMatch(/[\u2028\u202e\u2066]/); + expect(source).toContain('table("t\\u202e\\u2066\\u2028x", {'); + }); + + test("what schema.ts leaves out is listed by raw name, for the drop check", () => { + const status = { name: "status", schema: "app", values: ["a"] }; + const { unexpressed } = renderSchemaSource( + [ + keyed( + "t", + [ + col("n", "numeric"), + col("s", "app.status", { generated: true, enum: status }), + col("d", "text", { defaultExpr: "lower('X')" }), + col("x", "integer"), + col("y", "integer"), + col("p", "integer"), + ], + [fk("p", "t")], + [["x", "y"]], + ), + tableOf({ name: "logs", columns: [col("line", "text")] }), + ], + "app", + ); + expect(unexpressed).toEqual([ + { kind: "table", name: "logs", reason: "it has no primary key" }, + { + kind: "constraint", + name: "t", + member: "t_x_y_key", + reason: + "unique constraint (x, y) spans several columns, which is not supported", + }, + { + kind: "column", + name: "t", + member: "n", + reason: 'type "numeric" is not supported', + }, + { + kind: "column", + name: "t", + member: "s", + reason: "generated columns are not supported", + }, + { + kind: "default", + name: "t", + member: "d", + reason: "default lower('X') is not supported", + }, + ...["p_fkey", "t_p_t_id_fk"].map((member) => ({ + kind: "constraint", + name: "t", + member, + reason: + "foreign key to t(id) is not kept: introspect does not write self-references", + })), + { + kind: "type", + name: "status", + reason: "only what schema.ts leaves out uses it", + }, + ]); + }); + + test("a uuid FK keeps the key and skips gen_random_uuid(), which fk() does not take", () => { + const { source, skipped, warnings } = renderSchemaSource( + [ + tableOf({ + name: "u", + primaryKey: ["id"], + columns: [ + col("id", "uuid", { + notNull: true, + defaultExpr: "gen_random_uuid()", + }), + ], + }), + keyed( + "t", + [ + col("u_id", "uuid", { + notNull: true, + defaultExpr: "gen_random_uuid()", + }), + ], + [fk("u_id", "u", { name: "t_u_id_u_id_fk" })], + ), + ], + "app", + ); + expect(source).toContain(" u_id: fk(() => u.id).notNull(),"); + expect(source).not.toContain("fk(() => u.id).notNull().defaultRandom()"); + expect(skipped).toEqual([ + { + table: "t", + column: "u_id", + reason: + "default gen_random_uuid() is not supported on a foreign key column", + }, + ]); + expect(warnings).toEqual([ + "app.t.u_id: its default gen_random_uuid() is not in schema.ts, so the API requires a value for it on insert.", + ]); + }); + + test("SET DEFAULT needs a default fk() can take", () => { + const { skipped } = renderSchemaSource( + [ + tableOf({ + name: "u", + primaryKey: ["id"], + columns: [col("id", "uuid", { notNull: true })], + }), + keyed( + "t", + [col("u_id", "uuid", { defaultExpr: "gen_random_uuid()" })], + [fk("u_id", "u", { onDelete: "set default" })], + ), + ], + "app", + ); + expect(skipped.map((skip) => skip.reason)).toEqual([ + "foreign key to u(id) is not kept: SET DEFAULT without a default the builder can express", + ]); + }); +}); diff --git a/packages/appkit/src/database/tooling/introspect/tests/roundtrip.test.ts b/packages/appkit/src/database/tooling/introspect/tests/roundtrip.test.ts new file mode 100644 index 000000000..f3d52d3c2 --- /dev/null +++ b/packages/appkit/src/database/tooling/introspect/tests/roundtrip.test.ts @@ -0,0 +1,368 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { getTableConfig, type PgTable } from "drizzle-orm/pg-core"; +import { afterEach, describe, expect, test } from "vitest"; + +import { loadDefaultDatabaseSchema } from "../../../../plugins/database/load-schema"; +import { drizzleKitImports } from "../../drizzle-exports"; +import { generateMigrationFiles } from "../../migrations"; +import type { IntrospectedTable } from "../catalog"; +import { renderSchemaSource } from "../render"; +import { + BUILDER, + col, + foreignKey, + richTables, + tableOf, + uniqueKey, +} from "./fixture"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) + fs.rmSync(root, { recursive: true, force: true }); +}); + +/** Write the rendered source as an app's schema.ts and load it the way the plugin does. */ +async function load(tables: IntrospectedTable[]) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "appkit-introspect-")); + roots.push(root); + const file = path.join(root, "config/database/schema.ts"); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync( + file, + renderSchemaSource(tables, "app", { importSource: BUILDER }).source, + ); + return { root, schema: await loadDefaultDatabaseSchema(root) }; +} + +interface KitColumn { + name: string; + type: string; + notNull: boolean; + primaryKey: boolean; + default?: unknown; +} +interface KitTable { + name: string; + schema: string; + columns: Record; + foreignKeys: Record< + string, + { + tableTo: string; + columnsFrom: string[]; + columnsTo: string[]; + onDelete?: string; + } + >; +} + +describe("introspect round trip", () => { + test("the rendered source loads through loadDefaultDatabaseSchema, and drizzle-kit sees the same tables", async () => { + const { schema } = await load(richTables); + expect(schema.$schemaName).toBe("app"); + expect(Object.keys(schema.$tables).sort()).toEqual([ + "authors", + "default", + "posts", + ]); + + const { generateDrizzleJson } = await import("drizzle-kit/api"); + const snapshot = generateDrizzleJson(drizzleKitImports(schema), undefined, [ + "app", + ]) as unknown as { + tables: Record; + enums: Record; + }; + const columns = (table: string) => + Object.fromEntries( + Object.values(snapshot.tables[`app.${table}`]?.columns ?? {}).map( + (c) => [ + c.name, + `${c.type}${c.notNull ? " not null" : ""}${c.primaryKey ? " pk" : ""}`, + ], + ), + ); + + expect(columns("authors")).toEqual({ + id: "integer not null pk", + email: "varchar(255) not null", + bio: "text", + active: "boolean not null", + rank: "integer", + created_at: "timestamp with time zone not null", + seen_at: "timestamp", + }); + expect(columns("posts")).toEqual({ + id: "uuid not null pk", + author_id: "integer not null", + editor_id: "integer", + status: "ticket_status not null", + body: "jsonb", + views: "bigint not null", + "my col": "text", + }); + expect(columns("default")).toEqual({ id: "bigint not null pk" }); + expect( + Object.values(snapshot.tables["app.posts"]?.foreignKeys ?? {}), + ).toEqual([ + expect.objectContaining({ + tableTo: "authors", + columnsFrom: ["author_id"], + columnsTo: ["id"], + onDelete: "cascade", + }), + ]); + expect(snapshot.enums["app.ticket_status"]).toMatchObject({ + values: ["open", "closed"], + }); + }); + + test("the baseline is generated from the empty snapshot: it creates every rendered table", async () => { + const { root, schema } = await load(richTables); + const folder = path.join(root, "config/database/migrations"); + + const migration = await generateMigrationFiles({ + folder, + schema, + tag: "baseline", + interactive: false, + }); + + expect(migration?.tag).toBe("0000_baseline"); + const sql = migration?.statements.join("\n") ?? ""; + for (const table of ["authors", "posts", "default"]) { + expect(sql).toContain(`CREATE TABLE "app"."${table}"`); + } + expect(sql).toContain( + `CREATE TYPE "app"."ticket_status" AS ENUM('open', 'closed')`, + ); + expect(sql).not.toContain("CREATE SCHEMA"); + }); + + test("every skip path still yields a loadable schema", async () => { + const tables: IntrospectedTable[] = [ + tableOf({ + name: "a", + primaryKey: ["id"], + unique: [uniqueKey("a", "x", "y")], + foreignKeys: [ + foreignKey({ + name: "self", + columns: ["parent_id"], + schema: "app", + table: "a", + refColumns: ["id"], + onDelete: "no action", + onUpdate: "no action", + }), + foreignKey({ + name: "to_b", + columns: ["b_id"], + schema: "app", + table: "b", + refColumns: ["id"], + onDelete: "set null", + onUpdate: "no action", + }), + ], + columns: [ + col("id", "integer", { notNull: true, identity: "a" }), + col("parent_id", "integer"), + col("b_id", "integer", { notNull: true }), + col("x", "integer"), + col("y", "integer"), + col("g", "integer", { generated: true, defaultExpr: "(x + y)" }), + col("seq", "integer", { identity: "d" }), + col("tags", "text[]"), + ], + }), + tableOf({ + name: "b", + primaryKey: ["id"], + unique: [], + foreignKeys: [ + foreignKey({ + name: "to_a", + columns: ["a_id"], + schema: "app", + table: "a", + refColumns: ["id"], + onDelete: "no action", + onUpdate: "no action", + }), + ], + columns: [ + col("id", "integer", { notNull: true, identity: "d" }), + col("a_id", "integer"), + ], + }), + ]; + const { schema } = await load(tables); + expect(Object.keys(schema.$tables).sort()).toEqual(["a", "b"]); + expect(Object.keys(schema.$tables.a?.$columns ?? {})).toEqual([ + "id", + "parent_id", + "b_id", + "x", + "y", + ]); + }); + + test("a uuid FK with gen_random_uuid() and names with line breaks still load, with no extra code", async () => { + const marker = "__introspect_injected__"; + const { schema } = await load([ + tableOf({ + name: "u", + primaryKey: ["id"], + columns: [ + col("id", "uuid", { + notNull: true, + defaultExpr: "gen_random_uuid()", + }), + ], + }), + tableOf({ + name: "t", + primaryKey: ["id"], + foreignKeys: [ + foreignKey({ + name: "t_u_id_fkey", + columns: ["u_id"], + schema: "app", + table: "u", + refColumns: ["id"], + onDelete: "no action", + onUpdate: "no action", + }), + ], + columns: [ + col("id", "integer", { notNull: true, identity: "d" }), + col("u_id", "uuid", { + notNull: true, + defaultExpr: "gen_random_uuid()", + }), + col(`x\nglobalThis.${marker} = true; //`, "numeric"), + col(`y\u2028globalThis.${marker} = true; //`, "numeric"), + ], + }), + ]); + + expect((globalThis as Record)[marker]).toBeUndefined(); + expect(Object.keys(schema.$tables).sort()).toEqual(["t", "u"]); + expect(Object.keys(schema.$tables.t?.$columns ?? {})).toEqual([ + "id", + "u_id", + ]); + }); + + test("hostile names reach every place render writes them and load back unchanged, with no code run", async () => { + const marker = "__introspect_hostile__"; + const payload = `globalThis.${marker} = true; //`; + const [ + lf, + cr, + ls, + ps, + quote, + apostrophe, + backslash, + backtick, + bidi, + close, + ] = [ + `lf\n${payload}`, + `cr\r${payload}`, + `ls\u2028${payload}`, + `ps\u2029${payload}`, + `q"]; ${payload}`, + `s'); ${payload}`, + `b\\"; ${payload}`, + `t\`\${globalThis.${marker} = true}\``, + "bidi\u202e\u2066x", + `*/ ${payload}`, + ]; + const texts = [lf, cr, ls, ps, apostrophe, backslash, backtick, bidi]; + const literal = (value: string) => `'${value.replace(/'/g, "''")}'::text`; + const kind = { name: close, schema: "app", values: [quote, cr, bidi] }; + const level = { name: "level", schema: "app", values: [ls, bidi] }; + const tables = [ + tableOf({ + name: "target", + primaryKey: [quote], + columns: [col(quote, "integer", { notNull: true, identity: "d" })], + }), + tableOf({ + name: "source", + primaryKey: ["id"], + foreignKeys: [ + foreignKey({ + name: apostrophe, + columns: [backslash], + schema: "app", + table: "target", + refColumns: [quote], + onDelete: "no action", + onUpdate: "no action", + }), + ], + columns: [ + col("id", "integer", { notNull: true, identity: "d" }), + col(backslash, "integer", { notNull: true }), + ...texts.map((name) => + col(`c ${name}`, "text", { defaultExpr: literal(name) }), + ), + col("kind", `app."${close}"`, { enum: kind }), + col("level", "app.level", { + defaultExpr: `'${bidi}'::app.level`, + enum: level, + }), + ...[lf, cr, ls, ps].map((name) => col(`skipped ${name}`, "numeric")), + ], + }), + tableOf({ + name: close, + primaryKey: ["id"], + columns: [col("id", "integer", { notNull: true, identity: "d" })], + }), + tableOf({ name: ps, columns: [col(cr, "text")] }), + ]; + + const { schema } = await load(tables); + + expect((globalThis as Record)[marker]).toBeUndefined(); + expect(Object.keys(schema.$tables).sort()).toEqual( + ["source", "target", close].sort(), + ); + const config = (name: string) => + getTableConfig(schema.$engine[name] as unknown as PgTable); + expect(config(close).name).toBe(close); + expect(config("target").columns.map((c) => c.name)).toEqual([quote]); + const source = config("source"); + expect(source.columns.map((c) => c.name)).toEqual([ + "id", + backslash, + ...texts.map((name) => `c ${name}`), + "kind", + "level", + ]); + const column = (name: string) => + source.columns.find((c) => c.name === name); + for (const name of texts) expect(column(`c ${name}`)?.default).toBe(name); + expect( + (column("kind") as unknown as { enum: { enumName: string } }).enum + .enumName, + ).toBe(close); + expect(column("kind")?.enumValues).toEqual(kind.values); + expect(column("level")?.enumValues).toEqual(level.values); + expect(column("level")?.default).toBe(bidi); + expect( + source.foreignKeys.map((fk) => + fk.reference().foreignColumns.map((c) => c.name), + ), + ).toEqual([[quote]]); + }); +}); diff --git a/packages/appkit/src/database/tooling/migrate.ts b/packages/appkit/src/database/tooling/migrate.ts index ec2343450..1613bb9d9 100644 --- a/packages/appkit/src/database/tooling/migrate.ts +++ b/packages/appkit/src/database/tooling/migrate.ts @@ -45,7 +45,7 @@ function readJournalText(folder: string): string | null { const file = journalFile(folder); return fs.existsSync(file) ? fs.readFileSync(file, "utf-8") : null; } -const DATABASE_TYPES_PATH = path.join( +export const DATABASE_TYPES_PATH = path.join( "shared/appkit-types", DATABASE_TYPES_FILE, ); diff --git a/packages/appkit/src/database/tooling/migrations.ts b/packages/appkit/src/database/tooling/migrations.ts index 81786a253..0983b5b74 100644 --- a/packages/appkit/src/database/tooling/migrations.ts +++ b/packages/appkit/src/database/tooling/migrations.ts @@ -27,8 +27,12 @@ interface GenerateOptions { /** Absolute migrations folder. */ readonly folder: string; readonly schema: Schema; + /** Default "migration" → "NNNN_migration". */ + readonly tag?: string; /** false: an ambiguous rename throws instead of prompting. */ readonly interactive: boolean; + /** Sees the SQL before anything is written; throwing writes nothing. */ + readonly beforeWrite?: (statements: readonly string[]) => void; } export interface GeneratedMigration { @@ -60,7 +64,7 @@ export class MigrationRenameAmbiguityError extends Error { kind === "column" ? `columns of table "${table}"` : `${kind}s`; super( `Cannot tell whether ${scope} were renamed: removed ${deleted.join(", ")}; added ${created.join(", ")}. ` + - "Run `appkit db migrate` in an interactive terminal to choose between a rename and a drop plus create.", + "Run the command in an interactive terminal to choose between a rename and a drop plus create.", ); this.name = "MigrationRenameAmbiguityError"; this.kind = kind; @@ -73,6 +77,7 @@ export class MigrationRenameAmbiguityError extends Error { /** drizzle-kit's statement separator in v7 migration files. */ const BREAKPOINT = "--> statement-breakpoint\n"; const ORIGIN_ID = "00000000-0000-0000-0000-000000000000"; +const TAG_NAME = /^[a-z0-9_]+$/; /** The parts of a drizzle-kit PostgreSQL snapshot this module reads. */ interface Snapshot { @@ -290,12 +295,18 @@ function writeFileAtomic(file: string, content: string): void { /** * Diff the schema against the latest committed snapshot with drizzle-kit and - * write the next v7 migration: `NNNN_migration.sql`, `meta/NNNN_snapshot.json`, + * write the next v7 migration: `NNNN_.sql`, `meta/NNNN_snapshot.json`, * and the appended `meta/_journal.json`. null when nothing changed. */ export async function generateMigrationFiles( options: GenerateOptions, ): Promise { + const name = options.tag ?? "migration"; + if (!TAG_NAME.test(name)) { + throw new Error( + `Migration tag "${name}" must use lowercase letters, digits and underscores`, + ); + } const { generateDrizzleJson, generateMigration } = await loadDrizzleKit(); const journal = readJournal(options.folder); @@ -319,11 +330,12 @@ export async function generateMigrationFiles( cur as unknown as KitSnapshot, ); if (statements.length === 0) return null; + options.beforeWrite?.(statements); const entries = journal?.entries ?? []; const last = entries.at(-1); const idx = last ? last.idx + 1 : 0; - const tag = `${migrationPrefix(idx)}_migration`; + const tag = `${migrationPrefix(idx)}_${name}`; const entry: JournalEntry = { idx, version: "7", diff --git a/packages/appkit/src/database/tooling/tests/contract.test.ts b/packages/appkit/src/database/tooling/tests/contract.test.ts index f4e3765d3..192305dcf 100644 --- a/packages/appkit/src/database/tooling/tests/contract.test.ts +++ b/packages/appkit/src/database/tooling/tests/contract.test.ts @@ -10,6 +10,7 @@ const exports: DatabaseTooling = tooling; const contractKeys = { createDbContext: true, + runIntrospect: true, runMigrate: true, } satisfies Record; diff --git a/packages/appkit/src/database/tooling/tests/migrations.test.ts b/packages/appkit/src/database/tooling/tests/migrations.test.ts index edfdf0fb8..c187abf92 100644 --- a/packages/appkit/src/database/tooling/tests/migrations.test.ts +++ b/packages/appkit/src/database/tooling/tests/migrations.test.ts @@ -303,6 +303,17 @@ describe("generateMigrationFiles", () => { expect(listFiles(folder)).toEqual(before); }); + test("rejects a tag that is not a safe file name", async () => { + await expect( + generateMigrationFiles({ + folder: tempFolder(), + schema: blog(), + tag: "../escape", + interactive: false, + }), + ).rejects.toThrow(/lowercase letters/); + }); + test("refuses to diff when the latest snapshot is missing", async () => { const folder = tempFolder(); await generateMigrationFiles({ @@ -342,6 +353,65 @@ describe("generateMigrationFiles", () => { ), ); }); + + test("beforeWrite sees the SQL first, and throwing from it writes nothing", async () => { + const folder = tempFolder(); + const seen: string[][] = []; + + const error = await generateMigrationFiles({ + folder, + schema: blog(), + interactive: false, + beforeWrite: (statements) => { + seen.push([...statements]); + throw new Error("refused"); + }, + }).catch((caught: unknown) => caught); + + expect((error as Error).message).toBe("refused"); + expect(seen).toHaveLength(1); + expect(seen[0]?.join("\n")).toContain('CREATE TABLE "app"."authors"'); + expect(listFiles(folder)).toEqual([]); + }); +}); + +describe("discardMigrationFiles", () => { + test("a first migration goes with the folders it created", async () => { + const folder = tempFolder(); + const migration = await generateMigrationFiles({ + folder, + schema: blog(), + interactive: false, + }); + + discardMigrationFiles(folder, migration as never, null); + + expect(fs.existsSync(folder)).toBe(false); + }); + + test("a later migration goes, and the previous journal comes back byte for byte", async () => { + const folder = tempFolder(); + await generateMigrationFiles({ + folder, + schema: blog(), + interactive: false, + }); + const journalFile = path.join(folder, "meta/_journal.json"); + const journalBefore = `${fs.readFileSync(journalFile, "utf-8")}\n`; + fs.writeFileSync(journalFile, journalBefore); + const filesBefore = listFiles(folder); + const migration = await generateMigrationFiles({ + folder, + schema: blog({ summary: text() }), + interactive: false, + }); + expect(listFiles(folder)).not.toEqual(filesBefore); + + discardMigrationFiles(folder, migration as never, journalBefore); + + expect(listFiles(folder)).toEqual(filesBefore); + expect(fs.readFileSync(journalFile, "utf-8")).toBe(journalBefore); + }); }); describe("breakingStatements", () => { diff --git a/packages/shared/src/cli/commands/db/index.ts b/packages/shared/src/cli/commands/db/index.ts index aca41b888..35ff2f721 100644 --- a/packages/shared/src/cli/commands/db/index.ts +++ b/packages/shared/src/cli/commands/db/index.ts @@ -1,5 +1,6 @@ import { Command } from "commander"; +import { dbIntrospectCommand } from "./introspect"; import { dbMigrateCommand } from "./migrate"; /** @@ -8,4 +9,5 @@ import { dbMigrateCommand } from "./migrate"; */ export const dbCommand = new Command("db") .description("Lakebase database lifecycle") - .addCommand(dbMigrateCommand); + .addCommand(dbMigrateCommand) + .addCommand(dbIntrospectCommand); diff --git a/packages/shared/src/cli/commands/db/introspect.test.ts b/packages/shared/src/cli/commands/db/introspect.test.ts new file mode 100644 index 000000000..8c9e2e6c1 --- /dev/null +++ b/packages/shared/src/cli/commands/db/introspect.test.ts @@ -0,0 +1,171 @@ +import { + afterEach, + beforeEach, + describe, + expect, + type MockInstance, + test, + vi, +} from "vitest"; + +const tooling = vi.hoisted(() => ({ + createDbContext: vi.fn(() => ({ root: "/app", env: {} })), + runIntrospect: vi.fn(), +})); + +vi.mock("@databricks/appkit/database-tooling", () => tooling); + +import { dbCommand } from "./index"; + +let log: MockInstance; +let error: MockInstance; +const output = (spy: MockInstance) => + spy.mock.calls.map((call) => call.join(" ")).join("\n"); + +const report = { + branch: "projects/p/branches/dev-alice-3f9a2c", + schemaName: "app", + tables: ["authors", "posts"], + schemaFile: `${process.cwd()}/config/database/schema.ts`, + backupFile: `${process.cwd()}/config/database/schema.ts.bak`, + skipped: [ + { + table: "events", + reason: "composite primary key (a, b) is not supported", + }, + { + table: "posts", + column: "score", + reason: 'type "numeric(10,2)" is not supported', + }, + ], + warnings: [ + "app.posts.score is NOT NULL with no default but is not in schema.ts.", + ], + approximated: [ + "app.authors.id: a serial key is declared as id(), an identity generated BY DEFAULT", + ], + migration: { + tag: "0000_baseline", + file: `${process.cwd()}/config/database/migrations/0000_baseline.sql`, + statements: ["CREATE TABLE a", "CREATE TABLE b"], + }, + recorded: ["0000_baseline"], + typesFile: `${process.cwd()}/shared/appkit-types/database.d.ts`, +}; + +beforeEach(() => { + for (const option of dbCommand.commands.flatMap((c) => + c.options.map((o) => [c, o] as const), + )) { + option[0].setOptionValueWithSource( + option[1].attributeName(), + undefined, + "default", + ); + } + tooling.runIntrospect.mockReset().mockResolvedValue(report); + log = vi.spyOn(console, "log").mockImplementation(() => undefined); + error = vi.spyOn(console, "error").mockImplementation(() => undefined); +}); + +afterEach(() => { + vi.restoreAllMocks(); + process.exitCode = undefined; +}); + +describe("appkit db introspect", () => { + test("prints the written schema, each skip, the loud warnings, and the recorded baseline", async () => { + await dbCommand.parseAsync(["introspect"], { from: "user" }); + + expect(tooling.runIntrospect).toHaveBeenCalledWith( + { root: "/app", env: {} }, + { interactive: false }, + ); + const printed = output(log); + expect(printed).toContain( + "Wrote config/database/schema.ts from schema app on dev-alice-3f9a2c (2 tables); the previous file is config/database/schema.ts.bak", + ); + expect(printed).toContain( + "Skipped table events: composite primary key (a, b) is not supported", + ); + expect(printed).toContain( + 'Skipped posts.score: type "numeric(10,2)" is not supported', + ); + expect(printed).toContain("app.posts.score is NOT NULL with no default"); + expect(printed).toContain( + "Approximated app.authors.id: a serial key is declared as id(), an identity generated BY DEFAULT", + ); + expect(printed).toContain( + "This branch keeps the approximated objects as they are; a database the migrations create gets what schema.ts declares.", + ); + expect(printed).toContain( + "Generated config/database/migrations/0000_baseline.sql (2 statements)", + ); + expect(printed).toContain( + "Recorded 0000_baseline as applied on dev-alice-3f9a2c: the branch already has what it describes", + ); + expect(printed).toContain("Generated shared/appkit-types/database.d.ts"); + expect(process.exitCode).toBeUndefined(); + }); + + test("--schema-name is passed through", async () => { + await dbCommand.parseAsync(["introspect", "--schema-name", "app"], { + from: "user", + }); + + expect(tooling.runIntrospect).toHaveBeenCalledWith(expect.anything(), { + interactive: false, + schemaName: "app", + }); + }); + + test("migrations an interrupted run left unrecorded are listed with the one it recorded", async () => { + tooling.runIntrospect.mockResolvedValue({ + ...report, + migration: null, + recorded: ["0002_introspect"], + }); + + await dbCommand.parseAsync(["introspect"], { from: "user" }); + + expect(output(log)).toContain( + "Recorded 0002_introspect as applied on dev-alice-3f9a2c: the branch already has what it describes", + ); + }); + + test.each([ + [[], "No migration needed: schema.ts has no tables"], + [ + ["authors"], + "No migration needed: the migrations already describe this schema", + ], + ])("without a migration, %j tables print %j", async (tables, line) => { + tooling.runIntrospect.mockResolvedValue({ + ...report, + tables, + migration: null, + recorded: [], + }); + + await dbCommand.parseAsync(["introspect"], { from: "user" }); + + expect(output(log)).toContain(line); + expect(output(log)).not.toContain("Recorded"); + }); + + test("a refusal is printed and exits 1", async () => { + tooling.runIntrospect.mockRejectedValue( + new Error( + "Refusing to introspect: production is not your dev or session branch (it is the project's default branch).", + ), + ); + + await dbCommand.parseAsync(["introspect"], { from: "user" }); + + expect(output(error)).toContain( + "Refusing to introspect: production is not your dev or session branch", + ); + expect(process.exitCode).toBe(1); + }); +}); diff --git a/packages/shared/src/cli/commands/db/introspect.ts b/packages/shared/src/cli/commands/db/introspect.ts new file mode 100644 index 000000000..1d2b37031 --- /dev/null +++ b/packages/shared/src/cli/commands/db/introspect.ts @@ -0,0 +1,43 @@ +import { Command } from "commander"; + +import { loadDatabaseTooling } from "./load-tooling"; +import { fail, isInteractive, printIntrospectReport } from "./output"; + +async function runDbIntrospect(flags: { schemaName?: string }): Promise { + try { + const tooling = await loadDatabaseTooling(); + const report = await tooling.runIntrospect(tooling.createDbContext(), { + interactive: isInteractive(), + ...(flags.schemaName ? { schemaName: flags.schemaName } : {}), + }); + printIntrospectReport(report); + } catch (error) { + fail(error); + } +} + +export const dbIntrospectCommand = new Command("introspect") + .description( + "Write config/database/schema.ts from the .env branch's tables, plus the migration that brings history up to them", + ) + .option("--schema-name ", "Postgres schema to read (default: app)") + .addHelpText( + "after", + ` +The previous schema.ts is kept as schema.ts.bak. What the schema builder cannot +express (composite keys, unsupported types or defaults, some foreign keys) is left +out with an "// appkit: skipped …" comment. + +Without migration history this writes 0000_baseline; with history, the next +NNNN_introspect, once the branch has applied every committed migration. The +migration is recorded as applied, so introspect runs only on your dev or session +branch; \`appkit db init\` creates one from the project's default branch. The +deployed app adopts 0000_baseline on start when its tables already exist. If +anything fails, schema.ts, database.d.ts and the migrations are left as they +were. + +Examples: + $ appkit db introspect + $ appkit db introspect --schema-name sales`, + ) + .action(runDbIntrospect); diff --git a/packages/shared/src/cli/commands/db/output.ts b/packages/shared/src/cli/commands/db/output.ts index f3dc540a4..b73bbe163 100644 --- a/packages/shared/src/cli/commands/db/output.ts +++ b/packages/shared/src/cli/commands/db/output.ts @@ -3,7 +3,7 @@ import process from "node:process"; import pc from "picocolors"; -import type { MigrateReport } from "./tooling-contract"; +import type { IntrospectReport, MigrateReport } from "./tooling-contract"; /** Prompts only when a human can answer them. */ export function isInteractive(): boolean { @@ -14,6 +14,11 @@ export function relative(file: string): string { return path.relative(process.cwd(), file) || file; } +/** The last segment of a Lakebase resource name: projects/p/branches/dev-a → dev-a. */ +function shortName(name: string): string { + return name.slice(name.lastIndexOf("/") + 1); +} + export const ok = (message: string) => console.log(`${pc.green("✔")} ${message}`); export const info = (message: string) => @@ -73,3 +78,44 @@ export function printMigrateReport(report: MigrateReport): void { ok(`Generated ${relative(report.typesFile)}`); } + +export function printIntrospectReport(report: IntrospectReport): void { + const count = report.tables.length; + ok( + `Wrote ${relative(report.schemaFile)} from schema ${report.schemaName} on ${shortName(report.branch)} (${count} table${count === 1 ? "" : "s"})${report.backupFile ? `; the previous file is ${relative(report.backupFile)}` : ""}`, + ); + for (const skip of report.skipped) { + warn( + `Skipped ${skip.column ? `${skip.table}.${skip.column}` : `table ${skip.table}`}: ${skip.reason}`, + ); + } + for (const warning of report.warnings) { + console.log(`${pc.bold(pc.red("!"))} ${pc.bold(warning)}`); + } + for (const difference of report.approximated) { + info(`Approximated ${difference}`); + } + if (report.approximated.length > 0) { + info( + "This branch keeps the approximated objects as they are; a database the migrations create gets what schema.ts declares.", + ); + } + if (report.migration) { + const statements = report.migration.statements.length; + ok( + `Generated ${relative(report.migration.file)} (${statements} statement${statements === 1 ? "" : "s"})`, + ); + } else { + info( + report.tables.length === 0 + ? "No migration needed: schema.ts has no tables" + : "No migration needed: the migrations already describe this schema", + ); + } + if (report.recorded.length > 0) { + ok( + `Recorded ${report.recorded.join(", ")} as applied on ${shortName(report.branch)}: the branch already has ${report.recorded.length === 1 ? "what it describes" : "what they describe"}`, + ); + } + ok(`Generated ${relative(report.typesFile)}`); +} diff --git a/packages/shared/src/cli/commands/db/tooling-contract.ts b/packages/shared/src/cli/commands/db/tooling-contract.ts index 06e73e359..cb39f69b2 100644 --- a/packages/shared/src/cli/commands/db/tooling-contract.ts +++ b/packages/shared/src/cli/commands/db/tooling-contract.ts @@ -19,6 +19,26 @@ export interface MigrateReport { readonly typesFile: string; } +export interface Skip { + readonly table: string; + readonly column?: string; + readonly reason: string; +} + +export interface IntrospectReport { + readonly branch: string; + readonly schemaName: string; + readonly tables: readonly string[]; + readonly schemaFile: string; + readonly backupFile: string | null; + readonly skipped: readonly Skip[]; + readonly warnings: readonly string[]; + readonly approximated: readonly string[]; + readonly migration: GeneratedMigration | null; + readonly recorded: readonly string[]; + readonly typesFile: string; +} + /** * The exports of `@databricks/appkit/database-tooling` the CLI calls. `shared` * must not depend on appkit, so the CLI imports the tooling dynamically and @@ -35,4 +55,8 @@ export interface DatabaseTooling { ctx: Ctx, options: { interactive: boolean }, ) => Promise; + readonly runIntrospect: ( + ctx: Ctx, + options: { interactive: boolean; schemaName?: string }, + ) => Promise; }