diff --git a/.github/workflows/integration-protect-ffi.yml b/.github/workflows/integration-protect-ffi.yml index 909f5bca8..6e058a24a 100644 --- a/.github/workflows/integration-protect-ffi.yml +++ b/.github/workflows/integration-protect-ffi.yml @@ -97,6 +97,35 @@ on: - 'packages/eql/crates/eql-bindings/src/v3/**' - 'packages/eql/crates/eql-bindings/sql/**' - 'packages/eql/Cargo.toml' + # EQL v4 and the codec the migration suites (`tests/migrate-v*-to-v4.test.ts`) + # re-encrypt through. `eql:v4:install` builds the v4 bundle from the SQL + # source with `tasks/build.sh` (which runs eql-codegen), and `eql:v4:codec` + # builds `tests/encryption/examples/v4_codec.rs` over eql-bindings' v4 + # targets, stack-encrypt and stack-kms. No published package carries any of + # it, so these paths are the only inputs that change what those suites run. + # The crate entries are the eql-bindings modules the codec compiles and the + # native binding does not (`v4`, `encryption`, `encoding`), and the + # eql-codegen / eql-domains sources `tasks/build.sh` renders the v4 bundle + # from. test-eql.yml runs the Rust and SQL suites on those files but not + # this one, so without them a v4 change skips the migration suites. + # Derived in scripts/__tests__/lib/eql-v4-suite-inputs.mjs and held here by + # eql-v4-suite-triggers.test.mjs; wasm-build-inputs-paths-filter.test.mjs + # allows exactly these beyond the native binding's files, and only here. + - 'packages/eql/crates/eql-bindings/src/encoding.rs' + - 'packages/eql/crates/eql-bindings/src/encryption.rs' + - 'packages/eql/crates/eql-bindings/src/encryption/**' + - 'packages/eql/crates/eql-bindings/src/v4/**' + - 'packages/eql/crates/eql-codegen/Cargo.toml' + - 'packages/eql/crates/eql-codegen/src/**' + - 'packages/eql/crates/eql-codegen/templates/**' + - 'packages/eql/crates/eql-domains/Cargo.toml' + - 'packages/eql/crates/eql-domains/src/**' + - 'packages/eql/Cargo.lock' + - 'packages/eql/src/**' + - 'packages/eql/tasks/**' + - 'packages/eql/tests/encryption/**' + - 'packages/stack-encrypt/**' + - 'packages/stack-kms/**' - '.github/workflows/integration-protect-ffi.yml' - '.github/actions/build-ffi-binding/**' - '.github/actions/build-auth-binding/**' @@ -146,6 +175,21 @@ on: - 'packages/eql/crates/eql-bindings/src/v3/**' - 'packages/eql/crates/eql-bindings/sql/**' - 'packages/eql/Cargo.toml' + - 'packages/eql/crates/eql-bindings/src/encoding.rs' + - 'packages/eql/crates/eql-bindings/src/encryption.rs' + - 'packages/eql/crates/eql-bindings/src/encryption/**' + - 'packages/eql/crates/eql-bindings/src/v4/**' + - 'packages/eql/crates/eql-codegen/Cargo.toml' + - 'packages/eql/crates/eql-codegen/src/**' + - 'packages/eql/crates/eql-codegen/templates/**' + - 'packages/eql/crates/eql-domains/Cargo.toml' + - 'packages/eql/crates/eql-domains/src/**' + - 'packages/eql/Cargo.lock' + - 'packages/eql/src/**' + - 'packages/eql/tasks/**' + - 'packages/eql/tests/encryption/**' + - 'packages/stack-encrypt/**' + - 'packages/stack-kms/**' - '.github/workflows/integration-protect-ffi.yml' - '.github/actions/build-ffi-binding/**' - '.github/actions/build-auth-binding/**' @@ -361,11 +405,24 @@ jobs: # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs cache_key_prefix: mise-v1-ubuntu-2404 - # One task, four things the suite cannot run without (see + # The EQL workspace's Rust, which `mise run setup` compiles for the + # migration suites: eql-codegen (the v4 bundle) and the v4 codec. Without + # a cache that is a cold build of stack-encrypt and its HTTP stack on every + # run. Keyed apart from the `sqlx-tests` cache test-eql.yml saves, which + # carries the SQLx graph this job never builds. + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + workspaces: packages/eql + shared-key: eql-v4-codec + + # One task, six things the suite cannot run without (see # integration-tests/tasks.toml): `pnpm install --frozen-lockfile` (a # no-op here — the step above already ran it), `docker compose up` for - # Postgres, the EQL **v2** bundle from a GitHub release, and EQL **v3** - # built from the in-tree `@cipherstash/eql`. + # Postgres, the EQL **v2** bundle from a GitHub release, EQL **v3** + # built from the in-tree `@cipherstash/eql`, EQL **v4** built from the + # in-tree EQL source, and the Rust v4 codec. The last two serve + # `tests/migrate-v*-to-v4.test.ts`: Stack Encrypt is the only v4 + # producer, and no TypeScript SDK reaches it yet. # # BOTH EQL versions are required and nothing else in this repo installs # v2: `tests/postgres.test.ts` creates `eql_v2_encrypted` columns and @@ -381,7 +438,7 @@ jobs: # `integration-db`'s per-job project names deliberately do not produce. # The fixed host port (5436) that comes with it cannot collide with the # shared stacks, which publish on ephemeral ports. - - name: Provision the suite (Postgres, EQL v2 + v3) + - name: Provision the suite (Postgres, EQL v2 + v3 + v4, v4 codec) working-directory: languages/typescript/packages/protect-ffi run: mise run setup diff --git a/AGENTS.md b/AGENTS.md index 73a0a56c3..9e7fbf3bc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -225,10 +225,10 @@ stays optional for everyone else. `languages/typescript/packages/protect-ffi/integration-tests/` is 19 files of **live** coverage — encrypt/decrypt, lock context, keysets, JS auth strategies, JSON SteVec, -Postgres (EQL v2 *and* v3), and a WASM round trip — and it is the only place -several of those paths are exercised at all. It needs three things a normal -`pnpm test` does not have: **Docker**, **CipherStash credentials**, and **both -EQL versions installed** in the database. +Postgres (EQL v2 *and* v3), EQL v3 -> v4 migration, and a WASM round trip — and +it is the only place several of those paths are exercised at all. It needs +three things a normal `pnpm test` does not have: **Docker**, **CipherStash +credentials**, and **EQL v2, v3 and v4 installed** in the database. - **It is a pnpm workspace member, and `pnpm test` must never reach it.** Named literally in `pnpm-workspace.yaml` (the `languages/typescript/packages/*` glob is one level deep and @@ -290,6 +290,16 @@ EQL versions installed** in the database. `eql_v2.add_encrypted_constraint`) while `tests/postgres-v3.test.ts` needs the `eql_v3_*` domains. Skip either and half the suite fails on missing SQL functions. +- **The migration suites need EQL v4 and a Rust codec.** + `tests/migrate-v*-to-v4.test.ts` decrypt legacy rows with protect-ffi and + re-encrypt them as v4, and Stack Encrypt is the only v4 producer, so they call + `packages/eql/tests/encryption/examples/v4_codec.rs` through + `tests/v4-codec.ts`. `eql:v4:install` builds the v4 bundle from the in-tree + EQL source (`bash tasks/build.sh`, called directly so it needs cargo and not + `packages/eql`'s mise config) and `eql:v4:codec` builds the codec; both are + part of `mise run setup`. The CI filter cannot name `packages/eql/crates/**` + (`wasm-build-inputs-paths-filter.test.mjs`), so an edit to eql-codegen or + eql-bindings' v4 module reaches this suite only through `test-eql.yml`. - **`eql:v3:install` builds `@cipherstash/eql` first, and has to.** The task reads the bundle through `@cipherstash/eql/sql`, which the package's `exports` map resolves to `dist/sql.js` — a tsup output. That was free while `npm ci` diff --git a/languages/typescript/packages/protect-ffi/integration-tests/package.json b/languages/typescript/packages/protect-ffi/integration-tests/package.json index 20010bb06..41ef13656 100644 --- a/languages/typescript/packages/protect-ffi/integration-tests/package.json +++ b/languages/typescript/packages/protect-ffi/integration-tests/package.json @@ -2,7 +2,7 @@ "name": "@cipherstash/ffi-integration-tests", "version": "1.0.2", "private": true, - "description": "Live integration coverage for @cipherstash/protect-ffi — real ZeroKMS, real Postgres, EQL v2 and v3", + "description": "Live integration coverage for @cipherstash/protect-ffi — real ZeroKMS, real Postgres, EQL v2, v3 and v4", "license": "MIT", "scripts": { "typecheck": "tsc", diff --git a/languages/typescript/packages/protect-ffi/integration-tests/tasks.toml b/languages/typescript/packages/protect-ffi/integration-tests/tasks.toml index 0ba49f62e..4ac0ae873 100644 --- a/languages/typescript/packages/protect-ffi/integration-tests/tasks.toml +++ b/languages/typescript/packages/protect-ffi/integration-tests/tasks.toml @@ -45,6 +45,30 @@ fi docker exec -i protect-ffi-postgres psql postgresql://cipherstash:password@postgres:5432/cipherstash -f- < "$tmp" """ +# EQL v4, for the migration suites (`tests/migrate-v*-to-v4.test.ts`). No npm +# package ships the v4 bundle yet, so it is built from the in-tree EQL source: +# `tasks/build.sh` is what `mise run build` runs in `packages/eql`, called +# directly so this task needs only cargo, not that directory's mise config (which +# a runner has not trusted). It writes both bundles; only v4 is installed here, +# v3 comes from `eql:v3:install` above. +["eql:v4:install"] +description = "Build EQL v4 from the in-tree source and install it beside v2 and v3" +dir = "{{ config_root }}/../../../../packages/eql" +run = """ +set -e +bash tasks/build.sh +test -s release/cipherstash-encrypt-v4.sql +docker exec -i protect-ffi-postgres psql -v ON_ERROR_STOP=1 -q postgresql://cipherstash:password@postgres:5432/cipherstash -f- < release/cipherstash-encrypt-v4.sql +""" + +# The Rust codec the migration suites re-encrypt through: Stack Encrypt is the +# only EQL v4 producer and no TypeScript SDK reaches it yet. See +# `tests/v4-codec.ts` and `packages/eql/tests/encryption/examples/v4_codec.rs`. +["eql:v4:codec"] +description = "Build the EQL v4 codec the migration suites call" +dir = "{{ config_root }}/../../../../packages/eql" +run = "cargo build --locked -p eql-encryption-tests --example v4_codec" + # `pnpm install`, not `npm ci`: this directory is a pnpm workspace member as of # CIP-3744, so its dependencies come from the repo lockfile and its # `@cipherstash/protect-ffi` and `@cipherstash/eql` resolve to the packages in @@ -62,6 +86,8 @@ run = [ "mise run eql:download", "mise run eql:install", "mise run eql:v3:install", + "mise run eql:v4:install", + "mise run eql:v4:codec", ] ["test:integration"] diff --git a/languages/typescript/packages/protect-ffi/integration-tests/tests/migrate-v3-to-v4.test.ts b/languages/typescript/packages/protect-ffi/integration-tests/tests/migrate-v3-to-v4.test.ts new file mode 100644 index 000000000..19aba2889 --- /dev/null +++ b/languages/typescript/packages/protect-ffi/integration-tests/tests/migrate-v3-to-v4.test.ts @@ -0,0 +1,477 @@ +import 'dotenv/config' +import { + decryptBulk, + type EncryptConfig, + type EncryptedPayload, + encryptBulk, + encryptQuery, + newClient, +} from '@cipherstash/protect-ffi' +import { Client } from 'pg' +import { afterAll, beforeAll, describe, expect, test } from 'vitest' +import { + CHECK_VIOLATION, + decryptV4, + encryptV4, + type Plaintext, + queryV4, + requireEqlV4, + type V4Type, +} from './v4-codec' + +// EQL v3 -> v4 migration, end to end, against live ZeroKMS and Postgres. +// +// v3 is what Stack 1.x writes: protect-ffi in `eqlVersion: 3` is the engine +// `@cipherstash/stack` encrypts through. v4 is what Stack Encrypt writes. The +// two use different ciphertext formats and derive their search terms +// differently, so a migration cannot be a cast or an edit of `v`: every value +// is decrypted with the v3 client and encrypted again as v4. +// +// The shape is the one a customer runs, in one table: +// +// 1. the v3 columns hold live data; +// 2. add a v4 column beside each one; +// 3. backfill: read the v3 payloads, decrypt them, encrypt the plaintexts as +// v4, write them to the new columns; +// 4. switch queries to the v4 columns and v4 operands; +// 5. drop the v3 columns and rename the v4 ones into their place. +// +// The v4 values keep the v3 column's encryption context (`customers/email`, +// not `customers/email_v4`): the context is bound into the ciphertext and the +// search terms, so naming the logical column is what lets step 5 rename +// without re-encrypting again. +// +// Requires EQL v3 and v4 both installed (`mise run setup` installs v2, v3 and +// v4) and the v4 codec built (`mise run eql:v4:codec`); see ./v4-codec.ts. + +const TABLE = 'customers' + +const encryptConfig: EncryptConfig = { + v: 1, + tables: { + [TABLE]: { + email: { cast_as: 'text', indexes: { unique: {} } }, + age: { cast_as: 'int', indexes: { ore: {} } }, + rank: { cast_as: 'int', indexes: { ope: {} } }, + bio: { cast_as: 'text', indexes: { match: {} } }, + active: { cast_as: 'boolean', indexes: {} }, + }, + }, +} + +type Customer = { + email: string + age: number + rank: number + bio: string + active: boolean +} + +const customers: Customer[] = [ + { + email: 'ada@example.com', + age: 36, + rank: 3, + bio: 'analytical engine notes', + active: true, + }, + { + email: 'grace@example.com', + age: 85, + rank: 1, + bio: 'compiler pioneer and admiral', + active: true, + }, + { + email: 'alan@example.com', + age: 41, + rank: 2, + bio: 'computable numbers and machines', + active: false, + }, + { + email: 'edsger@example.com', + age: 72, + rank: 5, + bio: 'structured programming advocate', + active: false, + }, + { + email: 'barbara@example.com', + age: 29, + rank: 4, + bio: 'abstraction and programming methodology', + active: true, + }, +] + +type Column = keyof Customer + +/** Each column's v3 domain and the v4 type it migrates to, capability for capability. */ +const columns: Record = { + email: { + v3: 'public.eql_v3_text_eq', + v4: 'TextEq', + v4Domain: 'public.eql_v4_text_eq', + }, + age: { + v3: 'public.eql_v3_integer_ord_ore', + v4: 'IntegerOrdOre', + v4Domain: 'public.eql_v4_integer_ord_ore', + }, + rank: { + v3: 'public.eql_v3_integer_ord_ope', + v4: 'IntegerOrdOpe', + v4Domain: 'public.eql_v4_integer_ord_ope', + }, + bio: { + v3: 'public.eql_v3_text_match', + v4: 'TextMatch', + v4Domain: 'public.eql_v4_text_match', + }, + active: { + v3: 'public.eql_v3_boolean', + v4: 'Boolean', + v4Domain: 'public.eql_v4_boolean', + }, +} + +const columnNames = Object.keys(columns) as Column[] + +/** The emails of `rows` in id order: the oracle every query is compared to. */ +const emails = (rows: Customer[]) => rows.map((row) => row.email) + +describe('EQL v3 -> v4 migration', () => { + const pg = new Client() + let connected = false + let v3Client: Awaited> + + /** Ids the query returns, read as emails through the plaintext oracle. */ + async function emailsWhere(sql: string, params: unknown[] = []) { + const res = await pg.query<{ id: number }>( + `SELECT id FROM ${TABLE} ${sql}`, + params, + ) + return res.rows.map((row) => customers[row.id - 1].email) + } + + async function v3Operand( + column: Column, + plaintext: Plaintext, + indexType: 'unique' | 'ore' | 'ope' | 'match', + ) { + return encryptQuery(v3Client, { + plaintext, + column, + table: TABLE, + indexType, + }) + } + + function v4Operand(column: Column, value: Plaintext) { + const [operand] = queryV4([ + { type: columns[column].v4, table: TABLE, column, value }, + ]) + return operand + } + + beforeAll(async () => { + v3Client = await newClient({ encryptConfig, eqlVersion: 3 }) + await pg.connect() + connected = true + await requireEqlV4(pg) + + // Step 1: the table as Stack 1.x left it — v3 columns only. + await pg.query(`DROP TABLE IF EXISTS ${TABLE}`) + await pg.query(` + CREATE TABLE ${TABLE} ( + id integer PRIMARY KEY, + ${columnNames.map((c) => `${c} ${columns[c].v3}`).join(',\n ')} + ) + `) + + const encrypted = await encryptBulk(v3Client, { + plaintexts: customers.flatMap((customer) => + columnNames.map((column) => ({ + plaintext: customer[column], + column, + table: TABLE, + })), + ), + }) + for (const index of customers.keys()) { + const row = encrypted.slice( + index * columnNames.length, + (index + 1) * columnNames.length, + ) + await pg.query( + `INSERT INTO ${TABLE} (id, ${columnNames.join(', ')}) + VALUES ($1, ${columnNames.map((_, i) => `$${i + 2}::jsonb`).join(', ')})`, + [index + 1, ...row], + ) + } + }) + + // In `afterAll`, not returned from `beforeAll`: a returned teardown runs only + // when setup succeeds, so a throw after `connect()` would leave the socket + // open and the table behind. + afterAll(async () => { + if (!connected) return + try { + await pg.query(`DROP TABLE IF EXISTS ${TABLE}`) + } finally { + await pg.end() + } + }) + + test('the v3 table answers the queries the migration must preserve', async () => { + const all = await pg.query<{ v: number }>( + `SELECT DISTINCT (email::jsonb ->> 'v')::int AS v FROM ${TABLE}`, + ) + expect(all.rows).toEqual([{ v: 3 }]) + + expect( + await emailsWhere('WHERE email = $1::jsonb::eql_v3.query_text_eq', [ + await v3Operand('email', 'alan@example.com', 'unique'), + ]), + ).toEqual(['alan@example.com']) + + expect( + await emailsWhere( + 'WHERE age > $1::jsonb::eql_v3.query_integer_ord_ore ORDER BY id', + [await v3Operand('age', 40, 'ore')], + ), + ).toEqual(emails(customers.filter((c) => c.age > 40))) + + expect( + await emailsWhere( + 'WHERE bio @@ $1::jsonb::eql_v3.query_text_match ORDER BY id', + [await v3Operand('bio', 'programming', 'match')], + ), + ).toEqual(emails(customers.filter((c) => c.bio.includes('programming')))) + }) + + test('a v3 payload cannot be cast into a v4 column', async () => { + await pg.query( + `ALTER TABLE ${TABLE} ADD COLUMN probe ${columns.email.v4Domain}`, + ) + try { + await expect( + pg.query( + `UPDATE ${TABLE} SET probe = email::jsonb::${columns.email.v4Domain}`, + ), + ).rejects.toMatchObject({ code: CHECK_VIOLATION }) + } finally { + await pg.query(`ALTER TABLE ${TABLE} DROP COLUMN probe`) + } + }) + + test('relabelling a v3 payload as v4 does not make it a v4 value', async () => { + const res = await pg.query<{ email: EncryptedPayload }>( + `SELECT email::jsonb AS email FROM ${TABLE} WHERE id = 3`, + ) + const relabelled = { + ...(res.rows[0].email as { hm: string }), + v: 4, + } + + // Stack Encrypt refuses to open a cipherstash-client ciphertext. + expect(() => + decryptV4([ + { + type: 'TextEq', + table: TABLE, + column: 'email', + payload: relabelled, + }, + ]), + ).toThrowError() + + // And its search term was derived by the other producer, so a v4 query for + // the same plaintext does not match it. + const [v4Query] = queryV4([ + { + type: 'TextEq', + table: TABLE, + column: 'email', + value: 'alan@example.com', + }, + ]) + // Both terms must exist, or `not.toEqual` passes on `undefined`. + expect(typeof v4Query.hm).toBe('string') + expect(typeof relabelled.hm).toBe('string') + expect(v4Query.hm).not.toEqual(relabelled.hm) + }) + + test('backfill: decrypt every v3 value and re-encrypt it as v4 beside it', async () => { + // Step 2: one v4 column per v3 column. + for (const column of columnNames) { + await pg.query( + `ALTER TABLE ${TABLE} ADD COLUMN ${column}_v4 ${columns[column].v4Domain}`, + ) + } + + // Step 3: read, decrypt with the v3 client, re-encrypt as v4, write back. + const res = await pg.query< + Record & { id: number } + >( + `SELECT id, ${columnNames.map((c) => `${c}::jsonb AS ${c}`).join(', ')} + FROM ${TABLE} ORDER BY id`, + ) + const plaintexts = await decryptBulk(v3Client, { + ciphertexts: res.rows.flatMap((row) => + columnNames.map((column) => ({ ciphertext: row[column] })), + ), + }) + const v4 = encryptV4( + res.rows.flatMap((_, r) => + columnNames.map((column, c) => ({ + type: columns[column].v4, + table: TABLE, + column, + value: plaintexts[r * columnNames.length + c] as Plaintext, + })), + ), + ) + for (const [r, row] of res.rows.entries()) { + const values = v4.slice( + r * columnNames.length, + (r + 1) * columnNames.length, + ) + await pg.query( + `UPDATE ${TABLE} SET ${columnNames + .map((c, i) => `${c}_v4 = $${i + 2}::jsonb`) + .join(', ')} WHERE id = $1`, + [row.id, ...values], + ) + } + + const versions = await pg.query<{ v: number; n: number }>( + `SELECT (email_v4::jsonb ->> 'v')::int AS v, count(*)::int AS n FROM ${TABLE} GROUP BY 1`, + ) + expect(versions.rows).toEqual([{ v: 4, n: customers.length }]) + }) + + test('every migrated value decrypts as v4 to the original plaintext', async () => { + const res = await pg.query & { id: number }>( + `SELECT id, ${columnNames.map((c) => `${c}_v4::jsonb AS ${c}`).join(', ')} + FROM ${TABLE} ORDER BY id`, + ) + const opened = decryptV4( + res.rows.flatMap((row) => + columnNames.map((column) => ({ + type: columns[column].v4, + table: TABLE, + column, + payload: row[column], + })), + ), + ) + expect(opened).toEqual( + customers.flatMap((customer) => columnNames.map((c) => customer[c])), + ) + }) + + test('equality: the v4 column finds the row the v3 column found', async () => { + const v3 = await emailsWhere( + 'WHERE email = $1::jsonb::eql_v3.query_text_eq', + [await v3Operand('email', 'alan@example.com', 'unique')], + ) + const v4 = await emailsWhere( + 'WHERE email_v4 = $1::jsonb::eql_v4.query_text_eq', + [v4Operand('email', 'alan@example.com')], + ) + expect(v4).toEqual(v3) + expect(v4).toEqual(['alan@example.com']) + }) + + test('range: ORE comparisons agree across versions', async () => { + const expected = emails(customers.filter((c) => c.age > 40)) + const v3 = await emailsWhere( + 'WHERE age > $1::jsonb::eql_v3.query_integer_ord_ore ORDER BY id', + [await v3Operand('age', 40, 'ore')], + ) + const v4 = await emailsWhere( + 'WHERE age_v4 > $1::jsonb::eql_v4.query_integer_ord_ore ORDER BY id', + [v4Operand('age', 40)], + ) + expect(v3).toEqual(expected) + expect(v4).toEqual(expected) + }) + + test('ordering: ORE and OPE sort orders agree across versions', async () => { + const byAge = emails([...customers].sort((a, b) => a.age - b.age)) + expect(await emailsWhere('ORDER BY eql_v3.ord_term_ore(age)')).toEqual( + byAge, + ) + expect(await emailsWhere('ORDER BY eql_v4.ord_term_ore(age_v4)')).toEqual( + byAge, + ) + + const byRank = emails([...customers].sort((a, b) => a.rank - b.rank)) + expect(await emailsWhere('ORDER BY eql_v3.ord_term(rank)')).toEqual(byRank) + expect(await emailsWhere('ORDER BY eql_v4.ord_term(rank_v4)')).toEqual( + byRank, + ) + }) + + test('match: the v4 bloom filter finds the rows the v3 one found', async () => { + const expected = emails( + customers.filter((c) => c.bio.includes('programming')), + ) + const v3 = await emailsWhere( + 'WHERE bio @@ $1::jsonb::eql_v3.query_text_match ORDER BY id', + [await v3Operand('bio', 'programming', 'match')], + ) + const v4 = await emailsWhere( + 'WHERE bio_v4 @@ $1::jsonb::eql_v4.query_text_match ORDER BY id', + [v4Operand('bio', 'programming')], + ) + expect(v3).toEqual(expected) + expect(v4).toEqual(expected) + }) + + test('a v3 operand against a v4 column matches nothing, silently', async () => { + // The cut-over hazard: the column moved to v4 but the query still comes + // from the v3 client. No operator is defined for the pair, so Postgres + // falls back to `jsonb = jsonb` and returns no rows rather than erroring + // (pinned in packages/eql/tests/sqlx/tests/v3_v4_coexistence_tests.rs). + // Move the reads and the query client together. + expect( + await emailsWhere('WHERE email_v4 = $1::jsonb::eql_v3.query_text_eq', [ + await v3Operand('email', 'alan@example.com', 'unique'), + ]), + ).toEqual([]) + }) + + test('cut-over: drop the v3 columns and the v4 values answer under the original names', async () => { + // Step 5. The v4 values were encrypted under `customers/`, so + // renaming the database column changes nothing they are bound to. + for (const column of columnNames) { + await pg.query(`ALTER TABLE ${TABLE} DROP COLUMN ${column}`) + await pg.query( + `ALTER TABLE ${TABLE} RENAME COLUMN ${column}_v4 TO ${column}`, + ) + } + + expect( + await emailsWhere('WHERE email = $1::jsonb::eql_v4.query_text_eq', [ + v4Operand('email', 'grace@example.com'), + ]), + ).toEqual(['grace@example.com']) + + const res = await pg.query<{ active: unknown }>( + `SELECT active::jsonb AS active FROM ${TABLE} ORDER BY id`, + ) + expect( + decryptV4( + res.rows.map((row) => ({ + type: 'Boolean' as const, + table: TABLE, + column: 'active', + payload: row.active, + })), + ), + ).toEqual(customers.map((c) => c.active)) + }) +}) diff --git a/languages/typescript/packages/protect-ffi/integration-tests/tests/v4-codec.ts b/languages/typescript/packages/protect-ffi/integration-tests/tests/v4-codec.ts new file mode 100644 index 000000000..d930cc126 --- /dev/null +++ b/languages/typescript/packages/protect-ffi/integration-tests/tests/v4-codec.ts @@ -0,0 +1,139 @@ +import { spawnSync } from 'node:child_process' +import { existsSync } from 'node:fs' +import { resolve } from 'node:path' +import type { Client } from 'pg' + +// EQL v4 has one producer, Stack Encrypt, and it is reached from Rust and Go +// only: no TypeScript SDK writes `v: 4` yet. The migration suites +// (`migrate-v*-to-v4.test.ts`) therefore decrypt legacy rows here with +// protect-ffi and re-encrypt the plaintexts through a small Rust codec, +// `packages/eql/tests/encryption/examples/v4_codec.rs`. It dispatches on the +// EQL type name through `eql_bindings::encryption::targets`, the same entry +// point the Go guest uses, under live ZeroKMS and the same `CS_*` credentials +// as protect-ffi. +// +// Built by `mise run eql:v4:codec` (part of `mise run setup`). Set +// `EQL_V4_CODEC` to use a binary somewhere else. + +const repoRoot = resolve(__dirname, '../../../../../..') + +export const codecPath = + process.env.EQL_V4_CODEC ?? + resolve(repoRoot, 'packages/eql/target/debug/examples/v4_codec') + +/** An EQL v4 type name, e.g. `TextEq`, as `targets::targets()` lists it. */ +export type V4Type = + | 'Boolean' + | 'IntegerOrdOpe' + | 'IntegerOrdOre' + | 'TextEq' + | 'TextMatch' + +export type Plaintext = string | number | boolean + +export type V4Item = { + type: V4Type + table: string + column: string + value: Plaintext +} + +export type V4Payload = { v: 4; i: { t: string; c: string } } & Record< + string, + unknown +> + +type Request = { + encrypt?: V4Item[] + query?: V4Item[] + decrypt?: { + type: V4Type + table: string + column: string + payload: unknown + }[] +} + +type Response = { + encrypt: V4Payload[] + query: V4Payload[] + decrypt: Plaintext[] +} + +const CODEC_TIMEOUT_MS = 60_000 + +function run(request: Request): Response { + if (!existsSync(codecPath)) { + throw new Error( + `EQL v4 codec not found at ${codecPath}. Build it with \`mise run eql:v4:codec\` from languages/typescript/packages/protect-ffi, or set EQL_V4_CODEC.`, + ) + } + const result = spawnSync(codecPath, { + input: JSON.stringify(request), + encoding: 'utf8', + env: process.env, + maxBuffer: 64 * 1024 * 1024, + // The codec authenticates against live ZeroKMS. `spawnSync` blocks the + // event loop, so vitest's own timeout cannot interrupt a stall; this does. + timeout: CODEC_TIMEOUT_MS, + killSignal: 'SIGKILL', + }) + if (result.error) { + if ((result.error as NodeJS.ErrnoException).code === 'ETIMEDOUT') { + throw new Error( + `EQL v4 codec did not finish within ${CODEC_TIMEOUT_MS / 1000}s and was killed.`, + ) + } + throw result.error + } + if (result.status !== 0) { + throw new Error( + `EQL v4 codec exited ${result.status}: ${result.stderr.trim()}`, + ) + } + return JSON.parse(result.stdout) as Response +} + +/** Encrypt each item as the stored EQL v4 payload for its type. */ +export function encryptV4(items: V4Item[]): V4Payload[] { + if (items.length === 0) return [] + return run({ encrypt: items }).encrypt +} + +/** + * The term-only EQL v4 query operand for each item. Bind it as the type's + * query domain, e.g. `$1::jsonb::eql_v4.query_text_eq`. + */ +export function queryV4(items: V4Item[]): V4Payload[] { + if (items.length === 0) return [] + return run({ query: items }).query +} + +/** + * Decrypt stored EQL v4 payloads. Throws when Stack Encrypt refuses one, + * which is what a payload from any other producer gets. + */ +export function decryptV4( + items: { type: V4Type; table: string; column: string; payload: unknown }[], +): Plaintext[] { + if (items.length === 0) return [] + return run({ decrypt: items }).decrypt +} + +/** + * Fail with a setup hint unless EQL v4 is installed next to the legacy + * version the suite migrates from. + */ +export async function requireEqlV4(pg: Client): Promise { + const res = await pg.query<{ installed: boolean }>( + "SELECT to_regnamespace('eql_v4') IS NOT NULL AS installed", + ) + if (!res.rows[0]?.installed) { + throw new Error( + 'EQL v4 is not installed. Run `mise run eql:v4:install` from languages/typescript/packages/protect-ffi.', + ) + } +} + +/** SQLSTATE `check_violation`: a payload of the wrong version for a domain. */ +export const CHECK_VIOLATION = '23514' diff --git a/packages/eql/Cargo.lock b/packages/eql/Cargo.lock index a71c1d4e3..818dc01f1 100644 --- a/packages/eql/Cargo.lock +++ b/packages/eql/Cargo.lock @@ -478,6 +478,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.4" @@ -1681,7 +1687,7 @@ dependencies = [ "hickory-proto", "idna", "ipnet", - "jni", + "jni 0.22.4", "rand 0.10.1", "thiserror 2.0.20", "tinyvec", @@ -1699,7 +1705,7 @@ dependencies = [ "data-encoding", "idna", "ipnet", - "jni", + "jni 0.22.4", "once_cell", "prefix-trie", "rand 0.10.1", @@ -1722,7 +1728,7 @@ dependencies = [ "hickory-proto", "ipconfig", "ipnet", - "jni", + "jni 0.22.4", "moka", "ndk-context", "once_cell", @@ -2126,6 +2132,22 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + [[package]] name = "jni" version = "0.22.4" @@ -2135,7 +2157,7 @@ dependencies = [ "cfg-if", "combine", "jni-macros", - "jni-sys", + "jni-sys 0.4.1", "log", "simd_cesu8", "thiserror 2.0.20", @@ -2156,6 +2178,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + [[package]] name = "jni-sys" version = "0.4.1" @@ -3323,7 +3354,7 @@ dependencies = [ "quinn", "rustls", "rustls-pki-types", - "rustls-platform-verifier", + "rustls-platform-verifier 0.7.0", "serde", "serde_json", "serde_urlencoded", @@ -3504,6 +3535,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni 0.21.1", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + [[package]] name = "rustls-platform-verifier" version = "0.7.0" @@ -3512,7 +3564,7 @@ checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", - "jni", + "jni 0.22.4", "log", "once_cell", "rustls", @@ -4134,10 +4186,15 @@ version = "0.44.0" dependencies = [ "aquamarine", "base64 0.22.1", + "bytes", "cts-common 0.43.0", "jsonwebtoken", "miette", "open", + "reqwest", + "rustls", + "rustls-platform-verifier 0.6.2", + "rustls-webpki", "serde", "serde_json", "serde_urlencoded", @@ -4172,6 +4229,7 @@ dependencies = [ "serde_json", "serdect", "sha2 0.11.0", + "stack-auth 0.44.0", "stack-encrypt-derive", "stack-kms", "thiserror 1.0.69", @@ -4205,9 +4263,12 @@ dependencies = [ "base64ct", "blake3", "futures", + "lazy_static", "miette", "opaque-debug", "recipher 0.3.1", + "reqwest", + "rustls", "serde", "serde_cbor", "serde_json", @@ -5737,6 +5798,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -5782,6 +5852,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -5830,6 +5915,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -5848,6 +5939,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -5866,6 +5963,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -5896,6 +5999,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -5914,6 +6023,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -5932,6 +6047,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -5950,6 +6071,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" diff --git a/packages/eql/tests/encryption/Cargo.toml b/packages/eql/tests/encryption/Cargo.toml index 787f19abc..fde5bad33 100644 --- a/packages/eql/tests/encryption/Cargo.toml +++ b/packages/eql/tests/encryption/Cargo.toml @@ -27,7 +27,10 @@ eql-bindings = { path = "../../crates/eql-bindings", features = ["targets", "chr eql-domains = { path = "../../crates/eql-domains" } # `dynamic`: `ValueFrame`, the runtime plaintext the target dispatch takes, # as eql-bindings names it. -stack-encrypt = { path = "../../../stack-encrypt", default-features = false, features = ["dynamic"] } +# `http`: `StackCipher::new()` over live ZeroKMS, for `examples/v4_codec.rs`, +# the codec the protect-ffi migration suites re-encrypt through. The tests +# here still build their ciphers over explicit fake key sources. +stack-encrypt = { path = "../../../stack-encrypt", default-features = false, features = ["dynamic", "http"] } # `plaintext::Sink`, for a plaintext type with its own `Encrypt` impl; and # chrono's and rust_decimal's own sources, which seal the leaf a date, # timestamp or decimal from any other producer seals. diff --git a/packages/eql/tests/encryption/examples/v4_codec.rs b/packages/eql/tests/encryption/examples/v4_codec.rs new file mode 100644 index 000000000..ddf599341 --- /dev/null +++ b/packages/eql/tests/encryption/examples/v4_codec.rs @@ -0,0 +1,192 @@ +//! A stdin/stdout EQL v4 codec over live ZeroKMS, for migration tests in other +//! languages. +//! +//! TypeScript cannot produce EQL v4 yet: the only v4 producer is Stack +//! Encrypt, reached from Rust and Go. The live migration suite in +//! `languages/typescript/packages/protect-ffi/integration-tests` (EQL v2 and +//! v3 to v4) decrypts legacy rows with protect-ffi and hands the plaintexts +//! here to re-encrypt them as v4. Every value goes through +//! `eql_bindings::encryption::targets`, the by-name dispatch the Go guest +//! uses, so the payloads are the ones a v4 SDK writes. +//! +//! Run from `packages/eql`: +//! +//! ```sh +//! cargo build --locked -p eql-encryption-tests --example v4_codec +//! echo '{"encrypt":[{"type":"TextEq","table":"users","column":"email","value":"a@b.c"}]}' \ +//! | target/debug/examples/v4_codec +//! ``` +//! +//! Credentials come from the environment (`CS_CLIENT_ACCESS_KEY`, +//! `CS_WORKSPACE_CRN`, `CS_CLIENT_ID`, `CS_CLIENT_KEY`) or the CLI profile, the +//! same way `StackCipher::new()` finds them everywhere else. +//! +//! The request is one JSON object with three optional lists: +//! +//! ```json +//! { +//! "encrypt": [{ "type": "TextEq", "table": "t", "column": "c", "value": "x" }], +//! "query": [{ "type": "IntegerOrd", "table": "t", "column": "c", "value": 7 }], +//! "decrypt": [{ "type": "TextEq", "table": "t", "column": "c", "payload": { … } }] +//! } +//! ``` +//! +//! The response has the same three keys, each a list in request order: +//! stored payloads, query operands (bind as the target's `query_sql_domain`) +//! and decrypted plaintexts. `type` is a target name from +//! `eql_bindings::encryption::targets::targets()`. A query uses the target's +//! first index, which for every scalar target is the one its operators need. +//! Plaintexts are JSON strings, booleans and numbers; the target's +//! `plaintext` kind decides how a number is read. + +use std::io::{Read, Write}; + +use eql_bindings::encryption::targets::{self, Opener}; +use serde::Deserialize; +use serde_json::{json, Value}; +use stack_encrypt::dynamic::ValueFrame; +use stack_encrypt::target::IndexSpec; +use stack_encrypt::{Decrypt, Encrypt, Label, StackCipher}; + +type Error = Box; + +#[derive(Deserialize, Default)] +#[serde(deny_unknown_fields)] +struct Request { + #[serde(default)] + encrypt: Vec, + #[serde(default)] + query: Vec, + #[serde(default)] + decrypt: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Item { + r#type: String, + table: String, + column: String, + value: Value, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Stored { + r#type: String, + table: String, + column: String, + payload: Value, +} + +/// The target's plaintext kind, e.g. `"int32"`, or an error naming the type. +fn plaintext_kind(name: &str) -> Result<&'static str, Error> { + let target = targets::target(name).ok_or_else(|| format!("no EQL v4 target named {name}"))?; + target + .plaintext + .ok_or_else(|| format!("{name} has no scalar plaintext kind").into()) +} + +fn label(table: &str, column: &str) -> Result { + Ok(Label::new([table, column])?) +} + +/// A JSON plaintext as the engine's runtime frame, read as `name`'s kind. +fn frame(name: &str, value: &Value) -> Result { + let kind = plaintext_kind(name)?; + let wrong = || format!("{name} takes a {kind} plaintext, not {value}"); + let frame = match kind { + "string" => ValueFrame::of(value.as_str().ok_or_else(wrong)?.to_owned()), + "boolean" | "bool" => ValueFrame::of(value.as_bool().ok_or_else(wrong)?), + "int16" => ValueFrame::of(i16::try_from(value.as_i64().ok_or_else(wrong)?)?), + "int32" => ValueFrame::of(i32::try_from(value.as_i64().ok_or_else(wrong)?)?), + "int64" => ValueFrame::of(value.as_i64().ok_or_else(wrong)?), + "float32" => ValueFrame::of(value.as_f64().ok_or_else(wrong)? as f32), + "float64" => ValueFrame::of(value.as_f64().ok_or_else(wrong)?), + other => { + return Err(format!("{name}: plaintext kind {other} is not supported here").into()) + } + }; + Ok(frame?) +} + +/// An opened frame as JSON, read as `name`'s kind. +fn plaintext(name: &str, opened: ValueFrame) -> Result { + let kind = plaintext_kind(name)?; + let value = match kind { + "string" => json!(opened.visit(String::sink())?), + "boolean" | "bool" => json!(opened.visit(bool::sink())?), + "int16" => json!(opened.visit(i16::sink())?), + "int32" => json!(opened.visit(i32::sink())?), + "int64" => json!(opened.visit(i64::sink())?), + "float32" => json!(opened.visit(f32::sink())?), + "float64" => json!(opened.visit(f64::sink())?), + other => { + return Err(format!("{name}: plaintext kind {other} is not supported here").into()) + } + }; + Ok(value) +} + +fn index(name: &str) -> Result { + let target = targets::target(name).ok_or_else(|| format!("no EQL v4 target named {name}"))?; + target + .indexes + .first() + .and_then(|key| IndexSpec::parse(key)) + .ok_or_else(|| format!("{name} is storage-only and answers no query").into()) +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Error> { + let mut input = String::new(); + std::io::stdin().read_to_string(&mut input)?; + let request: Request = serde_json::from_str(&input)?; + + let cipher = StackCipher::new().await?; + let keyset = cipher.default_keyset(); + + let mut encrypted = Vec::with_capacity(request.encrypt.len()); + for item in &request.encrypt { + let bytes = targets::encrypt( + &item.r#type, + &keyset, + &label(&item.table, &item.column)?, + frame(&item.r#type, &item.value)?, + )? + .await?; + encrypted.push(serde_json::from_slice::(&bytes)?); + } + + let mut queries = Vec::with_capacity(request.query.len()); + for item in &request.query { + let bytes = targets::query( + &item.r#type, + &index(&item.r#type)?, + &keyset, + &label(&item.table, &item.column)?, + frame(&item.r#type, &item.value)?, + )? + .await?; + queries.push(serde_json::from_slice::(&bytes)?); + } + + let mut decrypted = Vec::with_capacity(request.decrypt.len()); + for item in &request.decrypt { + let stored = serde_json::to_vec(&item.payload)?; + let opened = targets::decrypt( + &item.r#type, + Opener::Client(&cipher), + &label(&item.table, &item.column)?, + &stored, + )? + .await?; + decrypted.push(plaintext(&item.r#type, opened)?); + } + + let response = json!({ "encrypt": encrypted, "query": queries, "decrypt": decrypted }); + let mut stdout = std::io::stdout().lock(); + serde_json::to_writer(&mut stdout, &response)?; + stdout.write_all(b"\n")?; + Ok(()) +} diff --git a/scripts/__tests__/eql-v4-suite-triggers.test.mjs b/scripts/__tests__/eql-v4-suite-triggers.test.mjs new file mode 100644 index 000000000..3d6924a91 --- /dev/null +++ b/scripts/__tests__/eql-v4-suite-triggers.test.mjs @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest' +import { eqlV4SuiteFilterEntries } from './lib/eql-v4-suite-inputs.mjs' +import { filterCovers } from './lib/paths-filter.mjs' +import { REPO_ROOT } from './lib/repo-root.mjs' +import { readWorkflow } from './lib/workflows.mjs' + +/** + * The protect-ffi migration suites (`migrate-v*-to-v4.test.ts`) re-encrypt + * through a Rust codec built over `eql-bindings`' v4 modules, and query SQL + * rendered by `eql-codegen`. No other workflow runs those suites: + * `test-eql.yml` runs the Rust and SQL suites, not this one. So + * `integration-protect-ffi.yml` has to start when any of those files change, + * or a change to v4 target dispatch merges with the end-to-end migration + * coverage skipped. The entries come from `lib/eql-v4-suite-inputs.mjs`. + */ +const WORKFLOW = '.github/workflows/integration-protect-ffi.yml' + +/** A literal or `/**`-tailed filter entry covering the input pattern. */ +function entryCovers(entry, input) { + const literal = (p) => { + const segments = p.split('/') + const at = segments.findIndex((segment) => /[*?[]/.test(segment)) + return (at === -1 ? segments : segments.slice(0, at)).join('/') + } + const prefix = literal(entry) + const tail = entry.slice(prefix.length).replace(/^\//, '') + if (tail !== '' && tail !== '**') return false + const target = literal(input) + return target === prefix || target.startsWith(`${prefix}/`) +} + +describe('the EQL v4 migration suites trigger on their v4 inputs', () => { + const { features, entries } = eqlV4SuiteFilterEntries(REPO_ROOT) + + it('derives the v4 modules and the codegen crates', () => { + // An empty or shrunken derivation would pass every check below. + expect([...features]).toContain('stack-encrypt') + expect(entries).toEqual( + expect.arrayContaining([ + 'packages/eql/crates/eql-bindings/src/v4/**', + 'packages/eql/crates/eql-bindings/src/encryption/**', + 'packages/eql/crates/eql-codegen/src/**', + ]), + ) + }) + + const { on } = readWorkflow(WORKFLOW) + for (const event of ['push', 'pull_request']) { + it(`${WORKFLOW} (${event}) triggers on every v4 codec and codegen input`, () => { + const paths = on?.[event]?.paths ?? [] + const uncovered = entries.filter( + (entry) => !filterCovers(paths, entry, entryCovers), + ) + expect( + uncovered, + `${WORKFLOW} runs the EQL v4 migration suites, but its \`${event}\` \`paths:\` filter does not list these inputs of the v4 codec or the v4 SQL build. Add each to BOTH copies of the filter.`, + ).toEqual([]) + }) + } +}) diff --git a/scripts/__tests__/lib/eql-v4-suite-inputs.mjs b/scripts/__tests__/lib/eql-v4-suite-inputs.mjs new file mode 100644 index 000000000..0eb3f6667 --- /dev/null +++ b/scripts/__tests__/lib/eql-v4-suite-inputs.mjs @@ -0,0 +1,120 @@ +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { compiledCrateFiles } from './rust-compiled-files.mjs' + +/** + * The `paths:` filter entries under `packages/eql/crates/` that the protect-ffi + * migration suites (`integration-tests/tests/migrate-v*-to-v4.test.ts`) need, + * derived from the two builds those suites run rather than listed. + * + * - `mise run eql:v4:codec` builds `packages/eql/tests/encryption/examples/v4_codec.rs`, + * which compiles `eql-bindings` with the features that crate's manifest asks + * for. Those turn on the `v4`, `encryption` and `encoding` modules the native + * binding never compiles. + * - `mise run eql:v4:install` runs `tasks/build.sh`, which renders the v4 SQL + * bundle through `eql-codegen` (and `eql-domains`). Its `#MISE sources` line + * is the list of what that render reads. + * + * `integration-protect-ffi.yml` must trigger on every entry returned here, or a + * change to the v4 codec or the SQL it is checked against starts no suite that + * runs it. `wasm-build-inputs-paths-filter.test.mjs` forbids every other + * workflow from naming these paths, and allows exactly these for that one. + * + * Each compiled file becomes the smallest entry a reader would write: a file + * at the crate root or directly under `src/` is named itself, anything deeper + * becomes its top module directory with `/**` (`src/v4/**`, `sql/**`). + */ +const EQL_ROOT = 'packages/eql' +const EQL_CRATES = `${EQL_ROOT}/crates` +const CODEC_MANIFEST = `${EQL_ROOT}/tests/encryption/Cargo.toml` +const BINDINGS = `${EQL_CRATES}/eql-bindings` +const BUILD_SCRIPT = `${EQL_ROOT}/tasks/build.sh` + +/** The `features = [...]` the codec crate requests of `eql-bindings`. */ +function requestedBindingsFeatures(repoRoot) { + const text = readFileSync(resolve(repoRoot, CODEC_MANIFEST), 'utf8') + const line = text.split('\n').find((l) => /^\s*eql-bindings\s*=\s*\{/.test(l)) + if (!line) + throw new Error(`${CODEC_MANIFEST} no longer depends on eql-bindings`) + const list = /features\s*=\s*\[([^\]]*)\]/.exec(line) + if (!list) return [] + return [...list[1].matchAll(/"([^"]+)"/g)].map((m) => m[1]) +} + +/** `eql-bindings`' `[features]` table: name -> the entries it enables. */ +function bindingsFeatureTable(repoRoot) { + const text = readFileSync(resolve(repoRoot, `${BINDINGS}/Cargo.toml`), 'utf8') + const start = text.indexOf('\n[features]') + if (start === -1) return new Map() + const rest = text.slice(start + '\n[features]'.length) + const end = rest.search(/\n\[[^\]]+\]/) + const body = (end === -1 ? rest : rest.slice(0, end)).replace(/#.*$/gm, '') + const table = new Map() + for (const m of body.matchAll(/^\s*([\w-]+)\s*=\s*\[([^\]]*)\]/gm)) { + table.set( + m[1], + [...m[2].matchAll(/"([^"]+)"/g)].map((e) => e[1]), + ) + } + return table +} + +/** The requested features plus every crate-local feature they turn on. */ +function enabledFeatures(requested, table) { + const enabled = new Set() + const queue = [...requested] + while (queue.length > 0) { + const name = queue.shift() + if (enabled.has(name)) continue + enabled.add(name) + for (const entry of table.get(name) ?? []) { + if (!entry.includes('/') && !entry.startsWith('dep:')) queue.push(entry) + } + } + return enabled +} + +/** The smallest filter entry covering one compiled file of a crate. */ +function entryForCompiledFile(crateDir, file) { + const rel = file.slice(crateDir.length + 1).split('/') + if (rel.length === 1) return file + if (rel[0] === 'src') { + return rel.length === 2 ? file : `${crateDir}/src/${rel[1]}/**` + } + return `${crateDir}/${rel[0]}/**` +} + +/** `tasks/build.sh`'s `#MISE sources` that live under `crates/`. */ +function codegenSources(repoRoot) { + const text = readFileSync(resolve(repoRoot, BUILD_SCRIPT), 'utf8') + const line = /^#MISE sources=(\[.*\])\s*$/m.exec(text) + if (!line) throw new Error(`${BUILD_SCRIPT} no longer declares #MISE sources`) + return JSON.parse(line[1]) + .filter((source) => source.startsWith('crates/')) + .map((source) => `${EQL_ROOT}/${source}`) +} + +/** A glob reduced to its directory with `/**`, or a literal path as is. */ +function entryForSource(source) { + const segments = source.split('/') + const at = segments.findIndex((segment) => /[*?[]/.test(segment)) + return at === -1 ? source : `${segments.slice(0, at).join('/')}/**` +} + +export function eqlV4SuiteFilterEntries(repoRoot) { + const features = enabledFeatures( + requestedBindingsFeatures(repoRoot), + bindingsFeatureTable(repoRoot), + ) + const walk = compiledCrateFiles(repoRoot, BINDINGS, features) + if (walk.unresolved.length > 0) { + throw new Error( + `Could not resolve every eql-bindings file the v4 codec compiles:\n${walk.unresolved.join('\n')}`, + ) + } + const entries = new Set([ + ...walk.files.map((file) => entryForCompiledFile(BINDINGS, file)), + ...codegenSources(repoRoot).map(entryForSource), + ]) + return { features, entries: [...entries].sort() } +} diff --git a/scripts/__tests__/tests-workflow-gate.test.mjs b/scripts/__tests__/tests-workflow-gate.test.mjs index 4526c4b8b..8917fa990 100644 --- a/scripts/__tests__/tests-workflow-gate.test.mjs +++ b/scripts/__tests__/tests-workflow-gate.test.mjs @@ -165,6 +165,10 @@ const READS_EXCLUDED = new Map([ 'languages/typescript/packages/protect-ffi/src/integrationSuiteCi.test.ts', 'Reads `.github/workflows/`. protect-ffi’s suite also runs in `repo-checks`.', ], + [ + 'languages/typescript/packages/protect-ffi/integration-tests/tests/v4-codec.ts', + 'Reads the built codec under `packages/eql/target`. The live suite that imports it runs only in `integration-protect-ffi.yml`, which is not gated and builds the codec itself, never in `pnpm test`.', + ], [ 'packages/eql/packages/eql/scripts/sync-generated.mjs', 'Reads `crates/eql-bindings/bindings` and `schema`. Run by `check:generated` and the release, never by `pnpm test`.', diff --git a/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs b/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs index 457d59ef0..e25e65c65 100644 --- a/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs +++ b/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs @@ -2,6 +2,7 @@ import { existsSync, readFileSync } from 'node:fs' import { resolve } from 'node:path' import yaml from 'js-yaml' import { describe, expect, it } from 'vitest' +import { eqlV4SuiteFilterEntries } from './lib/eql-v4-suite-inputs.mjs' import { filterCovers } from './lib/paths-filter.mjs' import { REPO_ROOT } from './lib/repo-root.mjs' import { readWorkflow, workflowFiles } from './lib/workflows.mjs' @@ -548,12 +549,23 @@ describe('the FFI cache keys and the filters that trigger them agree', () => { // compile. The key now names the compiled files (held to them by // ffi-binding-action.test.mjs); a filter entry under the EQL crates must be // one of its entries, not a wider glob. + // + // One workflow builds more than the binding: integration-protect-ffi.yml + // runs the EQL v4 migration suites, which compile eql-bindings' v4 modules + // into a codec and render the v4 SQL through eql-codegen. It may also name + // exactly the entries lib/eql-v4-suite-inputs.mjs derives for those builds + // (eql-v4-suite-triggers.test.mjs requires them), and nothing wider. const EQL_CRATES = 'packages/eql/crates' const nativeInputs = new Set(buildInputs(NATIVE_CACHE_STEP)) + const V4_SUITE_WORKFLOW = '.github/workflows/integration-protect-ffi.yml' + const v4SuiteEntries = new Set(eqlV4SuiteFilterEntries(REPO_ROOT).entries) for (const { relPath, event, paths } of KEYS[0].workflows.flatMap( filteredEvents, )) { it(`${relPath} (${event}) names no EQL crate path the binding does not compile`, () => { + const allowed = (entry) => + nativeInputs.has(entry) || + (relPath === V4_SUITE_WORKFLOW && v4SuiteEntries.has(entry)) const wider = paths.filter( (entry) => !entry.startsWith('!') && @@ -561,7 +573,7 @@ describe('the FFI cache keys and the filters that trigger them agree', () => { literalPrefix(entry).startsWith(`${EQL_CRATES}/`) || EQL_CRATES.startsWith(`${literalPrefix(entry)}/`)) && literalPrefix(entry) !== '' && - !nativeInputs.has(entry), + !allowed(entry), ) expect( wider,