From 4968a46aab6eeb56e612cd40c24486152c1b037d Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Mon, 2 Nov 2026 12:01:54 +1100 Subject: [PATCH 1/2] test(protect-ffi): migrate live EQL v3 data to v4 end to end Nothing showed that data written by Stack 1.x (EQL v3) can move to EQL v4 and keep answering the same queries. The two versions come from different producers (cipherstash-client and Stack Encrypt), use different ciphertext formats and derive search terms differently, so the migration has to decrypt and re-encrypt every value. No test did that. The new suite runs the customer-shaped migration in one table, against live ZeroKMS and Postgres with EQL v3 and v4 installed side by side. It writes v3 with protect-ffi (the engine Stack 1.x encrypts through), adds a v4 column beside each v3 one, backfills by decrypting and re-encrypting, then checks that equality, ORE range, ORE and OPE ordering, and match queries return the same rows from both columns. It also checks that every v4 value decrypts to the original plaintext, and that the v4 columns still answer after the v3 columns are dropped and the v4 ones renamed into their place. Three hazards are pinned: a v3 payload cast into a v4 column fails the domain CHECK, relabelling `v` to 4 gives a value Stack Encrypt refuses to open, and a v3 query operand against a v4 column silently matches nothing. No TypeScript SDK writes v4 yet, so the suite re-encrypts through a small Rust stdin/stdout codec, an example in the EQL encryption test crate, that dispatches by EQL type name through eql_bindings::encryption::targets over live ZeroKMS. That needs stack-encrypt's `http` feature in that crate. The suite's `mise run setup` now also builds the v4 bundle from the in-tree EQL source and builds the codec, and the CI workflow caches the EQL Rust build and watches the inputs those two steps read. --- .github/workflows/integration-protect-ffi.yml | 43 +- AGENTS.md | 18 +- .../integration-tests/package.json | 2 +- .../protect-ffi/integration-tests/tasks.toml | 26 + .../tests/migrate-v3-to-v4.test.ts | 465 ++++++++++++++++++ .../integration-tests/tests/v4-codec.ts | 126 +++++ packages/eql/Cargo.lock | 139 +++++- packages/eql/tests/encryption/Cargo.toml | 5 +- .../eql/tests/encryption/examples/v4_codec.rs | 192 ++++++++ .../__tests__/tests-workflow-gate.test.mjs | 4 + 10 files changed, 1004 insertions(+), 16 deletions(-) create mode 100644 languages/typescript/packages/protect-ffi/integration-tests/tests/migrate-v3-to-v4.test.ts create mode 100644 languages/typescript/packages/protect-ffi/integration-tests/tests/v4-codec.ts create mode 100644 packages/eql/tests/encryption/examples/v4_codec.rs diff --git a/.github/workflows/integration-protect-ffi.yml b/.github/workflows/integration-protect-ffi.yml index 909f5bca8..e7992900e 100644 --- a/.github/workflows/integration-protect-ffi.yml +++ b/.github/workflows/integration-protect-ffi.yml @@ -97,6 +97,22 @@ on: - 'packages/eql/crates/eql-bindings/src/v3/**' - 'packages/eql/crates/eql-bindings/sql/**' - 'packages/eql/Cargo.toml' + # EQL v4 and the codec the migration suites (`tests/migrate-v*-to-v4.test.ts`) + # re-encrypt through. `eql:v4:install` builds the v4 bundle from the SQL + # source with `tasks/build.sh` (which runs eql-codegen), and `eql:v4:codec` + # builds `tests/encryption/examples/v4_codec.rs` over eql-bindings' v4 + # targets, stack-encrypt and stack-kms. No published package carries any of + # it, so these paths are the only inputs that change what those suites run. + # `packages/eql/crates/**` (eql-codegen, eql-bindings' v4 module) is left + # out on purpose: wasm-build-inputs-paths-filter.test.mjs holds this + # filter to the crate files the native binding compiles, and test-eql.yml + # already runs on every change there. + - 'packages/eql/Cargo.lock' + - 'packages/eql/src/**' + - 'packages/eql/tasks/**' + - 'packages/eql/tests/encryption/**' + - 'packages/stack-encrypt/**' + - 'packages/stack-kms/**' - '.github/workflows/integration-protect-ffi.yml' - '.github/actions/build-ffi-binding/**' - '.github/actions/build-auth-binding/**' @@ -146,6 +162,12 @@ on: - 'packages/eql/crates/eql-bindings/src/v3/**' - 'packages/eql/crates/eql-bindings/sql/**' - 'packages/eql/Cargo.toml' + - 'packages/eql/Cargo.lock' + - 'packages/eql/src/**' + - 'packages/eql/tasks/**' + - 'packages/eql/tests/encryption/**' + - 'packages/stack-encrypt/**' + - 'packages/stack-kms/**' - '.github/workflows/integration-protect-ffi.yml' - '.github/actions/build-ffi-binding/**' - '.github/actions/build-auth-binding/**' @@ -361,11 +383,24 @@ jobs: # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs cache_key_prefix: mise-v1-ubuntu-2404 - # One task, four things the suite cannot run without (see + # The EQL workspace's Rust, which `mise run setup` compiles for the + # migration suites: eql-codegen (the v4 bundle) and the v4 codec. Without + # a cache that is a cold build of stack-encrypt and its HTTP stack on every + # run. Keyed apart from the `sqlx-tests` cache test-eql.yml saves, which + # carries the SQLx graph this job never builds. + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + workspaces: packages/eql + shared-key: eql-v4-codec + + # One task, six things the suite cannot run without (see # integration-tests/tasks.toml): `pnpm install --frozen-lockfile` (a # no-op here — the step above already ran it), `docker compose up` for - # Postgres, the EQL **v2** bundle from a GitHub release, and EQL **v3** - # built from the in-tree `@cipherstash/eql`. + # Postgres, the EQL **v2** bundle from a GitHub release, EQL **v3** + # built from the in-tree `@cipherstash/eql`, EQL **v4** built from the + # in-tree EQL source, and the Rust v4 codec. The last two serve + # `tests/migrate-v*-to-v4.test.ts`: Stack Encrypt is the only v4 + # producer, and no TypeScript SDK reaches it yet. # # BOTH EQL versions are required and nothing else in this repo installs # v2: `tests/postgres.test.ts` creates `eql_v2_encrypted` columns and @@ -381,7 +416,7 @@ jobs: # `integration-db`'s per-job project names deliberately do not produce. # The fixed host port (5436) that comes with it cannot collide with the # shared stacks, which publish on ephemeral ports. - - name: Provision the suite (Postgres, EQL v2 + v3) + - name: Provision the suite (Postgres, EQL v2 + v3 + v4, v4 codec) working-directory: languages/typescript/packages/protect-ffi run: mise run setup diff --git a/AGENTS.md b/AGENTS.md index 73a0a56c3..9e7fbf3bc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -225,10 +225,10 @@ stays optional for everyone else. `languages/typescript/packages/protect-ffi/integration-tests/` is 19 files of **live** coverage — encrypt/decrypt, lock context, keysets, JS auth strategies, JSON SteVec, -Postgres (EQL v2 *and* v3), and a WASM round trip — and it is the only place -several of those paths are exercised at all. It needs three things a normal -`pnpm test` does not have: **Docker**, **CipherStash credentials**, and **both -EQL versions installed** in the database. +Postgres (EQL v2 *and* v3), EQL v3 -> v4 migration, and a WASM round trip — and +it is the only place several of those paths are exercised at all. It needs +three things a normal `pnpm test` does not have: **Docker**, **CipherStash +credentials**, and **EQL v2, v3 and v4 installed** in the database. - **It is a pnpm workspace member, and `pnpm test` must never reach it.** Named literally in `pnpm-workspace.yaml` (the `languages/typescript/packages/*` glob is one level deep and @@ -290,6 +290,16 @@ EQL versions installed** in the database. `eql_v2.add_encrypted_constraint`) while `tests/postgres-v3.test.ts` needs the `eql_v3_*` domains. Skip either and half the suite fails on missing SQL functions. +- **The migration suites need EQL v4 and a Rust codec.** + `tests/migrate-v*-to-v4.test.ts` decrypt legacy rows with protect-ffi and + re-encrypt them as v4, and Stack Encrypt is the only v4 producer, so they call + `packages/eql/tests/encryption/examples/v4_codec.rs` through + `tests/v4-codec.ts`. `eql:v4:install` builds the v4 bundle from the in-tree + EQL source (`bash tasks/build.sh`, called directly so it needs cargo and not + `packages/eql`'s mise config) and `eql:v4:codec` builds the codec; both are + part of `mise run setup`. The CI filter cannot name `packages/eql/crates/**` + (`wasm-build-inputs-paths-filter.test.mjs`), so an edit to eql-codegen or + eql-bindings' v4 module reaches this suite only through `test-eql.yml`. - **`eql:v3:install` builds `@cipherstash/eql` first, and has to.** The task reads the bundle through `@cipherstash/eql/sql`, which the package's `exports` map resolves to `dist/sql.js` — a tsup output. That was free while `npm ci` diff --git a/languages/typescript/packages/protect-ffi/integration-tests/package.json b/languages/typescript/packages/protect-ffi/integration-tests/package.json index 20010bb06..41ef13656 100644 --- a/languages/typescript/packages/protect-ffi/integration-tests/package.json +++ b/languages/typescript/packages/protect-ffi/integration-tests/package.json @@ -2,7 +2,7 @@ "name": "@cipherstash/ffi-integration-tests", "version": "1.0.2", "private": true, - "description": "Live integration coverage for @cipherstash/protect-ffi — real ZeroKMS, real Postgres, EQL v2 and v3", + "description": "Live integration coverage for @cipherstash/protect-ffi — real ZeroKMS, real Postgres, EQL v2, v3 and v4", "license": "MIT", "scripts": { "typecheck": "tsc", diff --git a/languages/typescript/packages/protect-ffi/integration-tests/tasks.toml b/languages/typescript/packages/protect-ffi/integration-tests/tasks.toml index 0ba49f62e..4ac0ae873 100644 --- a/languages/typescript/packages/protect-ffi/integration-tests/tasks.toml +++ b/languages/typescript/packages/protect-ffi/integration-tests/tasks.toml @@ -45,6 +45,30 @@ fi docker exec -i protect-ffi-postgres psql postgresql://cipherstash:password@postgres:5432/cipherstash -f- < "$tmp" """ +# EQL v4, for the migration suites (`tests/migrate-v*-to-v4.test.ts`). No npm +# package ships the v4 bundle yet, so it is built from the in-tree EQL source: +# `tasks/build.sh` is what `mise run build` runs in `packages/eql`, called +# directly so this task needs only cargo, not that directory's mise config (which +# a runner has not trusted). It writes both bundles; only v4 is installed here, +# v3 comes from `eql:v3:install` above. +["eql:v4:install"] +description = "Build EQL v4 from the in-tree source and install it beside v2 and v3" +dir = "{{ config_root }}/../../../../packages/eql" +run = """ +set -e +bash tasks/build.sh +test -s release/cipherstash-encrypt-v4.sql +docker exec -i protect-ffi-postgres psql -v ON_ERROR_STOP=1 -q postgresql://cipherstash:password@postgres:5432/cipherstash -f- < release/cipherstash-encrypt-v4.sql +""" + +# The Rust codec the migration suites re-encrypt through: Stack Encrypt is the +# only EQL v4 producer and no TypeScript SDK reaches it yet. See +# `tests/v4-codec.ts` and `packages/eql/tests/encryption/examples/v4_codec.rs`. +["eql:v4:codec"] +description = "Build the EQL v4 codec the migration suites call" +dir = "{{ config_root }}/../../../../packages/eql" +run = "cargo build --locked -p eql-encryption-tests --example v4_codec" + # `pnpm install`, not `npm ci`: this directory is a pnpm workspace member as of # CIP-3744, so its dependencies come from the repo lockfile and its # `@cipherstash/protect-ffi` and `@cipherstash/eql` resolve to the packages in @@ -62,6 +86,8 @@ run = [ "mise run eql:download", "mise run eql:install", "mise run eql:v3:install", + "mise run eql:v4:install", + "mise run eql:v4:codec", ] ["test:integration"] diff --git a/languages/typescript/packages/protect-ffi/integration-tests/tests/migrate-v3-to-v4.test.ts b/languages/typescript/packages/protect-ffi/integration-tests/tests/migrate-v3-to-v4.test.ts new file mode 100644 index 000000000..97476333c --- /dev/null +++ b/languages/typescript/packages/protect-ffi/integration-tests/tests/migrate-v3-to-v4.test.ts @@ -0,0 +1,465 @@ +import 'dotenv/config' +import { + decryptBulk, + type EncryptConfig, + type EncryptedPayload, + encryptBulk, + encryptQuery, + newClient, +} from '@cipherstash/protect-ffi' +import { Client } from 'pg' +import { beforeAll, describe, expect, test } from 'vitest' +import { + CHECK_VIOLATION, + decryptV4, + encryptV4, + type Plaintext, + queryV4, + requireEqlV4, + type V4Type, +} from './v4-codec' + +// EQL v3 -> v4 migration, end to end, against live ZeroKMS and Postgres. +// +// v3 is what Stack 1.x writes: protect-ffi in `eqlVersion: 3` is the engine +// `@cipherstash/stack` encrypts through. v4 is what Stack Encrypt writes. The +// two use different ciphertext formats and derive their search terms +// differently, so a migration cannot be a cast or an edit of `v`: every value +// is decrypted with the v3 client and encrypted again as v4. +// +// The shape is the one a customer runs, in one table: +// +// 1. the v3 columns hold live data; +// 2. add a v4 column beside each one; +// 3. backfill: read the v3 payloads, decrypt them, encrypt the plaintexts as +// v4, write them to the new columns; +// 4. switch queries to the v4 columns and v4 operands; +// 5. drop the v3 columns and rename the v4 ones into their place. +// +// The v4 values keep the v3 column's encryption context (`customers/email`, +// not `customers/email_v4`): the context is bound into the ciphertext and the +// search terms, so naming the logical column is what lets step 5 rename +// without re-encrypting again. +// +// Requires EQL v3 and v4 both installed (`mise run setup` installs v2, v3 and +// v4) and the v4 codec built (`mise run eql:v4:codec`); see ./v4-codec.ts. + +const TABLE = 'customers' + +const encryptConfig: EncryptConfig = { + v: 1, + tables: { + [TABLE]: { + email: { cast_as: 'text', indexes: { unique: {} } }, + age: { cast_as: 'int', indexes: { ore: {} } }, + rank: { cast_as: 'int', indexes: { ope: {} } }, + bio: { cast_as: 'text', indexes: { match: {} } }, + active: { cast_as: 'boolean', indexes: {} }, + }, + }, +} + +type Customer = { + email: string + age: number + rank: number + bio: string + active: boolean +} + +const customers: Customer[] = [ + { + email: 'ada@example.com', + age: 36, + rank: 3, + bio: 'analytical engine notes', + active: true, + }, + { + email: 'grace@example.com', + age: 85, + rank: 1, + bio: 'compiler pioneer and admiral', + active: true, + }, + { + email: 'alan@example.com', + age: 41, + rank: 2, + bio: 'computable numbers and machines', + active: false, + }, + { + email: 'edsger@example.com', + age: 72, + rank: 5, + bio: 'structured programming advocate', + active: false, + }, + { + email: 'barbara@example.com', + age: 29, + rank: 4, + bio: 'abstraction and programming methodology', + active: true, + }, +] + +type Column = keyof Customer + +/** Each column's v3 domain and the v4 type it migrates to, capability for capability. */ +const columns: Record = { + email: { + v3: 'public.eql_v3_text_eq', + v4: 'TextEq', + v4Domain: 'public.eql_v4_text_eq', + }, + age: { + v3: 'public.eql_v3_integer_ord_ore', + v4: 'IntegerOrdOre', + v4Domain: 'public.eql_v4_integer_ord_ore', + }, + rank: { + v3: 'public.eql_v3_integer_ord_ope', + v4: 'IntegerOrdOpe', + v4Domain: 'public.eql_v4_integer_ord_ope', + }, + bio: { + v3: 'public.eql_v3_text_match', + v4: 'TextMatch', + v4Domain: 'public.eql_v4_text_match', + }, + active: { + v3: 'public.eql_v3_boolean', + v4: 'Boolean', + v4Domain: 'public.eql_v4_boolean', + }, +} + +const columnNames = Object.keys(columns) as Column[] + +/** The emails of `rows` in id order: the oracle every query is compared to. */ +const emails = (rows: Customer[]) => rows.map((row) => row.email) + +describe('EQL v3 -> v4 migration', () => { + const pg = new Client() + let v3Client: Awaited> + + /** Ids the query returns, read as emails through the plaintext oracle. */ + async function emailsWhere(sql: string, params: unknown[] = []) { + const res = await pg.query<{ id: number }>( + `SELECT id FROM ${TABLE} ${sql}`, + params, + ) + return res.rows.map((row) => customers[row.id - 1].email) + } + + async function v3Operand( + column: Column, + plaintext: Plaintext, + indexType: 'unique' | 'ore' | 'ope' | 'match', + ) { + return encryptQuery(v3Client, { + plaintext, + column, + table: TABLE, + indexType, + }) + } + + function v4Operand(column: Column, value: Plaintext) { + const [operand] = queryV4([ + { type: columns[column].v4, table: TABLE, column, value }, + ]) + return operand + } + + beforeAll(async () => { + v3Client = await newClient({ encryptConfig, eqlVersion: 3 }) + await pg.connect() + await requireEqlV4(pg) + + // Step 1: the table as Stack 1.x left it — v3 columns only. + await pg.query(`DROP TABLE IF EXISTS ${TABLE}`) + await pg.query(` + CREATE TABLE ${TABLE} ( + id integer PRIMARY KEY, + ${columnNames.map((c) => `${c} ${columns[c].v3}`).join(',\n ')} + ) + `) + + const encrypted = await encryptBulk(v3Client, { + plaintexts: customers.flatMap((customer) => + columnNames.map((column) => ({ + plaintext: customer[column], + column, + table: TABLE, + })), + ), + }) + for (const index of customers.keys()) { + const row = encrypted.slice( + index * columnNames.length, + (index + 1) * columnNames.length, + ) + await pg.query( + `INSERT INTO ${TABLE} (id, ${columnNames.join(', ')}) + VALUES ($1, ${columnNames.map((_, i) => `$${i + 2}::jsonb`).join(', ')})`, + [index + 1, ...row], + ) + } + + return async () => { + await pg.query(`DROP TABLE IF EXISTS ${TABLE}`) + await pg.end() + } + }) + + test('the v3 table answers the queries the migration must preserve', async () => { + const all = await pg.query<{ v: number }>( + `SELECT DISTINCT (email::jsonb ->> 'v')::int AS v FROM ${TABLE}`, + ) + expect(all.rows).toEqual([{ v: 3 }]) + + expect( + await emailsWhere('WHERE email = $1::jsonb::eql_v3.query_text_eq', [ + await v3Operand('email', 'alan@example.com', 'unique'), + ]), + ).toEqual(['alan@example.com']) + + expect( + await emailsWhere( + 'WHERE age > $1::jsonb::eql_v3.query_integer_ord_ore ORDER BY id', + [await v3Operand('age', 40, 'ore')], + ), + ).toEqual(emails(customers.filter((c) => c.age > 40))) + + expect( + await emailsWhere( + 'WHERE bio @@ $1::jsonb::eql_v3.query_text_match ORDER BY id', + [await v3Operand('bio', 'programming', 'match')], + ), + ).toEqual(emails(customers.filter((c) => c.bio.includes('programming')))) + }) + + test('a v3 payload cannot be cast into a v4 column', async () => { + await pg.query( + `ALTER TABLE ${TABLE} ADD COLUMN probe ${columns.email.v4Domain}`, + ) + try { + await expect( + pg.query( + `UPDATE ${TABLE} SET probe = email::jsonb::${columns.email.v4Domain}`, + ), + ).rejects.toMatchObject({ code: CHECK_VIOLATION }) + } finally { + await pg.query(`ALTER TABLE ${TABLE} DROP COLUMN probe`) + } + }) + + test('relabelling a v3 payload as v4 does not make it a v4 value', async () => { + const res = await pg.query<{ email: EncryptedPayload }>( + `SELECT email::jsonb AS email FROM ${TABLE} WHERE id = 3`, + ) + const relabelled = { + ...(res.rows[0].email as { hm: string }), + v: 4, + } + + // Stack Encrypt refuses to open a cipherstash-client ciphertext. + expect(() => + decryptV4([ + { + type: 'TextEq', + table: TABLE, + column: 'email', + payload: relabelled, + }, + ]), + ).toThrowError() + + // And its search term was derived by the other producer, so a v4 query for + // the same plaintext does not match it. + const [v4Query] = queryV4([ + { + type: 'TextEq', + table: TABLE, + column: 'email', + value: 'alan@example.com', + }, + ]) + expect(v4Query.hm).not.toEqual(relabelled.hm) + }) + + test('backfill: decrypt every v3 value and re-encrypt it as v4 beside it', async () => { + // Step 2: one v4 column per v3 column. + for (const column of columnNames) { + await pg.query( + `ALTER TABLE ${TABLE} ADD COLUMN ${column}_v4 ${columns[column].v4Domain}`, + ) + } + + // Step 3: read, decrypt with the v3 client, re-encrypt as v4, write back. + const res = await pg.query< + Record & { id: number } + >( + `SELECT id, ${columnNames.map((c) => `${c}::jsonb AS ${c}`).join(', ')} + FROM ${TABLE} ORDER BY id`, + ) + const plaintexts = await decryptBulk(v3Client, { + ciphertexts: res.rows.flatMap((row) => + columnNames.map((column) => ({ ciphertext: row[column] })), + ), + }) + const v4 = encryptV4( + res.rows.flatMap((_, r) => + columnNames.map((column, c) => ({ + type: columns[column].v4, + table: TABLE, + column, + value: plaintexts[r * columnNames.length + c] as Plaintext, + })), + ), + ) + for (const [r, row] of res.rows.entries()) { + const values = v4.slice( + r * columnNames.length, + (r + 1) * columnNames.length, + ) + await pg.query( + `UPDATE ${TABLE} SET ${columnNames + .map((c, i) => `${c}_v4 = $${i + 2}::jsonb`) + .join(', ')} WHERE id = $1`, + [row.id, ...values], + ) + } + + const versions = await pg.query<{ v: number; n: number }>( + `SELECT (email_v4::jsonb ->> 'v')::int AS v, count(*)::int AS n FROM ${TABLE} GROUP BY 1`, + ) + expect(versions.rows).toEqual([{ v: 4, n: customers.length }]) + }) + + test('every migrated value decrypts as v4 to the original plaintext', async () => { + const res = await pg.query & { id: number }>( + `SELECT id, ${columnNames.map((c) => `${c}_v4::jsonb AS ${c}`).join(', ')} + FROM ${TABLE} ORDER BY id`, + ) + const opened = decryptV4( + res.rows.flatMap((row) => + columnNames.map((column) => ({ + type: columns[column].v4, + table: TABLE, + column, + payload: row[column], + })), + ), + ) + expect(opened).toEqual( + customers.flatMap((customer) => columnNames.map((c) => customer[c])), + ) + }) + + test('equality: the v4 column finds the row the v3 column found', async () => { + const v3 = await emailsWhere( + 'WHERE email = $1::jsonb::eql_v3.query_text_eq', + [await v3Operand('email', 'alan@example.com', 'unique')], + ) + const v4 = await emailsWhere( + 'WHERE email_v4 = $1::jsonb::eql_v4.query_text_eq', + [v4Operand('email', 'alan@example.com')], + ) + expect(v4).toEqual(v3) + expect(v4).toEqual(['alan@example.com']) + }) + + test('range: ORE comparisons agree across versions', async () => { + const expected = emails(customers.filter((c) => c.age > 40)) + const v3 = await emailsWhere( + 'WHERE age > $1::jsonb::eql_v3.query_integer_ord_ore ORDER BY id', + [await v3Operand('age', 40, 'ore')], + ) + const v4 = await emailsWhere( + 'WHERE age_v4 > $1::jsonb::eql_v4.query_integer_ord_ore ORDER BY id', + [v4Operand('age', 40)], + ) + expect(v3).toEqual(expected) + expect(v4).toEqual(expected) + }) + + test('ordering: ORE and OPE sort orders agree across versions', async () => { + const byAge = emails([...customers].sort((a, b) => a.age - b.age)) + expect(await emailsWhere('ORDER BY eql_v3.ord_term_ore(age)')).toEqual( + byAge, + ) + expect(await emailsWhere('ORDER BY eql_v4.ord_term_ore(age_v4)')).toEqual( + byAge, + ) + + const byRank = emails([...customers].sort((a, b) => a.rank - b.rank)) + expect(await emailsWhere('ORDER BY eql_v3.ord_term(rank)')).toEqual(byRank) + expect(await emailsWhere('ORDER BY eql_v4.ord_term(rank_v4)')).toEqual( + byRank, + ) + }) + + test('match: the v4 bloom filter finds the rows the v3 one found', async () => { + const expected = emails( + customers.filter((c) => c.bio.includes('programming')), + ) + const v3 = await emailsWhere( + 'WHERE bio @@ $1::jsonb::eql_v3.query_text_match ORDER BY id', + [await v3Operand('bio', 'programming', 'match')], + ) + const v4 = await emailsWhere( + 'WHERE bio_v4 @@ $1::jsonb::eql_v4.query_text_match ORDER BY id', + [v4Operand('bio', 'programming')], + ) + expect(v3).toEqual(expected) + expect(v4).toEqual(expected) + }) + + test('a v3 operand against a v4 column matches nothing, silently', async () => { + // The cut-over hazard: the column moved to v4 but the query still comes + // from the v3 client. No operator is defined for the pair, so Postgres + // falls back to `jsonb = jsonb` and returns no rows rather than erroring + // (pinned in packages/eql/tests/sqlx/tests/v3_v4_coexistence_tests.rs). + // Move the reads and the query client together. + expect( + await emailsWhere('WHERE email_v4 = $1::jsonb::eql_v3.query_text_eq', [ + await v3Operand('email', 'alan@example.com', 'unique'), + ]), + ).toEqual([]) + }) + + test('cut-over: drop the v3 columns and the v4 values answer under the original names', async () => { + // Step 5. The v4 values were encrypted under `customers/`, so + // renaming the database column changes nothing they are bound to. + for (const column of columnNames) { + await pg.query(`ALTER TABLE ${TABLE} DROP COLUMN ${column}`) + await pg.query( + `ALTER TABLE ${TABLE} RENAME COLUMN ${column}_v4 TO ${column}`, + ) + } + + expect( + await emailsWhere('WHERE email = $1::jsonb::eql_v4.query_text_eq', [ + v4Operand('email', 'grace@example.com'), + ]), + ).toEqual(['grace@example.com']) + + const res = await pg.query<{ active: unknown }>( + `SELECT active::jsonb AS active FROM ${TABLE} ORDER BY id`, + ) + expect( + decryptV4( + res.rows.map((row) => ({ + type: 'Boolean' as const, + table: TABLE, + column: 'active', + payload: row.active, + })), + ), + ).toEqual(customers.map((c) => c.active)) + }) +}) diff --git a/languages/typescript/packages/protect-ffi/integration-tests/tests/v4-codec.ts b/languages/typescript/packages/protect-ffi/integration-tests/tests/v4-codec.ts new file mode 100644 index 000000000..a4cfb93ce --- /dev/null +++ b/languages/typescript/packages/protect-ffi/integration-tests/tests/v4-codec.ts @@ -0,0 +1,126 @@ +import { spawnSync } from 'node:child_process' +import { existsSync } from 'node:fs' +import { resolve } from 'node:path' +import type { Client } from 'pg' + +// EQL v4 has one producer, Stack Encrypt, and it is reached from Rust and Go +// only: no TypeScript SDK writes `v: 4` yet. The migration suites +// (`migrate-v*-to-v4.test.ts`) therefore decrypt legacy rows here with +// protect-ffi and re-encrypt the plaintexts through a small Rust codec, +// `packages/eql/tests/encryption/examples/v4_codec.rs`. It dispatches on the +// EQL type name through `eql_bindings::encryption::targets`, the same entry +// point the Go guest uses, under live ZeroKMS and the same `CS_*` credentials +// as protect-ffi. +// +// Built by `mise run eql:v4:codec` (part of `mise run setup`). Set +// `EQL_V4_CODEC` to use a binary somewhere else. + +const repoRoot = resolve(__dirname, '../../../../../..') + +export const codecPath = + process.env.EQL_V4_CODEC ?? + resolve(repoRoot, 'packages/eql/target/debug/examples/v4_codec') + +/** An EQL v4 type name, e.g. `TextEq`, as `targets::targets()` lists it. */ +export type V4Type = + | 'Boolean' + | 'IntegerOrdOpe' + | 'IntegerOrdOre' + | 'TextEq' + | 'TextMatch' + +export type Plaintext = string | number | boolean + +export type V4Item = { + type: V4Type + table: string + column: string + value: Plaintext +} + +export type V4Payload = { v: 4; i: { t: string; c: string } } & Record< + string, + unknown +> + +type Request = { + encrypt?: V4Item[] + query?: V4Item[] + decrypt?: { + type: V4Type + table: string + column: string + payload: unknown + }[] +} + +type Response = { + encrypt: V4Payload[] + query: V4Payload[] + decrypt: Plaintext[] +} + +function run(request: Request): Response { + if (!existsSync(codecPath)) { + throw new Error( + `EQL v4 codec not found at ${codecPath}. Build it with \`mise run eql:v4:codec\` from languages/typescript/packages/protect-ffi, or set EQL_V4_CODEC.`, + ) + } + const result = spawnSync(codecPath, { + input: JSON.stringify(request), + encoding: 'utf8', + env: process.env, + maxBuffer: 64 * 1024 * 1024, + }) + if (result.error) throw result.error + if (result.status !== 0) { + throw new Error( + `EQL v4 codec exited ${result.status}: ${result.stderr.trim()}`, + ) + } + return JSON.parse(result.stdout) as Response +} + +/** Encrypt each item as the stored EQL v4 payload for its type. */ +export function encryptV4(items: V4Item[]): V4Payload[] { + if (items.length === 0) return [] + return run({ encrypt: items }).encrypt +} + +/** + * The term-only EQL v4 query operand for each item. Bind it as the type's + * query domain, e.g. `$1::jsonb::eql_v4.query_text_eq`. + */ +export function queryV4(items: V4Item[]): V4Payload[] { + if (items.length === 0) return [] + return run({ query: items }).query +} + +/** + * Decrypt stored EQL v4 payloads. Throws when Stack Encrypt refuses one, + * which is what a payload from any other producer gets. + */ +export function decryptV4( + items: { type: V4Type; table: string; column: string; payload: unknown }[], +): Plaintext[] { + if (items.length === 0) return [] + return run({ decrypt: items }).decrypt +} + +/** + * Fail with a setup hint unless EQL v4 is installed next to the legacy + * version the suite migrates from. + */ +export async function requireEqlV4(pg: Client): Promise { + const res = await pg.query<{ installed: boolean }>( + "SELECT to_regnamespace('eql_v4') IS NOT NULL AS installed", + ) + if (!res.rows[0]?.installed) { + throw new Error( + 'EQL v4 is not installed. Run `mise run eql:v4:install` from languages/typescript/packages/protect-ffi.', + ) + } +} + +/** SQLSTATE `check_violation`: a payload of the wrong version for a domain. */ +export const CHECK_VIOLATION = '23514' diff --git a/packages/eql/Cargo.lock b/packages/eql/Cargo.lock index a71c1d4e3..818dc01f1 100644 --- a/packages/eql/Cargo.lock +++ b/packages/eql/Cargo.lock @@ -478,6 +478,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.4" @@ -1681,7 +1687,7 @@ dependencies = [ "hickory-proto", "idna", "ipnet", - "jni", + "jni 0.22.4", "rand 0.10.1", "thiserror 2.0.20", "tinyvec", @@ -1699,7 +1705,7 @@ dependencies = [ "data-encoding", "idna", "ipnet", - "jni", + "jni 0.22.4", "once_cell", "prefix-trie", "rand 0.10.1", @@ -1722,7 +1728,7 @@ dependencies = [ "hickory-proto", "ipconfig", "ipnet", - "jni", + "jni 0.22.4", "moka", "ndk-context", "once_cell", @@ -2126,6 +2132,22 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + [[package]] name = "jni" version = "0.22.4" @@ -2135,7 +2157,7 @@ dependencies = [ "cfg-if", "combine", "jni-macros", - "jni-sys", + "jni-sys 0.4.1", "log", "simd_cesu8", "thiserror 2.0.20", @@ -2156,6 +2178,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + [[package]] name = "jni-sys" version = "0.4.1" @@ -3323,7 +3354,7 @@ dependencies = [ "quinn", "rustls", "rustls-pki-types", - "rustls-platform-verifier", + "rustls-platform-verifier 0.7.0", "serde", "serde_json", "serde_urlencoded", @@ -3504,6 +3535,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni 0.21.1", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + [[package]] name = "rustls-platform-verifier" version = "0.7.0" @@ -3512,7 +3564,7 @@ checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", - "jni", + "jni 0.22.4", "log", "once_cell", "rustls", @@ -4134,10 +4186,15 @@ version = "0.44.0" dependencies = [ "aquamarine", "base64 0.22.1", + "bytes", "cts-common 0.43.0", "jsonwebtoken", "miette", "open", + "reqwest", + "rustls", + "rustls-platform-verifier 0.6.2", + "rustls-webpki", "serde", "serde_json", "serde_urlencoded", @@ -4172,6 +4229,7 @@ dependencies = [ "serde_json", "serdect", "sha2 0.11.0", + "stack-auth 0.44.0", "stack-encrypt-derive", "stack-kms", "thiserror 1.0.69", @@ -4205,9 +4263,12 @@ dependencies = [ "base64ct", "blake3", "futures", + "lazy_static", "miette", "opaque-debug", "recipher 0.3.1", + "reqwest", + "rustls", "serde", "serde_cbor", "serde_json", @@ -5737,6 +5798,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -5782,6 +5852,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -5830,6 +5915,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -5848,6 +5939,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -5866,6 +5963,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -5896,6 +5999,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -5914,6 +6023,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -5932,6 +6047,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -5950,6 +6071,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" diff --git a/packages/eql/tests/encryption/Cargo.toml b/packages/eql/tests/encryption/Cargo.toml index 787f19abc..fde5bad33 100644 --- a/packages/eql/tests/encryption/Cargo.toml +++ b/packages/eql/tests/encryption/Cargo.toml @@ -27,7 +27,10 @@ eql-bindings = { path = "../../crates/eql-bindings", features = ["targets", "chr eql-domains = { path = "../../crates/eql-domains" } # `dynamic`: `ValueFrame`, the runtime plaintext the target dispatch takes, # as eql-bindings names it. -stack-encrypt = { path = "../../../stack-encrypt", default-features = false, features = ["dynamic"] } +# `http`: `StackCipher::new()` over live ZeroKMS, for `examples/v4_codec.rs`, +# the codec the protect-ffi migration suites re-encrypt through. The tests +# here still build their ciphers over explicit fake key sources. +stack-encrypt = { path = "../../../stack-encrypt", default-features = false, features = ["dynamic", "http"] } # `plaintext::Sink`, for a plaintext type with its own `Encrypt` impl; and # chrono's and rust_decimal's own sources, which seal the leaf a date, # timestamp or decimal from any other producer seals. diff --git a/packages/eql/tests/encryption/examples/v4_codec.rs b/packages/eql/tests/encryption/examples/v4_codec.rs new file mode 100644 index 000000000..ddf599341 --- /dev/null +++ b/packages/eql/tests/encryption/examples/v4_codec.rs @@ -0,0 +1,192 @@ +//! A stdin/stdout EQL v4 codec over live ZeroKMS, for migration tests in other +//! languages. +//! +//! TypeScript cannot produce EQL v4 yet: the only v4 producer is Stack +//! Encrypt, reached from Rust and Go. The live migration suite in +//! `languages/typescript/packages/protect-ffi/integration-tests` (EQL v2 and +//! v3 to v4) decrypts legacy rows with protect-ffi and hands the plaintexts +//! here to re-encrypt them as v4. Every value goes through +//! `eql_bindings::encryption::targets`, the by-name dispatch the Go guest +//! uses, so the payloads are the ones a v4 SDK writes. +//! +//! Run from `packages/eql`: +//! +//! ```sh +//! cargo build --locked -p eql-encryption-tests --example v4_codec +//! echo '{"encrypt":[{"type":"TextEq","table":"users","column":"email","value":"a@b.c"}]}' \ +//! | target/debug/examples/v4_codec +//! ``` +//! +//! Credentials come from the environment (`CS_CLIENT_ACCESS_KEY`, +//! `CS_WORKSPACE_CRN`, `CS_CLIENT_ID`, `CS_CLIENT_KEY`) or the CLI profile, the +//! same way `StackCipher::new()` finds them everywhere else. +//! +//! The request is one JSON object with three optional lists: +//! +//! ```json +//! { +//! "encrypt": [{ "type": "TextEq", "table": "t", "column": "c", "value": "x" }], +//! "query": [{ "type": "IntegerOrd", "table": "t", "column": "c", "value": 7 }], +//! "decrypt": [{ "type": "TextEq", "table": "t", "column": "c", "payload": { … } }] +//! } +//! ``` +//! +//! The response has the same three keys, each a list in request order: +//! stored payloads, query operands (bind as the target's `query_sql_domain`) +//! and decrypted plaintexts. `type` is a target name from +//! `eql_bindings::encryption::targets::targets()`. A query uses the target's +//! first index, which for every scalar target is the one its operators need. +//! Plaintexts are JSON strings, booleans and numbers; the target's +//! `plaintext` kind decides how a number is read. + +use std::io::{Read, Write}; + +use eql_bindings::encryption::targets::{self, Opener}; +use serde::Deserialize; +use serde_json::{json, Value}; +use stack_encrypt::dynamic::ValueFrame; +use stack_encrypt::target::IndexSpec; +use stack_encrypt::{Decrypt, Encrypt, Label, StackCipher}; + +type Error = Box; + +#[derive(Deserialize, Default)] +#[serde(deny_unknown_fields)] +struct Request { + #[serde(default)] + encrypt: Vec, + #[serde(default)] + query: Vec, + #[serde(default)] + decrypt: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Item { + r#type: String, + table: String, + column: String, + value: Value, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Stored { + r#type: String, + table: String, + column: String, + payload: Value, +} + +/// The target's plaintext kind, e.g. `"int32"`, or an error naming the type. +fn plaintext_kind(name: &str) -> Result<&'static str, Error> { + let target = targets::target(name).ok_or_else(|| format!("no EQL v4 target named {name}"))?; + target + .plaintext + .ok_or_else(|| format!("{name} has no scalar plaintext kind").into()) +} + +fn label(table: &str, column: &str) -> Result { + Ok(Label::new([table, column])?) +} + +/// A JSON plaintext as the engine's runtime frame, read as `name`'s kind. +fn frame(name: &str, value: &Value) -> Result { + let kind = plaintext_kind(name)?; + let wrong = || format!("{name} takes a {kind} plaintext, not {value}"); + let frame = match kind { + "string" => ValueFrame::of(value.as_str().ok_or_else(wrong)?.to_owned()), + "boolean" | "bool" => ValueFrame::of(value.as_bool().ok_or_else(wrong)?), + "int16" => ValueFrame::of(i16::try_from(value.as_i64().ok_or_else(wrong)?)?), + "int32" => ValueFrame::of(i32::try_from(value.as_i64().ok_or_else(wrong)?)?), + "int64" => ValueFrame::of(value.as_i64().ok_or_else(wrong)?), + "float32" => ValueFrame::of(value.as_f64().ok_or_else(wrong)? as f32), + "float64" => ValueFrame::of(value.as_f64().ok_or_else(wrong)?), + other => { + return Err(format!("{name}: plaintext kind {other} is not supported here").into()) + } + }; + Ok(frame?) +} + +/// An opened frame as JSON, read as `name`'s kind. +fn plaintext(name: &str, opened: ValueFrame) -> Result { + let kind = plaintext_kind(name)?; + let value = match kind { + "string" => json!(opened.visit(String::sink())?), + "boolean" | "bool" => json!(opened.visit(bool::sink())?), + "int16" => json!(opened.visit(i16::sink())?), + "int32" => json!(opened.visit(i32::sink())?), + "int64" => json!(opened.visit(i64::sink())?), + "float32" => json!(opened.visit(f32::sink())?), + "float64" => json!(opened.visit(f64::sink())?), + other => { + return Err(format!("{name}: plaintext kind {other} is not supported here").into()) + } + }; + Ok(value) +} + +fn index(name: &str) -> Result { + let target = targets::target(name).ok_or_else(|| format!("no EQL v4 target named {name}"))?; + target + .indexes + .first() + .and_then(|key| IndexSpec::parse(key)) + .ok_or_else(|| format!("{name} is storage-only and answers no query").into()) +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Error> { + let mut input = String::new(); + std::io::stdin().read_to_string(&mut input)?; + let request: Request = serde_json::from_str(&input)?; + + let cipher = StackCipher::new().await?; + let keyset = cipher.default_keyset(); + + let mut encrypted = Vec::with_capacity(request.encrypt.len()); + for item in &request.encrypt { + let bytes = targets::encrypt( + &item.r#type, + &keyset, + &label(&item.table, &item.column)?, + frame(&item.r#type, &item.value)?, + )? + .await?; + encrypted.push(serde_json::from_slice::(&bytes)?); + } + + let mut queries = Vec::with_capacity(request.query.len()); + for item in &request.query { + let bytes = targets::query( + &item.r#type, + &index(&item.r#type)?, + &keyset, + &label(&item.table, &item.column)?, + frame(&item.r#type, &item.value)?, + )? + .await?; + queries.push(serde_json::from_slice::(&bytes)?); + } + + let mut decrypted = Vec::with_capacity(request.decrypt.len()); + for item in &request.decrypt { + let stored = serde_json::to_vec(&item.payload)?; + let opened = targets::decrypt( + &item.r#type, + Opener::Client(&cipher), + &label(&item.table, &item.column)?, + &stored, + )? + .await?; + decrypted.push(plaintext(&item.r#type, opened)?); + } + + let response = json!({ "encrypt": encrypted, "query": queries, "decrypt": decrypted }); + let mut stdout = std::io::stdout().lock(); + serde_json::to_writer(&mut stdout, &response)?; + stdout.write_all(b"\n")?; + Ok(()) +} diff --git a/scripts/__tests__/tests-workflow-gate.test.mjs b/scripts/__tests__/tests-workflow-gate.test.mjs index 4526c4b8b..8917fa990 100644 --- a/scripts/__tests__/tests-workflow-gate.test.mjs +++ b/scripts/__tests__/tests-workflow-gate.test.mjs @@ -165,6 +165,10 @@ const READS_EXCLUDED = new Map([ 'languages/typescript/packages/protect-ffi/src/integrationSuiteCi.test.ts', 'Reads `.github/workflows/`. protect-ffi’s suite also runs in `repo-checks`.', ], + [ + 'languages/typescript/packages/protect-ffi/integration-tests/tests/v4-codec.ts', + 'Reads the built codec under `packages/eql/target`. The live suite that imports it runs only in `integration-protect-ffi.yml`, which is not gated and builds the codec itself, never in `pnpm test`.', + ], [ 'packages/eql/packages/eql/scripts/sync-generated.mjs', 'Reads `crates/eql-bindings/bindings` and `schema`. Run by `check:generated` and the release, never by `pnpm test`.', From 2cd436f23597b9036ece163ed1a2c2b23d05003b Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Tue, 3 Nov 2026 10:04:50 +1100 Subject: [PATCH 2/2] fix(protect-ffi): address review feedback on the v3-to-v4 migration suite Run integration-protect-ffi.yml when the EQL v4 code the migration suite exercises changes: the eql-bindings v4, encryption and encoding modules the codec compiles, and the eql-codegen / eql-domains sources the v4 SQL bundle is rendered from. test-eql.yml runs the Rust and SQL suites on those files but never this suite, so a v4 change could merge with the end-to-end migration coverage skipped. The entries are derived in lib/eql-v4-suite-inputs.mjs and held by eql-v4-suite-triggers.test.mjs; the existing EQL crate path guard allows exactly those, and only in this workflow. Also, in the suite itself: - tear down in afterAll so a setup failure after connect() still closes the socket and drops the table; - give the codec spawnSync a 60s timeout, since vitest cannot interrupt a synchronous stall against live ZeroKMS; - assert both hm terms exist before comparing them, so the inequality cannot pass on undefined. --- .github/workflows/integration-protect-ffi.yml | 30 ++++- .../tests/migrate-v3-to-v4.test.ts | 16 ++- .../integration-tests/tests/v4-codec.ts | 15 ++- .../__tests__/eql-v4-suite-triggers.test.mjs | 60 +++++++++ scripts/__tests__/lib/eql-v4-suite-inputs.mjs | 120 ++++++++++++++++++ .../wasm-build-inputs-paths-filter.test.mjs | 14 +- 6 files changed, 247 insertions(+), 8 deletions(-) create mode 100644 scripts/__tests__/eql-v4-suite-triggers.test.mjs create mode 100644 scripts/__tests__/lib/eql-v4-suite-inputs.mjs diff --git a/.github/workflows/integration-protect-ffi.yml b/.github/workflows/integration-protect-ffi.yml index e7992900e..6e058a24a 100644 --- a/.github/workflows/integration-protect-ffi.yml +++ b/.github/workflows/integration-protect-ffi.yml @@ -103,10 +103,23 @@ on: # builds `tests/encryption/examples/v4_codec.rs` over eql-bindings' v4 # targets, stack-encrypt and stack-kms. No published package carries any of # it, so these paths are the only inputs that change what those suites run. - # `packages/eql/crates/**` (eql-codegen, eql-bindings' v4 module) is left - # out on purpose: wasm-build-inputs-paths-filter.test.mjs holds this - # filter to the crate files the native binding compiles, and test-eql.yml - # already runs on every change there. + # The crate entries are the eql-bindings modules the codec compiles and the + # native binding does not (`v4`, `encryption`, `encoding`), and the + # eql-codegen / eql-domains sources `tasks/build.sh` renders the v4 bundle + # from. test-eql.yml runs the Rust and SQL suites on those files but not + # this one, so without them a v4 change skips the migration suites. + # Derived in scripts/__tests__/lib/eql-v4-suite-inputs.mjs and held here by + # eql-v4-suite-triggers.test.mjs; wasm-build-inputs-paths-filter.test.mjs + # allows exactly these beyond the native binding's files, and only here. + - 'packages/eql/crates/eql-bindings/src/encoding.rs' + - 'packages/eql/crates/eql-bindings/src/encryption.rs' + - 'packages/eql/crates/eql-bindings/src/encryption/**' + - 'packages/eql/crates/eql-bindings/src/v4/**' + - 'packages/eql/crates/eql-codegen/Cargo.toml' + - 'packages/eql/crates/eql-codegen/src/**' + - 'packages/eql/crates/eql-codegen/templates/**' + - 'packages/eql/crates/eql-domains/Cargo.toml' + - 'packages/eql/crates/eql-domains/src/**' - 'packages/eql/Cargo.lock' - 'packages/eql/src/**' - 'packages/eql/tasks/**' @@ -162,6 +175,15 @@ on: - 'packages/eql/crates/eql-bindings/src/v3/**' - 'packages/eql/crates/eql-bindings/sql/**' - 'packages/eql/Cargo.toml' + - 'packages/eql/crates/eql-bindings/src/encoding.rs' + - 'packages/eql/crates/eql-bindings/src/encryption.rs' + - 'packages/eql/crates/eql-bindings/src/encryption/**' + - 'packages/eql/crates/eql-bindings/src/v4/**' + - 'packages/eql/crates/eql-codegen/Cargo.toml' + - 'packages/eql/crates/eql-codegen/src/**' + - 'packages/eql/crates/eql-codegen/templates/**' + - 'packages/eql/crates/eql-domains/Cargo.toml' + - 'packages/eql/crates/eql-domains/src/**' - 'packages/eql/Cargo.lock' - 'packages/eql/src/**' - 'packages/eql/tasks/**' diff --git a/languages/typescript/packages/protect-ffi/integration-tests/tests/migrate-v3-to-v4.test.ts b/languages/typescript/packages/protect-ffi/integration-tests/tests/migrate-v3-to-v4.test.ts index 97476333c..19aba2889 100644 --- a/languages/typescript/packages/protect-ffi/integration-tests/tests/migrate-v3-to-v4.test.ts +++ b/languages/typescript/packages/protect-ffi/integration-tests/tests/migrate-v3-to-v4.test.ts @@ -8,7 +8,7 @@ import { newClient, } from '@cipherstash/protect-ffi' import { Client } from 'pg' -import { beforeAll, describe, expect, test } from 'vitest' +import { afterAll, beforeAll, describe, expect, test } from 'vitest' import { CHECK_VIOLATION, decryptV4, @@ -143,6 +143,7 @@ const emails = (rows: Customer[]) => rows.map((row) => row.email) describe('EQL v3 -> v4 migration', () => { const pg = new Client() + let connected = false let v3Client: Awaited> /** Ids the query returns, read as emails through the plaintext oracle. */ @@ -177,6 +178,7 @@ describe('EQL v3 -> v4 migration', () => { beforeAll(async () => { v3Client = await newClient({ encryptConfig, eqlVersion: 3 }) await pg.connect() + connected = true await requireEqlV4(pg) // Step 1: the table as Stack 1.x left it — v3 columns only. @@ -208,9 +210,16 @@ describe('EQL v3 -> v4 migration', () => { [index + 1, ...row], ) } + }) - return async () => { + // In `afterAll`, not returned from `beforeAll`: a returned teardown runs only + // when setup succeeds, so a throw after `connect()` would leave the socket + // open and the table behind. + afterAll(async () => { + if (!connected) return + try { await pg.query(`DROP TABLE IF EXISTS ${TABLE}`) + } finally { await pg.end() } }) @@ -288,6 +297,9 @@ describe('EQL v3 -> v4 migration', () => { value: 'alan@example.com', }, ]) + // Both terms must exist, or `not.toEqual` passes on `undefined`. + expect(typeof v4Query.hm).toBe('string') + expect(typeof relabelled.hm).toBe('string') expect(v4Query.hm).not.toEqual(relabelled.hm) }) diff --git a/languages/typescript/packages/protect-ffi/integration-tests/tests/v4-codec.ts b/languages/typescript/packages/protect-ffi/integration-tests/tests/v4-codec.ts index a4cfb93ce..d930cc126 100644 --- a/languages/typescript/packages/protect-ffi/integration-tests/tests/v4-codec.ts +++ b/languages/typescript/packages/protect-ffi/integration-tests/tests/v4-codec.ts @@ -60,6 +60,8 @@ type Response = { decrypt: Plaintext[] } +const CODEC_TIMEOUT_MS = 60_000 + function run(request: Request): Response { if (!existsSync(codecPath)) { throw new Error( @@ -71,8 +73,19 @@ function run(request: Request): Response { encoding: 'utf8', env: process.env, maxBuffer: 64 * 1024 * 1024, + // The codec authenticates against live ZeroKMS. `spawnSync` blocks the + // event loop, so vitest's own timeout cannot interrupt a stall; this does. + timeout: CODEC_TIMEOUT_MS, + killSignal: 'SIGKILL', }) - if (result.error) throw result.error + if (result.error) { + if ((result.error as NodeJS.ErrnoException).code === 'ETIMEDOUT') { + throw new Error( + `EQL v4 codec did not finish within ${CODEC_TIMEOUT_MS / 1000}s and was killed.`, + ) + } + throw result.error + } if (result.status !== 0) { throw new Error( `EQL v4 codec exited ${result.status}: ${result.stderr.trim()}`, diff --git a/scripts/__tests__/eql-v4-suite-triggers.test.mjs b/scripts/__tests__/eql-v4-suite-triggers.test.mjs new file mode 100644 index 000000000..3d6924a91 --- /dev/null +++ b/scripts/__tests__/eql-v4-suite-triggers.test.mjs @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest' +import { eqlV4SuiteFilterEntries } from './lib/eql-v4-suite-inputs.mjs' +import { filterCovers } from './lib/paths-filter.mjs' +import { REPO_ROOT } from './lib/repo-root.mjs' +import { readWorkflow } from './lib/workflows.mjs' + +/** + * The protect-ffi migration suites (`migrate-v*-to-v4.test.ts`) re-encrypt + * through a Rust codec built over `eql-bindings`' v4 modules, and query SQL + * rendered by `eql-codegen`. No other workflow runs those suites: + * `test-eql.yml` runs the Rust and SQL suites, not this one. So + * `integration-protect-ffi.yml` has to start when any of those files change, + * or a change to v4 target dispatch merges with the end-to-end migration + * coverage skipped. The entries come from `lib/eql-v4-suite-inputs.mjs`. + */ +const WORKFLOW = '.github/workflows/integration-protect-ffi.yml' + +/** A literal or `/**`-tailed filter entry covering the input pattern. */ +function entryCovers(entry, input) { + const literal = (p) => { + const segments = p.split('/') + const at = segments.findIndex((segment) => /[*?[]/.test(segment)) + return (at === -1 ? segments : segments.slice(0, at)).join('/') + } + const prefix = literal(entry) + const tail = entry.slice(prefix.length).replace(/^\//, '') + if (tail !== '' && tail !== '**') return false + const target = literal(input) + return target === prefix || target.startsWith(`${prefix}/`) +} + +describe('the EQL v4 migration suites trigger on their v4 inputs', () => { + const { features, entries } = eqlV4SuiteFilterEntries(REPO_ROOT) + + it('derives the v4 modules and the codegen crates', () => { + // An empty or shrunken derivation would pass every check below. + expect([...features]).toContain('stack-encrypt') + expect(entries).toEqual( + expect.arrayContaining([ + 'packages/eql/crates/eql-bindings/src/v4/**', + 'packages/eql/crates/eql-bindings/src/encryption/**', + 'packages/eql/crates/eql-codegen/src/**', + ]), + ) + }) + + const { on } = readWorkflow(WORKFLOW) + for (const event of ['push', 'pull_request']) { + it(`${WORKFLOW} (${event}) triggers on every v4 codec and codegen input`, () => { + const paths = on?.[event]?.paths ?? [] + const uncovered = entries.filter( + (entry) => !filterCovers(paths, entry, entryCovers), + ) + expect( + uncovered, + `${WORKFLOW} runs the EQL v4 migration suites, but its \`${event}\` \`paths:\` filter does not list these inputs of the v4 codec or the v4 SQL build. Add each to BOTH copies of the filter.`, + ).toEqual([]) + }) + } +}) diff --git a/scripts/__tests__/lib/eql-v4-suite-inputs.mjs b/scripts/__tests__/lib/eql-v4-suite-inputs.mjs new file mode 100644 index 000000000..0eb3f6667 --- /dev/null +++ b/scripts/__tests__/lib/eql-v4-suite-inputs.mjs @@ -0,0 +1,120 @@ +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { compiledCrateFiles } from './rust-compiled-files.mjs' + +/** + * The `paths:` filter entries under `packages/eql/crates/` that the protect-ffi + * migration suites (`integration-tests/tests/migrate-v*-to-v4.test.ts`) need, + * derived from the two builds those suites run rather than listed. + * + * - `mise run eql:v4:codec` builds `packages/eql/tests/encryption/examples/v4_codec.rs`, + * which compiles `eql-bindings` with the features that crate's manifest asks + * for. Those turn on the `v4`, `encryption` and `encoding` modules the native + * binding never compiles. + * - `mise run eql:v4:install` runs `tasks/build.sh`, which renders the v4 SQL + * bundle through `eql-codegen` (and `eql-domains`). Its `#MISE sources` line + * is the list of what that render reads. + * + * `integration-protect-ffi.yml` must trigger on every entry returned here, or a + * change to the v4 codec or the SQL it is checked against starts no suite that + * runs it. `wasm-build-inputs-paths-filter.test.mjs` forbids every other + * workflow from naming these paths, and allows exactly these for that one. + * + * Each compiled file becomes the smallest entry a reader would write: a file + * at the crate root or directly under `src/` is named itself, anything deeper + * becomes its top module directory with `/**` (`src/v4/**`, `sql/**`). + */ +const EQL_ROOT = 'packages/eql' +const EQL_CRATES = `${EQL_ROOT}/crates` +const CODEC_MANIFEST = `${EQL_ROOT}/tests/encryption/Cargo.toml` +const BINDINGS = `${EQL_CRATES}/eql-bindings` +const BUILD_SCRIPT = `${EQL_ROOT}/tasks/build.sh` + +/** The `features = [...]` the codec crate requests of `eql-bindings`. */ +function requestedBindingsFeatures(repoRoot) { + const text = readFileSync(resolve(repoRoot, CODEC_MANIFEST), 'utf8') + const line = text.split('\n').find((l) => /^\s*eql-bindings\s*=\s*\{/.test(l)) + if (!line) + throw new Error(`${CODEC_MANIFEST} no longer depends on eql-bindings`) + const list = /features\s*=\s*\[([^\]]*)\]/.exec(line) + if (!list) return [] + return [...list[1].matchAll(/"([^"]+)"/g)].map((m) => m[1]) +} + +/** `eql-bindings`' `[features]` table: name -> the entries it enables. */ +function bindingsFeatureTable(repoRoot) { + const text = readFileSync(resolve(repoRoot, `${BINDINGS}/Cargo.toml`), 'utf8') + const start = text.indexOf('\n[features]') + if (start === -1) return new Map() + const rest = text.slice(start + '\n[features]'.length) + const end = rest.search(/\n\[[^\]]+\]/) + const body = (end === -1 ? rest : rest.slice(0, end)).replace(/#.*$/gm, '') + const table = new Map() + for (const m of body.matchAll(/^\s*([\w-]+)\s*=\s*\[([^\]]*)\]/gm)) { + table.set( + m[1], + [...m[2].matchAll(/"([^"]+)"/g)].map((e) => e[1]), + ) + } + return table +} + +/** The requested features plus every crate-local feature they turn on. */ +function enabledFeatures(requested, table) { + const enabled = new Set() + const queue = [...requested] + while (queue.length > 0) { + const name = queue.shift() + if (enabled.has(name)) continue + enabled.add(name) + for (const entry of table.get(name) ?? []) { + if (!entry.includes('/') && !entry.startsWith('dep:')) queue.push(entry) + } + } + return enabled +} + +/** The smallest filter entry covering one compiled file of a crate. */ +function entryForCompiledFile(crateDir, file) { + const rel = file.slice(crateDir.length + 1).split('/') + if (rel.length === 1) return file + if (rel[0] === 'src') { + return rel.length === 2 ? file : `${crateDir}/src/${rel[1]}/**` + } + return `${crateDir}/${rel[0]}/**` +} + +/** `tasks/build.sh`'s `#MISE sources` that live under `crates/`. */ +function codegenSources(repoRoot) { + const text = readFileSync(resolve(repoRoot, BUILD_SCRIPT), 'utf8') + const line = /^#MISE sources=(\[.*\])\s*$/m.exec(text) + if (!line) throw new Error(`${BUILD_SCRIPT} no longer declares #MISE sources`) + return JSON.parse(line[1]) + .filter((source) => source.startsWith('crates/')) + .map((source) => `${EQL_ROOT}/${source}`) +} + +/** A glob reduced to its directory with `/**`, or a literal path as is. */ +function entryForSource(source) { + const segments = source.split('/') + const at = segments.findIndex((segment) => /[*?[]/.test(segment)) + return at === -1 ? source : `${segments.slice(0, at).join('/')}/**` +} + +export function eqlV4SuiteFilterEntries(repoRoot) { + const features = enabledFeatures( + requestedBindingsFeatures(repoRoot), + bindingsFeatureTable(repoRoot), + ) + const walk = compiledCrateFiles(repoRoot, BINDINGS, features) + if (walk.unresolved.length > 0) { + throw new Error( + `Could not resolve every eql-bindings file the v4 codec compiles:\n${walk.unresolved.join('\n')}`, + ) + } + const entries = new Set([ + ...walk.files.map((file) => entryForCompiledFile(BINDINGS, file)), + ...codegenSources(repoRoot).map(entryForSource), + ]) + return { features, entries: [...entries].sort() } +} diff --git a/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs b/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs index 457d59ef0..e25e65c65 100644 --- a/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs +++ b/scripts/__tests__/wasm-build-inputs-paths-filter.test.mjs @@ -2,6 +2,7 @@ import { existsSync, readFileSync } from 'node:fs' import { resolve } from 'node:path' import yaml from 'js-yaml' import { describe, expect, it } from 'vitest' +import { eqlV4SuiteFilterEntries } from './lib/eql-v4-suite-inputs.mjs' import { filterCovers } from './lib/paths-filter.mjs' import { REPO_ROOT } from './lib/repo-root.mjs' import { readWorkflow, workflowFiles } from './lib/workflows.mjs' @@ -548,12 +549,23 @@ describe('the FFI cache keys and the filters that trigger them agree', () => { // compile. The key now names the compiled files (held to them by // ffi-binding-action.test.mjs); a filter entry under the EQL crates must be // one of its entries, not a wider glob. + // + // One workflow builds more than the binding: integration-protect-ffi.yml + // runs the EQL v4 migration suites, which compile eql-bindings' v4 modules + // into a codec and render the v4 SQL through eql-codegen. It may also name + // exactly the entries lib/eql-v4-suite-inputs.mjs derives for those builds + // (eql-v4-suite-triggers.test.mjs requires them), and nothing wider. const EQL_CRATES = 'packages/eql/crates' const nativeInputs = new Set(buildInputs(NATIVE_CACHE_STEP)) + const V4_SUITE_WORKFLOW = '.github/workflows/integration-protect-ffi.yml' + const v4SuiteEntries = new Set(eqlV4SuiteFilterEntries(REPO_ROOT).entries) for (const { relPath, event, paths } of KEYS[0].workflows.flatMap( filteredEvents, )) { it(`${relPath} (${event}) names no EQL crate path the binding does not compile`, () => { + const allowed = (entry) => + nativeInputs.has(entry) || + (relPath === V4_SUITE_WORKFLOW && v4SuiteEntries.has(entry)) const wider = paths.filter( (entry) => !entry.startsWith('!') && @@ -561,7 +573,7 @@ describe('the FFI cache keys and the filters that trigger them agree', () => { literalPrefix(entry).startsWith(`${EQL_CRATES}/`) || EQL_CRATES.startsWith(`${literalPrefix(entry)}/`)) && literalPrefix(entry) !== '' && - !nativeInputs.has(entry), + !allowed(entry), ) expect( wider,