diff --git a/.github/workflows/tests-python.yml b/.github/workflows/tests-python.yml index 7331960a7..f0fec514d 100644 --- a/.github/workflows/tests-python.yml +++ b/.github/workflows/tests-python.yml @@ -48,6 +48,10 @@ on: # in place, so a change to either reruns this suite. - packages/eql/crates/eql-bindings/** - languages/golang/encrypt/guest/src/targets.rs + # The EQL rows the Go SDK stored under a deterministic key source. + # `mise run test:deterministic` opens them and seals the same values, + # so a change to them must rerun that check. + - packages/eql/tests/encryption/fixtures/** # The Rust version the workspace builds under. The workspace's own # mise.toml is under languages/python/** above. - mise.toml @@ -159,8 +163,10 @@ jobs: # pyo3's build script asks the `python3` on PATH for its configuration. # That is why this runs after mise is installed, once per interpreter. + # With every feature, so the code behind the test features + # (deterministic-kms, gate-probe) is linted too. - name: clippy - run: cargo clippy --locked --no-deps --workspace --all-targets -- -D warnings + run: cargo clippy --locked --no-deps --workspace --all-targets --all-features -- -D warnings # `cargo test --locked` for both crates. The task puts the interpreter's # library directory on the loader path, because the engine crate links @@ -173,6 +179,14 @@ jobs: - name: pytest run: mise run test + # Rebuilds the extension with the deterministic-kms test feature, then + # runs tests_deterministic/: Python opens the EQL rows the Go SDK stored + # under a deterministic key source, the frozen ones among them, and + # seals the same values to the same terms. No credentials. It runs after + # `pytest` because it replaces the build in .venv with a test build. + - name: Python against the Go SDK's rows + run: mise run test:deterministic + # The committed stackencrypt/_eql.py is generated from the EQL catalog # by eql-codegen (#1385). `mise run types:check` in test-eql.yml diffs # it, but that workflow does not run on a change under languages/python, diff --git a/AGENTS.md b/AGENTS.md index a496a3a77..2a6e685ee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate included, needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. -- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and `query` against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. There is no Python lockfile yet. The package has no runtime dependency, so only the `test` extra could be locked: pytest, attrs, Pydantic, SQLAlchemy, and psycopg. The middle three are for the declaration tests, and psycopg is for the live tests. None has an upper bound, so the tests meet each new release when users do. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, the call shape of `query`), the spec in #1341. +- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and `query` against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. There is no Python lockfile yet. The package has no runtime dependency, so only the `test` extra could be locked: pytest, attrs, Pydantic, SQLAlchemy, and psycopg. The middle three are for the declaration tests, and psycopg is for the live tests. None has an upper bound, so the tests meet each new release when users do. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, the call shape of `query`), the spec in #1341. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) - `languages/typescript/examples/*`: Working apps (basic, prisma, supabase-worker) - `docs/plans/*`: Internal design plans. User-facing documentation lives at https://cipherstash.com/docs (not in this repo). diff --git a/languages/python/Cargo.lock b/languages/python/Cargo.lock index 5b021c441..f1746dd78 100644 --- a/languages/python/Cargo.lock +++ b/languages/python/Cargo.lock @@ -3107,6 +3107,7 @@ dependencies = [ "stack-encrypt", "stack-kms", "tokio", + "zeroize", "zerokms-protocol", ] @@ -3115,8 +3116,11 @@ name = "stackencrypt-engine" version = "0.1.0" dependencies = [ "pyo3", + "stack-encrypt", + "stack-kms", "stackencrypt-core", "tokio", + "zeroize", ] [[package]] diff --git a/languages/python/README.md b/languages/python/README.md index 6a73e8f23..93f03dcd1 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -110,8 +110,11 @@ The query type is the marker's `query_sql_domain`. For `TextEq` it is `eql_v4.qu From `languages/python`: ```sh -mise run test # no credentials needed -mise run test:live # needs the four CS_* variables and EQL_TEST_DATABASE_URL +mise run test # no credentials needed +mise run test:deterministic # no credentials needed +mise run test:live # needs the four CS_* variables and EQL_TEST_DATABASE_URL ``` +`mise run test:deterministic` builds a test version of the extension. Its keys come from a fixed seed, not from CipherStash. The tests in `tests_deterministic/` then open the EQL rows the Go SDK stored under that seed. They also check that Python seals the same values to the same terms. Run `mise run build` afterwards to get a normal build back. + `mise run test:live` installs the EQL v4 bundle into the database that `EQL_TEST_DATABASE_URL` names. Use a database you can throw away. diff --git a/languages/python/crates/core/Cargo.toml b/languages/python/crates/core/Cargo.toml index 03d9c887b..a7053ab67 100644 --- a/languages/python/crates/core/Cargo.toml +++ b/languages/python/crates/core/Cargo.toml @@ -17,6 +17,12 @@ default = ["eql"] # reads a feature of exactly this name. Every build of the SDK holds the EQL # types, so the crate refuses to compile without it. eql = [] +# A test build: the shared client takes its keys from stack-kms's +# `DeterministicSource` and a seed, not from ZeroKMS, once +# `client::use_deterministic` is called. The frozen-row tests read rows the Go +# SDK sealed under the same source (the Go guest's `deterministic-kms` build). +# Never in a wheel a user installs. +deterministic-kms = ["stack-kms/test-support"] [dependencies] # By path, like every other eql-bindings dependency in the repository: the @@ -35,6 +41,9 @@ stack-kms = { workspace = true } # One multi-thread runtime per process, beside the shared client: an # operation blocks its own thread on it with the GIL released. tokio = { version = "1", features = ["rt-multi-thread"] } +# To wipe the opened text and bytes a `Scalar` holds when it is dropped. +# The version the workspace already resolves. +zeroize = "1" # For the request-counting connection: the trait a ZeroKMS request # implements. The version stack-kms resolves. zerokms-protocol = "0.12.31" diff --git a/languages/python/crates/core/src/client.rs b/languages/python/crates/core/src/client.rs index b3eedf07d..921a3b121 100644 --- a/languages/python/crates/core/src/client.rs +++ b/languages/python/crates/core/src/client.rs @@ -23,7 +23,7 @@ use stack_encrypt::kms::{ HttpConnectionOpts, KeyProvider, KeyProviderError, ProfileStore, StackKms, StackKmsBuilderError, ZeroKMSConnection, ZeroKMSConnectionInit, ZeroKmsEndpoint, }; -use stack_encrypt::{KeysetCipher, StackCipher}; +use stack_encrypt::StackCipher; use zerokms_protocol::{ViturRequest, ViturRequestError}; use crate::credentials; @@ -55,12 +55,18 @@ static REQUESTS: AtomicU64 = AtomicU64::new(0); /// client key or the ZeroKMS client (`stack_kms_*`), or the keyset load /// (`stack_encrypt_*`). Nothing is cached on failure. pub fn shared() -> Result<&'static Client, Refusal> { + #[cfg(feature = "deterministic-kms")] + refuse_after_switch()?; if let Some(client) = CLIENT.get() { return Ok(client); } let _building = BUILDING .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); + // Again under the lock: a switch can finish while this thread waits for + // it, and the live client must then not be built. + #[cfg(feature = "deterministic-kms")] + refuse_after_switch()?; if let Some(client) = CLIENT.get() { return Ok(client); } @@ -69,6 +75,106 @@ pub fn shared() -> Result<&'static Client, Refusal> { Ok(CLIENT.get_or_init(|| client)) } +/// Run an operation's future on the process's client, and return its result. +/// +/// `run!(|cipher| future)` names the `StackCipher` the body builds its future +/// from. The body is written once and compiled for each client: the live +/// one, and in the test build the deterministic one, which takes its place +/// after the switch. The two have different key source types, so a function +/// that returned "the current client" would still leave each operation to +/// write a branch per client. +macro_rules! run { + (|$cipher:ident| $future:expr) => {{ + #[cfg(feature = "deterministic-kms")] + if let Some((cipher, runtime)) = $crate::client::deterministic() { + let $cipher = cipher; + return runtime.block_on($future); + } + let client = $crate::client::shared()?; + let $cipher = client.cipher(); + client.block_on($future) + }}; +} +pub(crate) use run; + +/// The client of the `deterministic-kms` test build, with its own runtime. +#[cfg(feature = "deterministic-kms")] +pub type Deterministic = ( + StackCipher, + tokio::runtime::Runtime, +); + +#[cfg(feature = "deterministic-kms")] +static DETERMINISTIC: OnceLock = OnceLock::new(); + +/// In the test build, an operation that asks for the live client after the +/// switch has not checked for the deterministic client, and would send +/// requests to ZeroKMS from a test process. Refuse it, so a new operation +/// that skips the check fails its test instead. +#[cfg(feature = "deterministic-kms")] +fn refuse_after_switch() -> Result<(), Refusal> { + if DETERMINISTIC.get().is_none() { + return Ok(()); + } + Err(Refusal { + code: "stack_test_build".to_owned(), + message: "this operation asked for the live client after the switch to the \ + deterministic one; call client::deterministic() first" + .to_owned(), + field: None, + reason: None, + }) +} + +/// Make every later operation in the process use a client whose keys come +/// from `seed` through `DeterministicSource`, not from ZeroKMS. For the +/// `deterministic-kms` test build only: rows sealed under the same seed open +/// in any process, the Go SDK's frozen rows among them. +/// +/// # Errors +/// +/// A refusal when the live client is already built, when a deterministic +/// client is already in place, or when the client does not build. +#[cfg(feature = "deterministic-kms")] +pub fn use_deterministic(seed: [u8; 32]) -> Result<(), Refusal> { + let refused = |message: String| Refusal { + code: "stack_test_build".to_owned(), + message, + field: None, + reason: None, + }; + // The lock `shared` builds under: a live build in flight on another + // thread finishes before this checks, so the two never both exist. + let _building = BUILDING + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if CLIENT.get().is_some() || DETERMINISTIC.get().is_some() { + return Err(refused( + "the process already has a client; call this before any operation".to_owned(), + )); + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|error| refused(format!("the runtime could not start: {error}")))?; + let cipher = runtime + .block_on( + StackCipher::builder() + .kms(stack_kms::DeterministicSource::new(seed)) + .init(), + ) + .map_err(|error| Refusal::of(&error))?; + DETERMINISTIC + .set((cipher, runtime)) + .map_err(|_| refused("a deterministic client is already in place".to_owned())) +} + +/// The deterministic client, once `use_deterministic` has made one. +#[cfg(feature = "deterministic-kms")] +pub fn deterministic() -> Option<&'static Deterministic> { + DETERMINISTIC.get() +} + /// How many times the shared client has been built in this process: 0 /// before the first operation, 1 after it. For the tests. pub fn build_count() -> u64 { @@ -82,11 +188,6 @@ pub fn request_count() -> u64 { } impl Client { - /// The cipher bound to the client's default keyset. - pub fn keyset(&self) -> KeysetCipher<'_, Kms> { - self.cipher.default_keyset() - } - /// The cipher itself. pub fn cipher(&self) -> &StackCipher { &self.cipher diff --git a/languages/python/crates/core/src/decrypt.rs b/languages/python/crates/core/src/decrypt.rs index 9a15187f9..cf7931c28 100644 --- a/languages/python/crates/core/src/decrypt.rs +++ b/languages/python/crates/core/src/decrypt.rs @@ -177,8 +177,7 @@ fn refusal(code: &str, field: &Field, message: String) -> Refusal { /// `decrypt_kind_mismatch` for a value stored as another kind than the /// field's. pub fn decrypt(plan: &ModelPlan, rows: Rows) -> Result>, Refusal> { - let client = client::shared()?; - client.block_on(decrypt_with(client.cipher(), plan, rows)) + client::run!(|cipher| decrypt_with(cipher, plan, rows)) } /// [`decrypt`] on `cipher`. Each value opens under the keyset that sealed @@ -344,7 +343,8 @@ impl<'a> Sink for OpenedRow<'a> { fn map(self, _: Option) -> Result, ReadError> { Ok(OpenedFields { - values: vec![None; self.fields.len()], + values: vec![Scalar::Null; self.fields.len()], + seen: vec![false; self.fields.len()], fields: self.fields, rows: self.rows, }) @@ -355,9 +355,13 @@ impl<'a> Sink for OpenedRow<'a> { } } +/// One row as it opens. Each value is written into its slot, and `end` +/// moves the whole buffer into the rows: no value is moved out of a buffer +/// that is then freed with its bytes still in it. struct OpenedFields<'a> { fields: &'a [Field], - values: Vec>, + values: Vec, + seen: Vec, rows: &'a mut Vec>, } @@ -377,30 +381,31 @@ impl MapSink for OpenedFields<'_> { .ok_or(ReadError("a field the plan does not have"))?; Ok(OpenedValue { slot: &mut self.values[at], + seen: &mut self.seen[at], }) } fn end(self) -> Result<(), ReadError> { - let row = self - .values - .into_iter() - .collect::>>() - .ok_or(ReadError("a row without one of the plan's fields"))?; - self.rows.push(row); + if !self.seen.iter().all(|&seen| seen) { + return Err(ReadError("a row without one of the plan's fields")); + } + self.rows.push(self.values); Ok(()) } } /// One opened field: one scalar. struct OpenedValue<'a> { - slot: &'a mut Option, + slot: &'a mut Scalar, + seen: &'a mut bool, } impl OpenedValue<'_> { fn put(self, scalar: Scalar) -> Result<(), ReadError> { - if self.slot.replace(scalar).is_some() { + if std::mem::replace(self.seen, true) { return not_expected("a field twice in one row"); } + *self.slot = scalar; Ok(()) } } diff --git a/languages/python/crates/core/src/encrypt.rs b/languages/python/crates/core/src/encrypt.rs index d8bbae87b..7c51b70e0 100644 --- a/languages/python/crates/core/src/encrypt.rs +++ b/languages/python/crates/core/src/encrypt.rs @@ -11,7 +11,9 @@ //! out as a [`Column`], which the extension turns into the dict's value. use stack_encrypt::dynamic::record::{self, Plan, EQL_KEY}; -use stack_encrypt::dynamic::{self, Output, ValueFrame}; +/// The engine's input frame, which the extension writes the models into. +pub use stack_encrypt::dynamic::ValueFrame; +use stack_encrypt::dynamic::{self, Output}; use stack_encrypt::kms::{DataKeySource, IndexKeySource}; use stack_encrypt::{CipherText, Encrypt, KeysetCipher, StackCipherText}; @@ -104,8 +106,7 @@ pub fn encrypt( source: ValueFrame, rows: usize, ) -> Result>, Refusal> { - let client = client::shared()?; - client.block_on(encrypt_with(&client.keyset(), plan, source, rows)) + client::run!(|cipher| encrypt_with(&cipher.default_keyset(), plan, source, rows)) } /// [`encrypt`] under `keyset`. @@ -174,12 +175,16 @@ fn read_row(plan: &ModelPlan, row: StackCipherText) -> Result, Refus field: Some(field.name.clone()), ..refused })?; - columns.push(match (&field.eql, scalar) { + let mut scalar = scalar; + columns.push(match (&field.eql, &mut scalar) { + // The EQL JSON is ciphertext, not plaintext: taken out of the + // scalar, which wipes what is left when dropped. (Some(_), Scalar::Bytes(json)) => Column::Eql( - String::from_utf8(json).map_err(|_| engine_fault("an EQL value is not UTF-8"))?, + String::from_utf8(std::mem::take(json)) + .map_err(|_| engine_fault("an EQL value is not UTF-8"))?, ), (Some(_), _) => return Err(engine_fault("an EQL value is not bytes")), - (None, scalar) => Column::Passthrough(scalar), + (None, _) => Column::Passthrough(scalar), }); } Ok(columns) diff --git a/languages/python/crates/core/src/query.rs b/languages/python/crates/core/src/query.rs index 5df850c43..818596bf5 100644 --- a/languages/python/crates/core/src/query.rs +++ b/languages/python/crates/core/src/query.rs @@ -35,8 +35,7 @@ pub fn query( value: Value<'_>, ) -> Result { answers_equality(plan, field)?; - let client = client::shared()?; - client.block_on(query_with(&client.keyset(), plan, field, kind, value)) + client::run!(|cipher| query_with(&cipher.default_keyset(), plan, field, kind, value)) } /// [`query`] under `keyset`. diff --git a/languages/python/crates/core/src/values.rs b/languages/python/crates/core/src/values.rs index 4efdde9c7..c00bc8f39 100644 --- a/languages/python/crates/core/src/values.rs +++ b/languages/python/crates/core/src/values.rs @@ -17,6 +17,7 @@ use stack_encrypt::dynamic::plaintext::{ kinds, Error as SinkError, FixedType, Float, Int, Logical, MapSink, Physical, SeqSink, Sink, SourceError, TimeUnit, }; +use zeroize::Zeroize; use std::convert::Infallible; use std::marker::PhantomData; @@ -248,6 +249,28 @@ pub enum Scalar { }, } +/// An opened value is plaintext, and the SDK's own copy of it: it is wiped +/// when dropped, the inline numbers in a row's buffer as well as the text +/// and bytes on the heap. The extension copies it into a Python object +/// first (`src/heap_hygiene.rs` in the engine crate checks the drop). +impl Drop for Scalar { + fn drop(&mut self) { + match self { + Self::Null => {} + Self::Text(text) => text.zeroize(), + Self::Bytes(bytes) => bytes.zeroize(), + Self::Int(int) | Self::Timestamp(int) => int.zeroize(), + Self::Float(float) => float.zeroize(), + Self::Bool(bool) => bool.zeroize(), + Self::Date(days) => days.zeroize(), + Self::Decimal { mantissa, scale } => { + mantissa.zeroize(); + scale.zeroize(); + } + } + } +} + impl Scalar { /// The scalar as the value the SDK writes, or `None` for a null, which /// the SDK never writes. diff --git a/languages/python/crates/engine/Cargo.toml b/languages/python/crates/engine/Cargo.toml index 8b86ba296..2548ad639 100644 --- a/languages/python/crates/engine/Cargo.toml +++ b/languages/python/crates/engine/Cargo.toml @@ -26,8 +26,13 @@ stackencrypt-core = { path = "../core" } # `extension-module` is deliberately NOT here. It tells pyo3 not to link # libpython, which is right for the wheel and wrong for `cargo test`: a test # binary has to link it or fails with undefined `_Py*` symbols. maturin turns -# the feature on through `[tool.maturin].features` in pyproject.toml. +# the feature on in two places. One is `[tool.maturin].features` in +# pyproject.toml. The other is the `test:deterministic` task in mise.toml: a +# `--features` flag replaces pyproject.toml's list, so the task names it again. pyo3 = { version = "0.29", features = ["abi3-py39"] } +# To wipe the owned copy of a `str` the extension reads (src/values.rs, +# `Read`). The version the workspace already resolves. +zeroize = "1" # Only for the `gate-probe` feature below. tokio = { version = "1", features = ["rt-multi-thread", "time"], optional = true } @@ -36,3 +41,15 @@ tokio = { version = "1", features = ["rt-multi-thread", "time"], optional = true # Python test can send SIGINT during the call. Off in every normal build; # see docs/gate-facts/keyboard-interrupt.md for how to run it. gate-probe = ["dep:tokio"] +# A test build whose client takes its keys from a seed, not from ZeroKMS, once +# `_use_deterministic_keys` is called. `mise run test:deterministic` builds it +# and runs the frozen-row tests in `tests_deterministic/`. Never in a wheel a +# user installs. +deterministic-kms = ["stackencrypt-core/deterministic-kms"] + +[dev-dependencies] +# For the heap-hygiene test (src/heap_hygiene.rs): a cipher over the +# in-memory key source, and a runtime to run the engine on. +stack-encrypt = { workspace = true } +stack-kms = { workspace = true, features = ["test-support"] } +tokio = { version = "1", features = ["rt"] } diff --git a/languages/python/crates/engine/src/decrypt.rs b/languages/python/crates/engine/src/decrypt.rs index 063d2ad40..843b5f7f3 100644 --- a/languages/python/crates/engine/src/decrypt.rs +++ b/languages/python/crates/engine/src/decrypt.rs @@ -43,9 +43,10 @@ pub fn decrypt<'py>( .detach(|| core::decrypt(&plan.plan, stored)) .map_err(|refused| raise(py, refused))?; // Every value first, then every instance: a value Python cannot hold - // fails the call before any constructor runs. + // fails the call before any constructor runs. Borrowed, not moved: each + // value is wiped in its row's buffer when `opened` drops. let mut arguments = Vec::with_capacity(opened.len()); - for values in opened { + for values in &opened { let kwargs = PyDict::new(py); for (field, value) in plan.plan.fields().iter().zip(values) { kwargs.set_item(&field.name, python_value(py, field, value)?)?; @@ -202,20 +203,16 @@ fn passthrough( } /// An opened value as the Python object of its field. -fn python_value(py: Python<'_>, field: &Field, value: Scalar) -> PyResult> { - let ranged = matches!(value, Scalar::Date(_) | Scalar::Timestamp(_)); - to_python(py, value).map_err(|error| { - if !ranged { - return error; - } +fn python_value(py: Python<'_>, field: &Field, value: &Scalar) -> PyResult> { + to_python(py, value)?.map_err(|unfit| { raise( py, Refusal { code: "decrypt_not_representable".to_owned(), message: format!( - "field {:?}: the opened value is outside the range Python's datetime \ - holds.", - field.name + "field {:?}: the opened value does not fit its Python type: {}.", + field.name, + unfit.cause() ), field: Some(field.name.clone()), reason: None, diff --git a/languages/python/crates/engine/src/encrypt.rs b/languages/python/crates/engine/src/encrypt.rs index 6a86bec11..d5571dda0 100644 --- a/languages/python/crates/engine/src/encrypt.rs +++ b/languages/python/crates/engine/src/encrypt.rs @@ -10,10 +10,11 @@ use std::cell::RefCell; +use pyo3::exceptions::PyValueError; use pyo3::prelude::*; -use pyo3::types::{PyDict, PyList, PyType}; +use pyo3::types::{PyDict, PyList, PyString, PyType}; use stackencrypt_core::encrypt::plaintext::{Encrypt, MapSink, SeqSink, Sink, SourceError}; -use stackencrypt_core::encrypt::{self as core, Column, Field, ModelPlan}; +use stackencrypt_core::encrypt::{self as core, Column, Field, ModelPlan, ValueFrame}; use stackencrypt_core::refusal::Refusal; use stackencrypt_core::values::{self, Kind}; @@ -107,38 +108,22 @@ pub fn encrypt<'py>( return Ok(PyList::empty(py)); } let rows = models.len(); - let failed = RefCell::new(None); - let source = Models { - plan, - models: &models, - failed: &failed, - }; - let frame = match core::frame_of(source) { - Ok(frame) => frame, - Err(error) => { - return Err(failed.into_inner().unwrap_or_else(|| { - raise( - py, - Refusal { - code: "stack_encrypt_unknown".to_owned(), - message: format!("the models could not be written as a frame: {error}"), - field: None, - reason: None, - }, - ) - })) - } - }; + let frame = input_frame(py, plan, &models)?; let encrypted = py .detach(|| core::encrypt(&plan.plan, frame, rows)) .map_err(|refused| raise(py, refused))?; let out = PyList::empty(py); - for columns in encrypted { + // Borrowed, not moved: a passthrough value is wiped in its row's buffer + // when `encrypted` drops. + for columns in &encrypted { let row = PyDict::new(py); for (field, column) in plan.plan.fields().iter().zip(columns) { let value = match column { - Column::Eql(json) => json.into_pyobject(py)?.into_any().unbind(), - Column::Passthrough(scalar) => to_python(py, scalar)?, + Column::Eql(json) => PyString::new(py, json).into_any().unbind(), + // A passthrough value came from Python, so it fits. + Column::Passthrough(scalar) => { + to_python(py, scalar)?.map_err(|unfit| PyValueError::new_err(unfit.cause()))? + } }; row.set_item(&field.name, value)?; } @@ -147,6 +132,33 @@ pub fn encrypt<'py>( Ok(out) } +/// The models as the engine's input frame, every value read and checked. +pub(crate) fn input_frame( + py: Python<'_>, + plan: &Plan, + models: &[Bound<'_, PyAny>], +) -> PyResult { + let failed = RefCell::new(None); + let source = Models { + plan, + models, + failed: &failed, + }; + core::frame_of(source).map_err(|error| { + failed.into_inner().unwrap_or_else(|| { + raise( + py, + Refusal { + code: "stack_encrypt_unknown".to_owned(), + message: format!("the models could not be written as a frame: {error}"), + field: None, + reason: None, + }, + ) + }) + }) +} + /// How many times the shared engine client has been built in this process. /// For the tests: two `Cipher` instances share one. #[pyfunction] diff --git a/languages/python/crates/engine/src/heap_hygiene.rs b/languages/python/crates/engine/src/heap_hygiene.rs new file mode 100644 index 000000000..b7bf4c5c6 --- /dev/null +++ b/languages/python/crates/engine/src/heap_hygiene.rs @@ -0,0 +1,398 @@ +//! No heap block the extension frees during `encrypt`, `query` or `decrypt` +//! still contains the plaintext. +//! +//! The scope is the extension's own copies: what this crate and the engine +//! allocate in Rust while they read a Python value, write the input frame, +//! seal it, derive a query term, open a row, and turn an opened value into a +//! Python object. Text and decimals are watched: the text by a marker it +//! contains, a decimal by its digits, as text and as the digit values a +//! `Decimal`'s tuple holds, and by its mantissa as the engine's `i128`. Python's own objects are not watched. +//! CPython allocates them with its own allocator, and the caller owns them, so +//! the SDK makes no promise about them. This includes the `bytes` object that +//! pyo3 makes on its way to a `str`'s UTF-8 under `abi3-py39`. +//! +//! The test binary installs a spy as its global allocator, as stack-encrypt's +//! `tests/heap_hygiene.rs` does. While a test watches its own thread, the spy +//! scans every block freed on that thread for a marker that the plaintext +//! contains. A block freed with the marker still in it is plaintext that a +//! later allocation can read back, and fails the test. The Go SDK's test reads +//! the guest's memory after the call instead, which allocator reuse has +//! mostly overwritten by then. This one sees each free as it happens. +//! +//! One free is excluded by name, as in stack-encrypt's test: vitaminc's +//! aws-lc backend frees an unwiped plaintext buffer on every seal, which is +//! vitaminc's to fix. The spy counts those, and recognises them by their +//! backtrace, so a build without symbols fails rather than passes. +//! +//! Unix only: the spy scans a freed block with libc's `memmem`. CI runs it on +//! Linux. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::Cell; +use std::ffi::c_void; +use std::sync::Mutex; + +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyString, PyType}; +use stack_encrypt::StackCipher; +use stack_kms::FakeDataKeySource; + +use stackencrypt_core::decrypt::{decrypt_with, Row, Rows}; +use stackencrypt_core::encrypt::{encrypt_with, Column}; +use stackencrypt_core::query::query_with; + +use crate::encrypt::{build_plan, field_value, input_frame, Plan}; +use crate::values::to_python; + +/// What every text plaintext under test contains. +const MARK: &[u8] = b"zq-heap-marker-7f3a9c"; + +/// The decimal under test, and its digits as the extension handles them: +/// as text, and as the digit values of `Decimal.as_tuple()`. +const DECIMAL: &str = "86753091415926.5358979"; +const DIGITS: &[u8] = b"867530914159265358979"; +const DIGIT_VALUES: &[u8] = &[ + 8, 6, 7, 5, 3, 0, 9, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9, 7, 9, +]; + +/// The decimal's mantissa as the engine holds it: an `i128`, little-endian, +/// to its last byte that is not zero. +const MANTISSA: [u8; 9] = { + let bytes = 867_530_914_159_265_358_979_i128.to_le_bytes(); + [ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], bytes[8], + ] +}; + +/// Every needle a freed block must not contain. +const NEEDLES: [&[u8]; 4] = [MARK, DIGITS, DIGIT_VALUES, &MANTISSA]; + +thread_local! { + static WATCHING: Cell = const { Cell::new(false) }; + static BUSY: Cell = const { Cell::new(false) }; +} + +/// Held for the whole of a watch: the harness runs tests on several threads, +/// and the findings below are one set. +static GATE: Mutex<()> = Mutex::new(()); + +/// Held for the whole of each test, taken before the test attaches to +/// Python. Without it, one test can hold [`GATE`] and wait for the GIL while +/// another holds the GIL and waits for [`GATE`]. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> std::sync::MutexGuard<'static, ()> { + SERIAL + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The marked frees seen, as backtraces; and how many were vitaminc's. +static FOUND: Mutex> = Mutex::new(Vec::new()); +static UPSTREAM: Mutex = Mutex::new(0); + +/// The vitaminc frame whose free is excluded (see the module docs). +const UPSTREAM_SEAL: &str = "vitaminc_encrypt::backend::aws_lc::CipherKey::seal"; + +/// Whether the `len` bytes at `ptr` contain one of [`NEEDLES`]. A freed +/// block can hold bytes Rust never initialised, and reading those through a +/// `&[u8]` is undefined behaviour, so the scan is libc's `memmem`. +/// +/// # Safety +/// +/// `ptr` must point to `len` allocated bytes. +unsafe fn has_mark(ptr: *const u8, len: usize) -> bool { + extern "C" { + fn memmem( + haystack: *const c_void, + haystack_len: usize, + needle: *const c_void, + needle_len: usize, + ) -> *mut c_void; + } + NEEDLES + .iter() + .any(|needle| !memmem(ptr.cast(), len, needle.as_ptr().cast(), needle.len()).is_null()) +} + +struct Spy; + +unsafe impl GlobalAlloc for Spy { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + // Zeroed, so a block holds only what was written into it during its + // own lifetime, not what an earlier free left there. + System.alloc_zeroed(layout) + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + let watching = WATCHING.with(Cell::get) && !BUSY.with(Cell::get); + // SAFETY: `ptr` is a live block of `layout.size()` bytes until the + // `System.dealloc` below. + if watching && has_mark(ptr, layout.size()) { + BUSY.with(|busy| busy.set(true)); + let trace = std::backtrace::Backtrace::force_capture().to_string(); + // Linux writes `<…::CipherKey>::seal`, macOS `…::CipherKey::seal`. + if trace.replace(['<', '>'], "").contains(UPSTREAM_SEAL) { + *UPSTREAM.lock().unwrap() += 1; + } else { + FOUND + .lock() + .unwrap() + .push(format!("{} bytes freed unwiped:\n{trace}", layout.size())); + } + BUSY.with(|busy| busy.set(false)); + } + System.dealloc(ptr, layout) + } +} + +#[global_allocator] +static SPY: Spy = Spy; + +/// Zero the stack below the caller's frame, so that a future the engine +/// boxes copies zeros, not what earlier work left on the stack. +#[inline(never)] +fn scrub_stack() { + let mut pad = [0u8; 256 * 1024]; + for byte in pad.iter_mut() { + // SAFETY: a volatile write to a live local, so it is not elided. + unsafe { std::ptr::write_volatile(byte, 0) }; + } + std::hint::black_box(&pad); +} + +/// Run `f` with this thread watched, and fail if any block it freed still +/// contained the marker. +fn watched(what: &str, f: impl FnOnce() -> T) -> T { + let _gate = GATE.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + FOUND.lock().unwrap().clear(); + *UPSTREAM.lock().unwrap() = 0; + scrub_stack(); + WATCHING.with(|on| on.set(true)); + let out = f(); + WATCHING.with(|on| on.set(false)); + let found = std::mem::take(&mut *FOUND.lock().unwrap()); + let upstream = std::mem::take(&mut *UPSTREAM.lock().unwrap()); + if upstream > 0 { + eprintln!("{what}: skipped {upstream} unwiped frees in vitaminc's aws-lc seal"); + } + assert!( + found.is_empty(), + "{what}: {} heap blocks were freed with plaintext in them:\n{}", + found.len(), + found.join("\n---\n") + ); + out +} + +fn marked(tail: &str) -> String { + format!("{}-{tail}", std::str::from_utf8(MARK).unwrap()) +} + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() +} + +/// A plan for `users`: a text field with an equality index, one with none, +/// and a decimal with an equality index. +fn plan(py: Python<'_>) -> Plan { + let text = || py.get_type::(); + let decimal = decimal_type(py).cast_into::().unwrap(); + build_plan( + py, + "users", + vec![ + ( + "email".into(), + Some("TextEq".into()), + Some("string".into()), + text(), + ), + ( + "note".into(), + Some("Text".into()), + Some("string".into()), + text(), + ), + ( + "amount".into(), + Some("NumericEq".into()), + Some("decimal".into()), + decimal, + ), + ], + ) + .unwrap() +} + +fn decimal_type(py: Python<'_>) -> Bound<'_, PyAny> { + py.import("decimal").unwrap().getattr("Decimal").unwrap() +} + +/// A model as `encrypt` reads it: an object with one attribute per field. +fn model<'py>(py: Python<'py>, email: &str, note: &str) -> Bound<'py, PyAny> { + let fields = PyDict::new(py); + fields.set_item("email", email).unwrap(); + fields.set_item("note", note).unwrap(); + fields + .set_item("amount", decimal_type(py).call1((DECIMAL,)).unwrap()) + .unwrap(); + py.import("types") + .unwrap() + .getattr("SimpleNamespace") + .unwrap() + .call((), Some(&fields)) + .unwrap() +} + +fn cipher(runtime: &tokio::runtime::Runtime) -> StackCipher { + runtime + .block_on(StackCipher::builder().kms(FakeDataKeySource::new()).init()) + .unwrap() +} + +/// Two marked models, encrypted outside any watch. +fn stored( + py: Python<'_>, + runtime: &tokio::runtime::Runtime, + cipher: &StackCipher, + plan: &Plan, +) -> Vec> { + let models = vec![ + model(py, &marked("a@example.com"), &marked("a note")), + model(py, &marked("b@example.com"), &marked("another note")), + ]; + let frame = input_frame(py, plan, &models).unwrap(); + runtime + .block_on(encrypt_with( + &cipher.default_keyset(), + &plan.plan, + frame, + models.len(), + )) + .unwrap() +} + +#[test] +fn encrypt_frees_no_copy_of_the_plaintext_unwiped() { + let _serial = serial(); + Python::initialize(); + let runtime = runtime(); + let cipher = cipher(&runtime); + Python::attach(|py| { + let plan = plan(py); + // The Python objects are made before the watch: they are the + // caller's, not the extension's. + let models = vec![ + model(py, &marked("a@example.com"), &marked("a note")), + model(py, &marked("b@example.com"), &marked("another note")), + ]; + + let rows = watched("encrypt", || { + let frame = input_frame(py, &plan, &models).unwrap(); + runtime + .block_on(encrypt_with( + &cipher.default_keyset(), + &plan.plan, + frame, + models.len(), + )) + .unwrap() + }); + + assert_eq!(rows.len(), 2); + }); +} + +#[test] +fn query_frees_no_copy_of_the_plaintext_unwiped() { + let _serial = serial(); + Python::initialize(); + let runtime = runtime(); + let cipher = cipher(&runtime); + Python::attach(|py| { + let plan = plan(py); + let text = PyString::new(py, &marked("a@example.com")).into_any(); + let decimal = decimal_type(py).call1((DECIMAL,)).unwrap(); + + for (at, value) in [(0, &text), (2, &decimal)] { + let (field, kind) = (&plan.plan.fields()[at], plan.kinds[at]); + let Some((_, engine_kind)) = field.eql else { + panic!("{} is sealed", field.name); + }; + let term = watched("query", || { + let read = field_value(value, field, kind).unwrap().unwrap(); + runtime + .block_on(query_with( + &cipher.default_keyset(), + &plan.plan, + &field.name, + engine_kind, + read.value(), + )) + .unwrap() + }); + assert!(!term.as_bytes().windows(MARK.len()).any(|w| w == MARK)); + } + }); +} + +#[test] +fn decrypt_frees_no_copy_of_the_plaintext_unwiped() { + let _serial = serial(); + Python::initialize(); + let runtime = runtime(); + let cipher = cipher(&runtime); + Python::attach(|py| { + let plan = plan(py); + let stored = stored(py, &runtime, &cipher, &plan); + + let opened = watched("decrypt", || { + let mut rows = Rows::with_capacity(stored.len()); + for columns in &stored { + let mut row = Row::new(); + for (field, column) in plan.plan.fields().iter().zip(columns) { + let Column::Eql(json) = column else { + panic!("{} is sealed", field.name); + }; + row.eql(&plan.plan, field, json.as_bytes()).unwrap(); + } + rows.push(row); + } + let opened = runtime + .block_on(decrypt_with(&cipher, &plan.plan, rows)) + .unwrap(); + // Each value as the Python object `decrypt` builds the instance + // from. The objects are the caller's once returned. + let objects = opened + .iter() + .flatten() + .map(|scalar| to_python(py, scalar).unwrap().unwrap()) + .collect::>(); + drop(opened); + objects + }); + + assert_eq!(opened.len(), 6); + assert_eq!( + opened[0].bind(py).extract::().unwrap(), + marked("a@example.com") + ); + }); +} + +/// The spy is not blind: a copy of each needle freed unwiped fails the +/// watch. If frees went unseen (another thread, an exclusion matching too +/// much), the three tests above would pass whatever the extension did. +#[test] +fn the_spy_fails_a_copy_freed_unwiped() { + let _serial = serial(); + for needle in NEEDLES { + let leaked = std::panic::catch_unwind(|| { + watched("a leak", || drop(std::hint::black_box(needle.to_vec()))); + }); + assert!(leaked.is_err(), "the spy missed {needle:?}"); + } +} diff --git a/languages/python/crates/engine/src/lib.rs b/languages/python/crates/engine/src/lib.rs index 1447c254d..048cba2d9 100644 --- a/languages/python/crates/engine/src/lib.rs +++ b/languages/python/crates/engine/src/lib.rs @@ -6,6 +6,8 @@ mod decrypt; mod encrypt; mod errors; +#[cfg(all(test, unix))] +mod heap_hygiene; mod query; mod values; @@ -88,6 +90,18 @@ fn _probe_detached_sleep(py: Python<'_>, millis: u64) -> u64 { }) } +/// Make every later operation in this process use keys derived from `seed` +/// (32 bytes) by stack-kms's `DeterministicSource`, not ZeroKMS. Only in the +/// `deterministic-kms` test build. Call it before any operation. +#[cfg(feature = "deterministic-kms")] +#[pyfunction] +fn _use_deterministic_keys(py: Python<'_>, seed: &[u8]) -> PyResult<()> { + let seed: [u8; 32] = seed + .try_into() + .map_err(|_| pyo3::exceptions::PyValueError::new_err("the seed is 32 bytes"))?; + stackencrypt_core::client::use_deterministic(seed).map_err(|refused| errors::raise(py, refused)) +} + #[pymodule] fn _engine(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(build_info, module)?)?; @@ -102,6 +116,8 @@ fn _engine(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(encrypt::request_count, module)?)?; #[cfg(feature = "gate-probe")] module.add_function(wrap_pyfunction!(_probe_detached_sleep, module)?)?; + #[cfg(feature = "deterministic-kms")] + module.add_function(wrap_pyfunction!(_use_deterministic_keys, module)?)?; Ok(()) } diff --git a/languages/python/crates/engine/src/values.rs b/languages/python/crates/engine/src/values.rs index 0bfb462d1..4d24fec7e 100644 --- a/languages/python/crates/engine/src/values.rs +++ b/languages/python/crates/engine/src/values.rs @@ -7,6 +7,7 @@ //! never changes a value: nothing here converts, only reads. use std::borrow::Cow; +use zeroize::{Zeroize, Zeroizing}; use pyo3::prelude::*; use pyo3::sync::PyOnceLock; @@ -118,12 +119,24 @@ impl PyKind { /// A field's value as read from its attribute. Text and bytes borrow from /// the attribute object where they can, so the plaintext is not copied /// before the engine writes it into the protected frame. +/// +/// Under `abi3-py39`, pyo3 cannot borrow a `str`'s UTF-8, so `Text` is an +/// owned copy on every Python version. That copy is the extension's own, and +/// is wiped when the `Read` is dropped (`src/heap_hygiene.rs` checks it). pub enum Read<'a> { Text(Cow<'a, str>), Bytes(&'a [u8]), Other(Value<'static>), } +impl Drop for Read<'_> { + fn drop(&mut self) { + if let Self::Text(Cow::Owned(text)) = self { + text.zeroize(); + } + } +} + impl Read<'_> { /// The value, as the core rules take it. pub fn value(&self) -> Value<'_> { @@ -228,7 +241,8 @@ fn decimal( ) -> PyResult, Refusal>> { let parts = value.call_method0("as_tuple")?; let sign: u8 = parts.getattr("sign")?.extract()?; - let digits: Vec = parts.getattr("digits")?.extract()?; + // The digits are plaintext: wiped when dropped. + let digits = Zeroizing::new(parts.getattr("digits")?.extract::>()?); let Ok(exponent) = parts.getattr("exponent")?.extract::() else { return Ok(Err(refusal( "value_not_representable", @@ -244,7 +258,7 @@ fn decimal( ) }; let mut mantissa: i128 = 0; - for digit in digits { + for &digit in digits.iter() { mantissa = match mantissa .checked_mul(10) .and_then(|m| m.checked_add(i128::from(digit))) @@ -304,51 +318,103 @@ fn type_name(value: &Bound<'_, PyAny>) -> String { .map_or_else(|_| "an unknown type".to_owned(), |name| name.to_string()) } +/// Why an opened date or timestamp has no Python object. The Go SDK can +/// store both. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Unfit { + /// `datetime` keeps microseconds, and rounding would change the value. + Nanoseconds, + /// `datetime` holds the years 1 to 9999. + Year, +} + +impl Unfit { + /// The cause, for a message. + pub fn cause(self) -> &'static str { + match self { + Self::Nanoseconds => { + "it has nanoseconds, and Python's datetime keeps only microseconds" + } + Self::Year => "its year is outside 1 to 9999, the years Python's datetime holds", + } + } +} + +/// `datetime`'s refusal of a value out of its range as [`Unfit::Year`]. +/// `fromordinal` raises `ValueError` for it, and `timedelta` and the +/// addition raise `OverflowError`. Any other error is Python's own. +fn in_range(py: Python<'_>, result: PyResult) -> PyResult> { + match result { + Ok(value) => Ok(Ok(value)), + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + Ok(Err(Unfit::Year)) + } + Err(error) => Err(error), + } +} + /// An opened scalar as a Python object, of the type the field declares. -pub fn to_python(py: Python<'_>, scalar: Scalar) -> PyResult> { - Ok(match scalar { +/// +/// The scalar is borrowed and copied into the Python object. It wipes its +/// own copy when dropped, so the caller drops it where it lies: a scalar +/// moved out of a row's buffer leaves its bytes behind in the buffer. +/// +/// # Errors +/// +/// The outer error is Python's own. The inner one is a date or timestamp +/// that `datetime` cannot hold, with the reason. +pub fn to_python(py: Python<'_>, scalar: &Scalar) -> PyResult, Unfit>> { + Ok(Ok(match scalar { Scalar::Null => py.None(), - Scalar::Text(text) => PyString::new(py, &text).into_any().unbind(), + Scalar::Text(text) => PyString::new(py, text).into_any().unbind(), Scalar::Int(i) => i.into_pyobject(py)?.into_any().unbind(), - Scalar::Float(f) => PyFloat::new(py, f).into_any().unbind(), - Scalar::Bool(b) => PyBool::new(py, b).to_owned().into_any().unbind(), - Scalar::Bytes(bytes) => PyBytes::new(py, &bytes).into_any().unbind(), - Scalar::Date(days) => PyKind::Date - .type_object(py)? - .call_method1("fromordinal", (i64::from(days) + 719_163,))? - .unbind(), - Scalar::Timestamp(nanos) => { - // Python's datetime keeps microseconds; the engine's timestamp - // is in nanoseconds, so the last three digits are dropped. An - // encrypted datetime had none: Python wrote it. + Scalar::Float(f) => PyFloat::new(py, *f).into_any().unbind(), + Scalar::Bool(b) => PyBool::new(py, *b).to_owned().into_any().unbind(), + Scalar::Bytes(bytes) => PyBytes::new(py, bytes).into_any().unbind(), + Scalar::Date(days) => { + let date = PyKind::Date.type_object(py)?; + let opened = date.call_method1("fromordinal", (i64::from(*days) + 719_163,)); + match in_range(py, opened)? { + Ok(date) => date.unbind(), + Err(unfit) => return Ok(Err(unfit)), + } + } + &Scalar::Timestamp(nanos) => { + if nanos.rem_euclid(1_000) != 0 { + return Ok(Err(Unfit::Nanoseconds)); + } let micros = nanos.div_euclid(1_000); let seconds = micros.div_euclid(1_000_000); let micros = micros.rem_euclid(1_000_000); - let delta = py - .import("datetime")? - .getattr("timedelta")? - .call1((0, seconds, micros))?; - epoch(py)?.add(delta)?.unbind() + let timedelta = py.import("datetime")?.getattr("timedelta")?; + let epoch = epoch(py)?; + let opened = timedelta + .call1((0, seconds, micros)) + .and_then(|delta| epoch.add(delta)); + match in_range(py, opened)? { + Ok(timestamp) => timestamp.unbind(), + Err(unfit) => return Ok(Err(unfit)), + } } - Scalar::Decimal { mantissa, scale } => { + &Scalar::Decimal { mantissa, scale } => { // Decimal((sign, digits, exponent)), so that the scale is kept: - // Decimal("1.50") is not Decimal("1.5") to Python. + // Decimal("1.50") is not Decimal("1.5") to Python. The digits + // are plaintext, and both copies are wiped when dropped. let sign = u8::from(mantissa < 0); - let digits: Vec = mantissa - .unsigned_abs() - .to_string() - .bytes() - .map(|b| b - b'0') - .collect(); + let text = Zeroizing::new(mantissa.unsigned_abs().to_string()); + let digits = Zeroizing::new(text.bytes().map(|b| b - b'0').collect::>()); // A tuple of ints: pyo3 converts a Vec to bytes, which // Decimal refuses. - let digits = PyTuple::new(py, digits)?; + let digits = PyTuple::new(py, digits.iter())?; PyKind::Decimal .type_object(py)? .call1(((sign, digits, -i64::from(scale)),))? .unbind() } - }) + })) } #[cfg(test)] @@ -362,7 +428,9 @@ mod tests { let decimal = py.import("decimal").unwrap().getattr("Decimal").unwrap(); for (mantissa, scale, text) in [(1_995, 2, "19.95"), (-150, 2, "-1.50"), (0, 2, "0.00")] { - let opened = to_python(py, Scalar::Decimal { mantissa, scale }).unwrap(); + let opened = to_python(py, &Scalar::Decimal { mantissa, scale }) + .unwrap() + .unwrap(); let opened = opened.bind(py); assert!(opened.get_type().is(&decimal), "{text}"); assert_eq!(opened.str().unwrap().to_string(), text); diff --git a/languages/python/mise.toml b/languages/python/mise.toml index 191bea47d..457aad949 100644 --- a/languages/python/mise.toml +++ b/languages/python/mise.toml @@ -53,6 +53,21 @@ description = "Run the pure-Python test suite against the package `build` instal depends = ["build"] run = ".venv/bin/python -m pytest" +[tasks."test:deterministic"] +description = "Build the extension with the deterministic-kms test feature into .venv and run tests_deterministic/: Python against the Go SDK's frozen EQL rows, with no credentials" +shell = "bash -c" +# The deterministic client replaces the live one for the whole process, so +# these tests run in their own pytest process, from their own folder. The +# build is a test build: run `mise run build` afterwards for a normal one. +# `--features` replaces pyproject.toml's list, so it names +# pyo3/extension-module again. Maturin 1.10 and later also set +# PYO3_BUILD_EXTENSION_MODULE, but pyproject.toml allows 1.9. +run = """ +set -euo pipefail +MATURIN_PEP517_ARGS="--profile dev --features pyo3/extension-module,deterministic-kms" mise run build +.venv/bin/python -m pytest tests_deterministic +""" + [tasks."test:cargo"] description = "Run both crates' Rust tests with --locked; puts libpython on the loader path so the engine crate's tests link" shell = "bash -c" diff --git a/languages/python/pyproject.toml b/languages/python/pyproject.toml index b32553992..836e39bd5 100644 --- a/languages/python/pyproject.toml +++ b/languages/python/pyproject.toml @@ -28,8 +28,9 @@ module-name = "stackencrypt._engine" python-source = "python" # The engine crate, not the workspace root: maturin builds one cdylib. manifest-path = "crates/engine/Cargo.toml" -# The one place `extension-module` is set. See crates/engine/Cargo.toml for -# why it is not in the crate manifest. +# Never set in the crate manifest: see crates/engine/Cargo.toml for why. A +# build that passes `--features` replaces this list, and names it again: +# the `test:deterministic` task in mise.toml. features = ["pyo3/extension-module"] [tool.pytest.ini_options] diff --git a/languages/python/tests_deterministic/conftest.py b/languages/python/tests_deterministic/conftest.py new file mode 100644 index 000000000..0c10c9000 --- /dev/null +++ b/languages/python/tests_deterministic/conftest.py @@ -0,0 +1,40 @@ +"""Every test here runs against the `deterministic-kms` test build. + +`mise run test:deterministic` builds the extension with that feature and runs +this folder in its own pytest process. The session switches the process to +the deterministic client before any test runs: its keys come from a seed +through stack-kms's `DeterministicSource`, not from ZeroKMS. So no test here +needs credentials or makes a request. + +The seed is the one the Go SDK's frozen rows were sealed under, so rows the +Go SDK stored open here. The switch is for the whole process, which is why +this folder is not under `tests/`. +""" + +import json +import os +from pathlib import Path + +import pytest + +from stackencrypt import _engine + +REPO = Path(__file__).resolve().parents[3] +# The Go SDK's EQL fixtures, all sealed under one seed. +FIXTURES = REPO / "packages/eql/tests/encryption/fixtures" + + +def pytest_sessionstart(session): + use = getattr(_engine, "_use_deterministic_keys", None) + if use is None: + pytest.exit( + "the extension was built without the deterministic-kms feature; " + "run `mise run test:deterministic`", + returncode=2, + ) + # Cipher checks that credentials are present before it builds a plan. + # These parse, and the deterministic client never uses them. + os.environ["CS_CLIENT_ACCESS_KEY"] = "CSAKtestKeyId.testKeySecret" + os.environ["CS_WORKSPACE_CRN"] = "crn:ap-southeast-2.aws:ZVATKW3VHMFG27DY" + frozen = json.loads((FIXTURES / "stored_v4_rows_v1.json").read_text()) + use(frozen["seed"].encode()) diff --git a/languages/python/tests_deterministic/test_go_rows.py b/languages/python/tests_deterministic/test_go_rows.py new file mode 100644 index 000000000..e6cd2d7d5 --- /dev/null +++ b/languages/python/tests_deterministic/test_go_rows.py @@ -0,0 +1,297 @@ +"""Python against the EQL rows the Go SDK stored, under the same seed. + +Every fixture here is in `packages/eql/tests/encryption/fixtures/`, written by +the Go SDK's generated code over the deterministic EQL guest. Each model +declares the same table, columns and EQL types as its Go type in +`languages/golang/encrypt/internal/testusers/`. + +- `stored_v4_rows_v1.json` is FROZEN: nothing writes it again. Go + (`TestFrozenV4RowsStillOpen`) and Rust (`tests/v4_types.rs`) check it too. + A build that cannot open it is red. +- `v4_ordered_rows.json`, `v4_smallint_rows.json` and `v4_ledger_rows.json` + are regenerated when the stored format changes on purpose. They hold what + the Go SDK seals today, for the types the frozen file lacks: the order + terms, `Smallint`, dates, timestamps and decimals. + +For each, Python opens the rows Go stored, and seals the same plaintexts to +the same stored values apart from each ciphertext `c`. The equality, order +and match terms are deterministic, so they must be byte-identical. Python's +equality query term for each value must be the term Go stored. +""" + +import dataclasses +import datetime +import decimal +import json +import struct +from pathlib import Path +from typing import Annotated + +import pytest + +import stackencrypt +from stackencrypt import ( + Bigint, + BigintEq, + BigintOrd, + BigintOrdOpe, + Boolean, + Cipher, + Date, + DateEq, + DecryptError, + Double, + DoubleEq, + DoubleOrd, + DoubleOrdOpe, + Integer, + IntegerEq, + IntegerOrd, + IntegerOrdOpe, + Numeric, + NumericEq, + Passthrough, + Real, + RealEq, + RealOrd, + RealOrdOpe, + Smallint, + SmallintEq, + Text, + TextEq, + TextMatch, + TextOrd, + TextOrdOpe, + Timestamp, + TimestampEq, +) + + +@stackencrypt.model("profiles") +@dataclasses.dataclass +class Profile: + id: Annotated[int, Passthrough] + visits: Annotated[int, Integer] + balance: Annotated[int, Bigint] + active: Annotated[bool, Boolean] + ratio: Annotated[float, Real] + score: Annotated[float, Double] + note: Annotated[str, Text] + name: Annotated[str, TextMatch] + age: Annotated[int, IntegerEq] + rank: Annotated[int, BigintEq] + weight: Annotated[float, RealEq] + price: Annotated[float, DoubleEq] + email: Annotated[str, TextEq] + + +@stackencrypt.model("rankings") +@dataclasses.dataclass +class Ranking: + id: Annotated[int, Passthrough] + level: Annotated[int, IntegerOrd] + floor: Annotated[int, IntegerOrdOpe] + points: Annotated[int, BigintOrd] + total: Annotated[int, BigintOrdOpe] + speed: Annotated[float, RealOrd] + pace: Annotated[float, RealOrdOpe] + score: Annotated[float, DoubleOrd] + mean: Annotated[float, DoubleOrdOpe] + title: Annotated[str, TextOrd] + handle: Annotated[str, TextOrdOpe] + + +@stackencrypt.model("profiles") +@dataclasses.dataclass +class Tally: + id: Annotated[int, Passthrough] + small: Annotated[int, Smallint] + tally: Annotated[int, SmallintEq] + wide: Annotated[int, SmallintEq] + + +@stackencrypt.model("profiles") +@dataclasses.dataclass +class Ledger: + id: Annotated[int, Passthrough] + opened: Annotated[datetime.date, Date] + due: Annotated[datetime.date, DateEq] + created: Annotated[datetime.datetime, Timestamp] + posted: Annotated[datetime.datetime, TimestampEq] + total: Annotated[decimal.Decimal, Numeric] + amount: Annotated[decimal.Decimal, NumericEq] + + +# The Go SDK's EQL fixtures. Not imported from conftest.py: `from conftest` +# resolves to whichever conftest.py is first on sys.path. +FIXTURES = Path(__file__).resolve().parents[3] / "packages/eql/tests/encryption/fixtures" + +# (fixture, model, the row ids Python's types can hold). Ledger rows 1 and 2 +# hold years past Python's 1 to 9999 and nanosecond timestamps: the tests +# at the end check each refusal on its own. +CASES = [ + ("stored_v4_rows_v1.json", Profile, [1, 2]), + ("v4_ordered_rows.json", Ranking, [1, 2, 3]), + ("v4_smallint_rows.json", Tally, [1, 2, 3]), + ("v4_ledger_rows.json", Ledger, [3]), +] +IDS = [fixture for fixture, _, _ in CASES] + + +def rows_of(fixture, ids): + rows = json.loads((FIXTURES / fixture).read_text())["rows"] + assert set(ids) <= {row["id"] for row in rows}, (fixture, ids) + return [row for row in rows if row["id"] in ids] + + +def sealed_fields(cls): + return [field for field in dataclasses.fields(cls) if field.name != "id"] + + +def value_of(python_type, plaintext): + """A Go fixture's JSON plaintext as the field's Python value.""" + if python_type is datetime.datetime: + return datetime.datetime.fromisoformat(plaintext.replace("Z", "+00:00")) + if python_type is datetime.date: + return datetime.date.fromisoformat(plaintext) + if python_type is decimal.Decimal: + return decimal.Decimal(plaintext) + # JSON gives 0 for a float field whose value is zero. + return python_type(plaintext) + + +def model_of(cls, row): + values = { + field.name: value_of(field.type.__origin__, row["plaintext"][field.name]) + for field in sealed_fields(cls) + } + return cls(id=row["id"], **values) + + +def stored_row(cls, row): + """The row as psycopg's dict_row gives it: EQL values as JSON text.""" + return { + "id": row["id"], + **{f.name: json.dumps(row["stored"][f.name]) for f in sealed_fields(cls)}, + } + + +def exact(value): + """A form whose == also compares a float's sign, a Decimal's scale, the + type (True is not 1), and an instant whatever its zone.""" + if isinstance(value, float): + return struct.pack("