diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 973fb0c..249e3d0 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "gitmemory", - "version": "0.1.0", - "description": "Registers gitmemory's hook shim on PreCompact and SessionEnd, so a session is captured at the compaction boundary instead of at the watcher's next sweep, and adds a recall skill that has the agent search the captured sessions with gitmemory recall and quote what was said, with the address of each line. The hook is only a doorbell, and the skill writes nothing but the search index: nothing is captured unless the gitmemory watcher is installed (uv tool install git+https://github.com/doronp/gitmemory) and running, and without the hook gitmemory is still correct.", + "version": "0.2.0", + "description": "Registers gitmemory's hook shim on PreCompact and SessionEnd, so a session is captured at the compaction boundary, and adds a recall skill that has the agent search past sessions with gitmemory recall and quote each line with its address. Neither captures anything: the gitmemory watcher must be installed (uv tool install --python 3.13 git+https://github.com/doronp/gitmemory), configured (gitmemory init claude-code) and running (gitmemory watch).", "author": { "name": "doronp" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 183f521..b6616d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,29 @@ tell you. ## [Unreleased] +## [0.2.0] - 2026-10-18 + ### Added +- **`gitmemory init claude-code`** (or `pi`) adds that agent's standard + `[[watch]]` table to `config.toml`, so setup no longer means writing TOML by + hand. Watch roots still have no default: the root is written only because you + named the agent. It appends without changing a byte already there, because + the file also holds the `[remote.*]` push opt-ins. It does nothing if the + table is already there, and it refuses, writing nothing and exiting 1, a root + the watcher would skip or another agent already watches, a `config.toml` + that is a symlink or a hard link, and one another `init` changed while it + ran. See + [docs/watching.md](docs/watching.md#configure-what-to-watch). +- **`gitmemory service install`** writes a systemd user unit (Linux) or a + LaunchAgent (macOS) that keeps `gitmemory watch` running, and prints the + commands that start it, keep it running after you log out (Linux: + `loginctl enable-linger`), check it, follow its log, restart it after an + upgrade and remove it. It starts nothing itself, and it refuses, exiting 1, + to overwrite a file that is already there. The file names the Python, the + store and git's directory outright, because a user service inherits none of + the shell's. See + [docs/watching.md](docs/watching.md#keep-it-running). - **`recall --policy`** decides which of two disagreeing hits comes first: `latest-user` (the user's own words newest first, then the rest), `latest` (newest first, whoever said it), `relevance` (best match first), or your own @@ -22,16 +43,93 @@ tell you. compaction or from an earlier session. It runs `gitmemory index` when the index is missing or stale, and writes nothing else. See [docs/USAGE.md](docs/USAGE.md#put-it-back-in-front-of-the-agent). What it - reads goes to the model provider, past no redaction gate. The plugin's - version is still 0.1.0, so a copy installed from 0.1.0 does not get the skill - from `claude plugin update`; reinstall it, or wait for the next release + reads goes to the model provider, past no redaction gate; `recall --redact` + (below) masks credential shapes in it, and blocks nothing. The skill was + added after 0.1.0 without a version bump, so a copy installed from 0.1.0 + lacks it; this release moves the plugin's version to 0.2.0, so + `claude plugin update` brings it ([hook/README.md](hook/README.md#as-a-claude-code-plugin)). +- **`gitmemory --version`** prints the installed release. The recall skill + runs it first and tells you when the CLI is older than the plugin; when the + CLI lacks a subcommand or a flag the skill needs, the skill stops and says + which instead of retrying without it. See + [hook/README.md](hook/README.md#as-a-claude-code-plugin). +- **`recall` marks a subagent's turns.** A turn a subagent or workflow agent + wrote prints its role as `subagent:`, as in `subagent:user/text`, in + the same column, so a line has as many fields as before. `Hit.is_sidechain` + carries the mark, read per turn from the transcript's `isSidechain`. See + [docs/USAGE.md](docs/USAGE.md#find-something). +- **Main sessions only.** [docs/watching.md](docs/watching.md#main-sessions-only-claude-code) + gives a Claude Code watch pattern, `pattern = "*/*.jsonl"`, that leaves + subagent and workflow transcripts out, and what that costs: those turns are + never captured, and the dashboard's spend leaves out what they cost. +- **`recall` says when no watcher is collecting.** When a hook record has + waited in the spool for more than an hour, `recall` prints one line on + stderr before the results, naming the store and asking whether + `gitmemory watch` is running for it. The time since the last capture cannot + tell a stopped watcher from an idle one; a record no pass has drained can. + Not under `--db`, and never on a store the hook does not ring. See + [docs/watching.md](docs/watching.md#check-it). +- **`gitmemory show
`** prints the whole turn at an address `recall` + printed, where `recall` stops at 160 characters. It reads the store, not the + index, so it needs no index and a stale one cannot change the answer. + `--context N` (`-C N`) adds N turns either side; a turn cut by a segment + boundary comes back whole; an offset inside a turn shows that turn; a + subagent's turn is headed `subagent:`. An address that is not one, or + is masked, exits 2, and one the store does not hold exits 1. The recall + skill uses it in place of reading `raw/` by hand. See + [USAGE.md](docs/USAGE.md#read-the-whole-turn). +- **`recall --redact`** and **`show --redact`** mask the credential shapes the + push gate detects in what they print, the text and the address (except, in + a session id, a key the store's lowercasing hides from its detector), as + `AKIA…[20 bytes]`; a private key is masked down to its END line, or to the + end of the text without one. `show` masks each block whole before it splits it into + lines, so a key's body goes with its header. Off by default. + `GITMEMORY_REDACT` makes it standing for both (unset, empty or `0` is off), + and the plugin's skill passes the flag to both whenever it is set to + anything but empty or `0`, and then reads nothing under the store by hand. + It masks only those shapes, and only in what the two commands print; see + [docs/USAGE.md](docs/USAGE.md#masking-credentials-in-what-it-prints) for + what it does not cover. +- **How much disk it takes.** Budget about twice what you capture, and more if + your transcripts compress poorly: `raw/` holds the bytes once, `.git/` a + compressed copy, and `index/` the searchable text again, which + `gitmemory index` rebuilds (and, while it runs, needs room for the old index + and the new). Measured at 2.3x to 3.4x on synthetic corpora; see + [docs/watching.md](docs/watching.md#how-much-disk-it-takes). ARCHITECTURE.md + said the store holds each byte exactly once, and DESIGN.md that it has zero + duplication; both are true of `raw/` only. ### Changed - `recall` now lists the user's own words newest first by default, not best match first. Pass `--policy relevance` for the old order. `index.search` is unchanged. +- **Install commands name Python 3.13.** Every documented `uv tool install` + passes `--python 3.13`, so where the machine's own Python is older (Ubuntu + 24.04 ships 3.12) uv builds the tool on 3.13. Left to choose, uv can settle + on the older one and refuse (it does with Python downloads off). pipx is + down to a footnote: it installs into the Python it runs under, so it needs a + 3.13 interpreter. A test reads the floor from `pyproject.toml` and holds + every install line to it. See + [README.md](README.md#quickstart). +- **Sizes for people.** `watch` and `capture` say how much a pass captured as + `+392.0 MB (391995999 B)` instead of `+391995999B`, keeping the exact count in + parentheses; under 1000 bytes it is `+N B`. A script that parsed `+NB` out of + either line needs the new form. `size=` stays an exact offset, and the commit + subject keeps `+NB`. See [USAGE.md](docs/USAGE.md#set-it-up-once). +- The watcher's warning for a root inside the store now reads + `root is inside the store; skipped`. A root that contains the store + still says `contains the store itself`. `gitmemory init` prints the same + sentence as its reason when it refuses such a root. + +### Fixed + +- **`latest-user` no longer takes a subagent's instructions for the user's + words.** A subagent's first turn is user-role text that the parent agent + wrote, its dispatch prompt, and while it was newer than the user's rule it + led the default ahead of that rule. It now ranks with the agent's turns. + `index.search` order is unchanged. ## [0.1.0] - 2026-09-28 @@ -69,5 +167,6 @@ The first public release. What it contains: - Two review rounds before publication: E7 (68 findings) and E7b (22 findings). See [SECURITY.md](SECURITY.md). -[Unreleased]: https://github.com/doronp/gitmemory/compare/v0.1.0...HEAD +[Unreleased]: https://github.com/doronp/gitmemory/compare/v0.2.0...HEAD +[0.2.0]: https://github.com/doronp/gitmemory/compare/v0.1.0...v0.2.0 [0.1.0]: https://github.com/doronp/gitmemory/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8ab1c87..c601e40 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -81,6 +81,22 @@ is supported when `tests/conformance.py::check_adapter` passes on a public corpus of that agent's transcripts. Say in the PR where the corpus comes from and what its licence is. +## Bumping the version + +Claude Code keeps an installed plugin's copy until the plugin's version +changes, so a change under `skills/`, `hooks/` or `.claude-plugin/`, or to +`hook/gitmemory-hook.sh`, reaches nobody who installed the plugin until the +version is bumped. A bump moves five places in one commit: `version` in +`pyproject.toml`, `__version__` in `src/gitmemory/__init__.py`, `version` in +`.claude-plugin/plugin.json`, the release `skills/recall/SKILL.md` says it is +written for, and `uv.lock`, which records the project's version: run `uv lock` +after editing `pyproject.toml`, or CI's `uv sync --locked` fails. The tests pin +the first four equal. The same commit turns CHANGELOG.md's `[Unreleased]` into +the release's heading, with its date and a compare link from the previous tag, +and starts `[Unreleased]`'s link at the new one. Once the commit is on main, +tag it `v` and push the tag: hook/README.md tells users to pin both +installs to that tag, and a pin to a tag that does not exist installs nothing. + ## Pull requests - Keep each PR to one change. Explain the *why* in the description; the diff diff --git a/README.md b/README.md index 9b8141b..d398b40 100644 --- a/README.md +++ b/README.md @@ -59,18 +59,40 @@ dependencies. Requires Python 3.13+ and git. ```sh -uv tool install git+https://github.com/doronp/gitmemory # or: pipx install git+https://… +uv tool install --python 3.13 git+https://github.com/doronp/gitmemory ``` -Tell it what to watch. Watch roots have no default, and the config lives in -`$GITMEMORY_HOME/config.toml` (by default `~/.gitmemory`): +`--python 3.13` has uv build the tool on 3.13, fetching one where the machine's +own Python is older (Ubuntu 24.04 ships 3.12); left to choose, uv can settle on +the older one and refuse. Where uv fetches Python only when asked +(`python-downloads = "manual"`), run `uv python install 3.13` first; where it +never does (`"never"`), put a Python 3.13 on `PATH`.[^pipx] + +[^pipx]: pipx installs into the Python it runs under, so it needs a 3.13 + interpreter: `pipx install --python python3.13 git+https://github.com/doronp/gitmemory`. + +Tell it what to watch. Watch roots have no default, so name the agent: + +```sh +gitmemory init claude-code # or: gitmemory init pi +``` + +That adds these lines to `$GITMEMORY_HOME/config.toml` (by default +`~/.gitmemory`): ```toml +# added by `gitmemory init claude-code` [[watch]] agent = "claude-code" roots = ["~/.claude/projects"] ``` +It appends without changing a byte already there, does nothing if the table is +already there, and refuses a root the watcher would skip. To write the table +yourself, or for another agent or root, +[docs/watching.md](docs/watching.md#configure-what-to-watch) has the rest of +the keys. + Then: ```sh @@ -89,17 +111,27 @@ agent search the store and quote what it finds; the watcher still does the capturing. For watcher flags and troubleshooting, see [docs/watching.md](docs/watching.md). +Budget disk for about twice what the store captures, and more if your +transcripts compress poorly: git keeps a compressed second copy, and the +search index holds the text again. `gitmemory index` rebuilds the index if +you delete it; rebuilding over an existing one briefly needs room for both. +[docs/watching.md](docs/watching.md#how-much-disk-it-takes) has the +measurements. + **What to do with it next** — reading `recall` output, handing it back to an agent after compaction, derived key ideas, the dashboard — is in [docs/USAGE.md](docs/USAGE.md). | Command | What it does | |---|---| +| `gitmemory init ` | Adds the standard watch table for `claude-code` or `pi` to `config.toml`; does nothing if it is already there | | `gitmemory watch` | Tails the configured roots, captures what grew, and commits | +| `gitmemory service install` | Writes a systemd user unit (Linux) or a LaunchAgent (macOS) that keeps `watch` running, and prints the line that starts it; it starts nothing itself ([docs/watching.md](docs/watching.md#keep-it-running)) | | `gitmemory capture ` | Copies out one transcript's new bytes, by hand | | `gitmemory verify` | Checks every manifest's contiguity proof | | `gitmemory index` | Rebuilds the SQLite FTS5 index from the store | -| `gitmemory recall ""` | Searches the index and prints one line per turn, dated, the user's own words newest first ([`--policy`](docs/USAGE.md#when-memories-disagree)) | +| `gitmemory recall ""` | Searches the index and prints one line per turn, dated, the user's own words newest first ([`--policy`](docs/USAGE.md#when-memories-disagree); [`--redact`](docs/USAGE.md#masking-credentials-in-what-it-prints) masks credentials) | +| `gitmemory show
` | Prints the whole turn at an address `recall` printed, read from the store; `--context N` adds N turns either side, `--redact` masks credentials ([more](docs/USAGE.md#read-the-whole-turn)) | | `gitmemory derive [--graph]` | Rebuilds key ideas and a timeline; `--graph` adds the decision graph (opt-in) | | `gitmemory dashboard` | Serves the index with Datasette, read-only, on loopback, behind a sign-in | | `gitmemory push` | Runs the redaction gate over what a push would send (it does not send yet) | @@ -107,7 +139,7 @@ agent after compaction, derived key ideas, the dashboard — is in Optional extras: `hybrid` (what the benchmarks' dense and rerank arms need; `recall` does not use them), `derive` (key ideas and the graph), and `serve` (the dashboard). Example: -`uv tool install "gitmemory[serve] @ git+https://github.com/doronp/gitmemory"`. +`uv tool install --python 3.13 "gitmemory[serve] @ git+https://github.com/doronp/gitmemory"`. ## Results @@ -185,13 +217,13 @@ real text, and that is a measurement rather than a suspicion. | What | How it was measured | Result | |---|---|---| | Hook cost in the agent's critical path | Timed against spawning `true` the same way, three runs of 400 | p50 **7.4 – 7.5 ms**, p99 **10.2 – 11.5 ms** | -| The suite | On a fresh checkout, no downloads | **1,037 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | -| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **606** negative controls | +| The suite | On a fresh checkout, no downloads | **1,134 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | +| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **662** negative controls | The two suite counts do not add up, and should not. Switching the corpora on -collects 1362, not 1365. Three conformance cases fill parametrisations that +collects 1459, not 1462. Three conformance cases fill parametrisations that collect as one empty placeholder each while the corpora are absent, so they -replace three of the 1037 rather than joining them. Every figure on this board +replace three of the 1134 rather than joining them. Every figure on this board is pinned by a test, which is how it stays true. **What is not measured**, and is shown as a row on the dashboard rather than diff --git a/SECURITY.md b/SECURITY.md index 8cd4d1d..24168a0 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -38,7 +38,8 @@ maintained back-branches. In scope: anything in this repository that runs on a user's machine. That means capture, the store and its contiguity proof, `verify`, the redaction gate -behind `push`, the index, `derive`, the dashboard, the watcher, the hook shim and the plugin's +behind `push`, the masking behind `recall --redact` and `show --redact`, the +index, `derive`, the dashboard, the watcher, the hook shim and the plugin's recall skill. Worth reporting, for example: @@ -47,7 +48,9 @@ Worth reporting, for example: `derive` crash, hang, write outside `$GITMEMORY_HOME`, or cost superlinear time - a way to make `verify` accept a store whose segments do not tile, or whose bytes do not match the recorded digest -- a secret that the redaction gate lets through on `push` +- a secret that the redaction gate lets through on `push`, or that + `recall --redact` or `show --redact` prints unmasked, of a shape its + detectors are meant to catch - the dashboard being reachable without the sign-in URL, or writable Out of scope: vulnerabilities in the third-party dependencies themselves @@ -90,7 +93,16 @@ that gets waved through on a deadline. The gate stands at `push` and nowhere else. The Claude Code plugin's recall skill reads the store into the agent's context, so each line it quotes goes to -the model provider with the rest of the conversation, unscanned. That includes -text another watched agent never sent to that provider, and a credential `push` -would refuse to ship. The skill quotes what the question needs and no more, -which is a rule it follows, not a gate. +the model provider with the rest of the conversation, unscanned unless you turn +on the masking below. That includes text another watched agent never sent to +that provider, and a credential `push` would refuse to ship. The skill quotes +what the question needs and no more, which is a rule it follows, not a gate. + +`recall --redact` and `show --redact` mask the gate's shapes in what those two +commands print. `GITMEMORY_REDACT` set to anything but empty or `0` turns it on +for both, and the skill then passes the flag to both and reads nothing under +the store by hand. It masks and never refuses, and it is off unless you turn it +on. What it does not guarantee is listed in +[USAGE.md](docs/USAGE.md#what-it-does-not-guarantee): it masks sixteen shapes +and nothing else, decodes nothing, and covers only what those two commands +print. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 9014a8a..c71a710 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -67,6 +67,7 @@ right of `git` can be deleted and rebuilt, so `index/` is gitignored and |---|---|---|---| | Trigger | [`hook/gitmemory-hook.sh`](../hook/gitmemory-hook.sh) | Writes the hook's stdin to one spool file and exits 0. No git, no Python, no network. It adds latency, not correctness | §2.3 | | Trigger | [`daemon.py`](../src/gitmemory/daemon.py) | The watcher. It tails the configured roots by `(path, inode, offset, mtime)`, drains the spool, captures and commits. This is what makes capture a guarantee | §2.3 | +| Trigger | [`service.py`](../src/gitmemory/service.py) | Writes the systemd user unit or LaunchAgent that keeps the watcher running, with this Python, the store and git's directory spelled out in it. It never starts one | §2.3 | | Parse | [`adapters/`](../src/gitmemory/adapters/) | Turns agent-native bytes into canonical records. An adapter is two symbols, `AGENT` and `parse(path)`, and it may not touch git, the index or derivation | §2.1–2.2 | | Parse | [`records.py`](../src/gitmemory/records.py) | Canonical `Session` / `Turn` / `Block` / `Event`, with content-derived IDs and sorted-key JSON, so the committed tree stays diffable | §2.1 | | Parse | [`jsonl.py`](../src/gitmemory/jsonl.py) | A streaming JSONL reader with exact byte offsets, vendored from fable (MIT) | §2.5b | @@ -74,9 +75,9 @@ right of `git` can be deleted and rebuilt, so `index/` is gitignored and | Store | [`gitrepo.py`](../src/gitmemory/gitrepo.py) | The only module that runs git, always as `git -C ` | §2.4 | | Derive | [`index.py`](../src/gitmemory/index.py) | A SQLite FTS5 (BM25) index rebuilt from raw bytes. It is never authoritative | §2.7 | | Derive | [`derive.py`](../src/gitmemory/derive.py), [`graph.py`](../src/gitmemory/graph.py) | Key ideas and a timeline. The decision extraction behind the graph is opt-in (`--graph`) | §2.6 | -| Surface | [`__main__.py`](../src/gitmemory/__main__.py) | The CLI: `capture watch verify push index derive dashboard recall` | §2.8 | +| Surface | [`__main__.py`](../src/gitmemory/__main__.py) | The CLI: `init capture watch service verify push index derive dashboard recall show` | §2.8 | | Surface | [`dashboard.py`](../src/gitmemory/dashboard.py) | Datasette over the index, `--immutable`, on loopback, behind a single-use sign-in URL | §2.9 | -| Egress | [`redact.py`](../src/gitmemory/redact.py) | The only place secrets are looked for. It stands at `push` and scans the files, the segment seams and the object graph a push would transmit | §2.4 | +| Egress | [`redact.py`](../src/gitmemory/redact.py) | The only place secrets are looked for. It stands at `push` and scans the files, the segment seams and the object graph a push would transmit. `recall --redact` and `show --redact` use its detectors to mask what they print | §2.4 | ## On disk @@ -92,13 +93,14 @@ $GITMEMORY_HOME/ # default ~/.gitmemory; a git repository index/ # gitignored: SQLite is not diffable spool/ # gitignored: one file per hook fire .locks/ # gitignored: one per session - config.toml # watch roots; there is no default + config.toml # watch roots; no default (`init` writes one) ``` A capture commits **only the bytes appended since the last recorded offset**, -so the store holds each byte the agent wrote exactly once. If the source is -rewritten (its prefix no longer matches), capture starts a new *generation* -`g` rather than treating the rewrite as corruption. §2.5a. +so `raw/` holds each byte the agent wrote exactly once; git's object store keeps +its own compressed copy ([how much disk](watching.md#how-much-disk-it-takes)). +If the source is rewritten (its prefix no longer matches), capture starts a new +*generation* `g` rather than treating the rewrite as corruption. §2.5a. ## Trust boundaries diff --git a/docs/DESIGN.md b/docs/DESIGN.md index df5cfb8..f9fb892 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -32,6 +32,9 @@ Two properties, and they are the whole product: **No data from this machine is ingested, copied, or committed. Ever.** - Test corpora are synthetic or public-benchmark-derived. - Userspace only: no `crontab`, no `sudo`, no Docker. LaunchAgent is the keep-alive. + **[2026-10-18 — and on Linux a systemd user unit. `gitmemory service install` writes either + one, as the user, and starts neither: enabling it is the user's step, as installing the hook + is.]** - Local-first. No network by default. No remote configured by default. - Apache-2.0. @@ -199,6 +202,10 @@ $GITMEMORY_HOME/ # default ~/.gitmemory Each capture commits **only the bytes appended since the last recorded offset**. Consequences: 1. **Zero duplication.** Stored bytes == bytes the agent ever wrote, not a multiple. + **[2026-10-18 — true of `raw/`, not of the store on disk. `.git/` keeps a zlib-compressed copy + of the same bytes and `index/` holds their text again, so budget about twice what you capture, + more if it compresses poorly; [watching.md](watching.md#how-much-disk-it-takes) has the + measurements.]** 2. **Plain text, never pre-compressed.** Let git's zlib + delta work on text it understands. Pre-compressing defeats git's native packing. 3. **Reconstruction is `cat` in filename order**, checked against a recorded whole-file sha256. @@ -212,6 +219,13 @@ coalescing only changes where the cut points are, never whether they tile. **Raw is committed locally, always** — that is the thesis. Redaction applies at exactly two boundaries: anything written to `derived/`, and anything **`gitmemory push`** sends. **`push` is opt-in per remote and refuses unless the redaction gate passed.** Default config has no remote. +**[2026-10-18 — `recall --redact` and `show --redact` are a third point, and they mask; they are +not a gate. Off by default, because a store on the owner's own disk is still not an egress. The +owner turns it on, or makes it standing with `GITMEMORY_REDACT`, when their output goes on to a +model provider through the plugin's skill. Both tiers are masked, unlike `derived/` below, because a +mask costs the span and not the sentence. Neither refuses anything, and `raw/` and the index stay +verbatim. [USAGE.md](USAGE.md#masking-credentials-in-what-it-prints) says what is masked, and +what is not.]** Not "anything that leaves the machine" — that claim was wider than the code and is withdrawn **[E7]**. The store is an ordinary git repository in a directory the owner can read. A `git push` @@ -225,8 +239,8 @@ deleted is absent from the first two and present in the third **[E7]**. A startup assertion refuses to run if `$GITMEMORY_HOME` is inside another work tree, and every git call is explicit `git -C $GITMEMORY_HOME` — never cwd-inherited **[R1]**. -The `derived/` boundary acts on the **HIGH tier only** — the nine issuer-anchored shapes, not the -three SUSPECT ones. A sentence carrying a high-confidence key is dropped before ranking and +The `derived/` boundary acts on the **HIGH tier only** — the twelve issuer-anchored shapes, not +the four SUSPECT ones. A sentence carrying a high-confidence key is dropped before ranking and counted in `sentences_redacted`; `derive._write` then refuses outright to publish a payload that still contains one, because an artifact reaching that door with a key in it is a bug upstream, not a sentence to quietly lose. SUSPECT is excluded deliberately: it is `password: "..."`-shaped and @@ -625,8 +639,14 @@ answers the token-burn objection.)* **UNVERIFIED, must be tested before claiming coexistence:** whether multiple `UserPromptSubmit` hooks' `additionalContext` outputs concatenate or last-wins. -**As built:** the CLI only. There is no MCP server and no `UserPromptSubmit` hook in `src/`, so -nothing is injected; injection cost is carried as `NOT BUILT` on the dashboard (§2.9). +**As built:** the CLI, and the Claude Code plugin's recall skill (`skills/recall/SKILL.md`, +`/gitmemory:recall`), which has the agent run `recall` and `show` itself and quote what they +print with each line's address. The skill is a search the agent runs when the user refers to an +earlier decision or runs the skill, not an injection: there is no MCP server and no +`UserPromptSubmit` hook in `src/`, so nothing is injected; injection cost is carried as +`NOT BUILT` on the dashboard (§2.9). `recall` prints each hit's address, and `show` resolves one +to the whole turn from the store, not the index, so a stale or tampered index cannot change what +the address is shown to say. ### 2.9 Dashboard diff --git a/docs/README.md b/docs/README.md index d15a9d6..4ecf443 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,7 +9,7 @@ | [USAGE.md](USAGE.md) | Getting value out of it: the daily loop, reading `recall`, handing memory back to an agent | | [RESULTS.md](RESULTS.md) | Every published result, including the ones it loses | | [REPRODUCE.md](REPRODUCE.md) | The command behind each result, what it needs, and how closely a rerun matches | -| [watching.md](watching.md) | Configuring `gitmemory watch`: roots, flags, and telling a misconfigured watcher from an idle one | +| [watching.md](watching.md) | Configuring `gitmemory watch`: roots, flags, telling a misconfigured watcher from an idle one, keeping it running as a user service, and how much disk the store takes | | [../hook/README.md](../hook/README.md) | Installing the optional hook shim, and its measured cost | | [agents.md](agents.md) | Which agents can be captured, which cannot, and why | diff --git a/docs/REPRODUCE.md b/docs/REPRODUCE.md index 98d17a4..e044edc 100644 --- a/docs/REPRODUCE.md +++ b/docs/REPRODUCE.md @@ -475,26 +475,34 @@ which is the load and not the shim. conformance cases the three third-party corpora add. ```sh -# the fresh-checkout count: point both corpus variables at nothing +# the fresh-checkout count: point both corpus variables at nothing, and HOME +# at a new directory +HOME=$(mktemp -d)/isolated-home \ GITMEMORY_CC_FIXTURES=$PWD/no-such-corpus GITMEMORY_PI_FIXTURES=$PWD/no-such-corpus \ uv run pytest -q --collect-only -p no:cacheprovider | tail -1 # with the corpora cloned by tests/fetch_fixtures.sh -uv run pytest -q --collect-only -p no:cacheprovider | tail -1 +HOME=$(mktemp -d)/isolated-home uv run pytest -q --collect-only -p no:cacheprovider | tail -1 uv run pytest -q --collect-only -p no:cacheprovider | grep -c '\.jsonl\]$' ``` -**Expected:** `1027/1028 tests collected (1 deselected)`, then -`1352/1353 tests collected (1 deselected)`, then `328`. The deselected test is +**Expected:** `1134/1135 tests collected (1 deselected)`, then +`1459/1460 tests collected (1 deselected)`, then `328`. The deselected test is the LongMemEval corpus test, which is opt-in: `uv run pytest -q -m corpus` after `bench/fetch_longmemeval.sh`. The README's -paragraph on why 1,027 + 328 is 1,352 and not 1,355 is the arithmetic of these +paragraph on why 1,134 + 328 is 1,459 and not 1,462 is the arithmetic of these three numbers. All three were run while writing this guide. -`tests/test_docs.py` holds the README to the first and third numbers, so a -drifted count fails the suite. To run the suite rather than count it: -`uv run pytest -q`, which CONTRIBUTING.md gives as about a minute and a half, -no network. +The `HOME` is the one the suite gives itself. The owner-data scan adds two +cases named after your account, the last part of `$HOME`, and leaves them out +when that name is shorter than four characters or a generic one such as +`ubuntu`, `runner` or `root`. Without it, such an account collects two fewer. + +`tests/test_docs.py` holds the README to all three numbers, the second and +third only where the corpora are cloned, and this guide, §5c's figure +included, to the README, so a drifted count fails the suite. To run the suite +rather than count it: `uv run pytest -q`, which CONTRIBUTING.md gives as about +a minute and a half, no network. ### 5c. Negative controls @@ -505,7 +513,7 @@ test fail. The count is the number of rows in `tests/mutate_index.py`. uv run python -c "import sys; sys.path.insert(0, 'tests'); from mutate_index import MUTANTS; print(len(MUTANTS))" ``` -**Expected:** `606`. Checked while writing this guide. +**Expected:** `662`. Checked while writing this guide. To run the controls rather than count them: @@ -518,7 +526,8 @@ uv run python tests/mutate_index.py "verify" # only mutants whose name contain and `bench/` in place and restores each one after its row, so run it on a clean checkout that nothing else is editing. Its docstring estimates a full pass at one to two and a half hours, from three rows timed at 3.7 s, 17.1 s and -17.2 s. It exits 0 only if every mutant is caught by its intended test. +17.2 s on a smaller suite. It exits 0 only if every mutant is caught by its +intended test. ## What cannot be reproduced exactly, and why diff --git a/docs/USAGE.md b/docs/USAGE.md index be113d2..dbefd85 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -25,18 +25,30 @@ repository, and lets you search it. So: ## Set it up once -1. Install and configure, as in the [README quickstart](../README.md#quickstart). +1. Install, as in the [README quickstart](../README.md#quickstart), then name + what to watch: + + ```sh + gitmemory init claude-code # or: gitmemory init pi + ``` + Watch roots have no default; `config.toml` names each agent and where its - transcripts live. [agents.md](agents.md) lists what can be captured. + transcripts live, and `init` adds that table for you. [agents.md](agents.md) + lists what can be captured. 2. Run the watcher and leave it running: ```sh gitmemory watch ``` - How it stays running (a LaunchAgent, a systemd user unit, a terminal you - leave open) is your choice; [watching.md](watching.md#run-it) covers the - flags and how to tell a misconfigured watcher from an idle one. + How it stays running is your choice. `gitmemory service install` writes a + systemd user unit (Linux) or a LaunchAgent (macOS) that starts it at every + login, and prints the line that starts it now; it starts nothing itself. On + a Linux machine you reach over SSH, the `loginctl enable-linger` line it + prints keeps it running after you log out + ([watching.md](watching.md#keep-it-running)). A terminal you leave open + works too. [watching.md](watching.md#run-it) covers the flags and how to + tell a misconfigured watcher from an idle one. 3. Optional: install the [hook shim](../hook/README.md) so a capture happens at the compaction boundary itself rather than at the next sweep. The watcher alone is enough for correctness. @@ -45,7 +57,7 @@ To check the loop end to end without waiting, run one pass: ```console $ gitmemory watch --once -captured 1 +1167B commit=f355b10ace4b872ffe77794cbf4183afb1dcf478 +captured 1 +1.2 kB (1167 B) commit=f355b10ace4b872ffe77794cbf4183afb1dcf478 $ gitmemory verify 0 problem(s) ``` @@ -75,10 +87,111 @@ newest first (see [When memories disagree](#when-memories-disagree)): | `2026-09-28T10:03Z` | When the turn was written, in UTC. `undated` if the transcript did not say | | `claude-code//g00` | Agent, session and generation. A new generation starts only when the source file was rewritten | | `@288` | The byte offset of the turn inside that generation's raw bytes | -| `assistant/text` | Role and block kind | +| `assistant/text` | Role and block kind. `subagent:` before them, as in `subagent:user/text` or `subagent:assistant/text`, marks a subagent's or workflow agent's turn, which the transcript marks `isSidechain`: its user text is the instructions your agent gave it, not your words | `-k` sets how many lines come back. +If the [hook](../hook/README.md) has rung and no watcher has collected the +record for an hour, `recall` says so on stderr, before the results it still +prints: +``1 hook record(s) uncollected, the oldest for 3 hours: is `gitmemory watch` running for ? These results may be missing everything said since.`` +A running watcher drains the spool on every pass, so none has run on that store +since: start `gitmemory watch` with the hook's `$GITMEMORY_HOME`, and +`gitmemory service install` to keep it running +([watching.md](watching.md#keep-it-running)). [Check it](watching.md#check-it) +covers the other causes, such as a watcher that runs passes less often than +hourly. With `--db`, recall does not look. + +## Read the whole turn + +A line stops at 160 characters. `show` prints the whole turn at an address, +read from the store rather than the index, so it needs no index and a stale +one cannot change what it prints. Copy the address whole: the ones above are +cut short with `…` to fit this page, and a shortened address is refused with +`not an address` and exit 2. + +```console +$ gitmemory show claude-code/0f3a9c2e-1b2d-4c5e-8f90-3a53cc4a5233-fc59494095426389/g00@288 --context 1 +- claude-code/0f3a9c2e-1b2d-4c5e-8f90-3a53cc4a5233-fc59494095426389/g00@0 2026-09-28T10:02Z user + text + Don't add a retry loop around the upload; the API is idempotent only per request id. +> claude-code/0f3a9c2e-1b2d-4c5e-8f90-3a53cc4a5233-fc59494095426389/g00@288 2026-09-28T10:03Z assistant + text + Understood. I dropped the retry loop and pass a request id instead, so a replay cannot double-charge. +- claude-code/0f3a9c2e-1b2d-4c5e-8f90-3a53cc4a5233-fc59494095426389/g00@592 2026-09-28T10:04Z user + text + Good. Ship it. +``` + +Each turn starts with a header: `>` for the turn at the address, `-` for the +neighbours `--context N` (or `-C N`) adds either side, then the turn's own +address, date and role. A subagent's turns say `subagent:user` and +`subagent:assistant`. Under the header comes each block's kind, with the tool's +name for a tool call, and then its text, indented. Only headers start in +column 0, so nothing in a transcript can print a line that reads as one. + +An offset inside a turn shows that turn, and its header gives where it starts. +An address the store does not hold (`not in the store`, `no turn at byte N`) +exits 1. `--redact` masks credentials in what it prints +([below](#masking-credentials-in-what-it-prints)). + +## Masking credentials in what it prints + +`recall` and `show` print what the transcript says, credentials included, +because the store is yours and on your disk. When their output is going +further, to an agent and with it to the agent's model provider, `--redact` +masks every shape the push gate knows, in the text and in the address, with +one exception for a session id ([below](#what-it-does-not-guarantee)). That is +twelve issuer-prefixed keys and four SUSPECT patterns, listed in +[`redact.py`](../src/gitmemory/redact.py). This sample is from a second +transcript, a one-turn session of its own: + +```console +$ gitmemory recall --redact "staging key" + -0.000 2026-10-02T09:41Z claude-code/7c41d2e9-…-61a9a184ea213ec3/g00@0 user/text Use AKIA…[20 bytes] for the staging key, never the prod one. +``` + +A masked span keeps its first four bytes and its length; in an address the +marker drops its space, so the address stays one field of a `recall` line or +a `show` header. A private key is masked from its BEGIN line to its END line, +or to the end of the text when there is none. Both commands mask the text as +written, and also find a header that a line break split, as a flattened line +would show it; `recall` then flattens its line and cuts it at 160 characters. +`show --redact` masks each block of the turn whole before it splits it into +lines, so a key's body goes with its header, and it masks a tool's name too. +A masked address is not one `show` can look up: it refuses it, exit 2, and +`recall` prints the full address when it is not masking. + +`GITMEMORY_REDACT=1` makes it standing for both: unset, empty or `0` is off, +and any other value is on. There is no flag that turns a standing setting off; +for one call, run `GITMEMORY_REDACT=0 gitmemory recall ...`. + +### What it does not guarantee + +- It masks only those sixteen shapes. Anything else prints as written: a bare + AWS secret key, `PGPASSWORD=…`, a custom `GH_TOKEN=…`, `glpat-` and `npm_` + tokens, an Azure `AccountKey=…`, a password in prose, `curl -u user:pass`, a + `.netrc` line. +- Nothing is decoded. Base64, URL-encoding and a token wrapped across lines get + through. JSON `\u` escapes are masked, because both commands read the parsed + text, not the raw bytes. +- It masks credentials; it does not make the output confidential. Hostnames, + customer data, personal data and code are untouched. +- It covers only what `recall` and `show` print. `raw/`, the index database, + the dashboard, `derived/` and any file an agent opens are not masked by it. + The environment setting is a floor, not a boundary: an agent with a shell can + unset it, or read `raw/`. +- A private key whose BEGIN line and body are in two blocks of one turn, such + as two tool results, is masked to the end of the first block only; `show` + prints the body under the second. +- The store lowercases session ids, so an AWS key id, a Google API key or a + JWT used as a session id, by `capture --session-id` or as a transcript's + file name, prints in the address lowercased and unmasked: those detectors + match only the case the issuer writes. The manifest's `source_path` keeps + it as it was. +- A mask shows the first four bytes and the length. +- The SUSPECT patterns also mask some prose about configuration. + ## When memories disagree A decision made three weeks ago and reversed last week are both in the store, @@ -88,7 +201,7 @@ tends to act on. | Policy | Order | Use it when | |---|---|---| -| `latest-user` (default) | The user's own words newest first, then the agent's turns and tool output newest first | The user's later word should win, and nothing else should | +| `latest-user` (default) | The user's own words newest first, then the agent's turns and tool output newest first. A subagent's instructions are the agent's words | The user's later word should win, and nothing else should | | `latest` | Newest first, whoever said it | Any later turn should win, the agent's included | | `relevance` | Best BM25 match first | You want the agent to decide: every line carries its date, so it can see which turn came later | | `module:function` | Whatever your function returns | Anything else: pinned decisions, a cut-off date, a different rule per project | @@ -116,8 +229,9 @@ If two lines disagree, the later date is the current decision; say so. A policy is a function from a list of `Hit`s to a list of `Hit`s. It can reorder and it can drop; it gets nothing that is not already in the store. A -`Hit` has the fields `recall` prints, plus `text` and `ts`, and -`gitmemory.index.when(hit)` reads `ts` as a UTC `datetime`. +`Hit` has the fields `recall` prints, plus `text`, `ts` and `is_sidechain` +(true for a subagent's turn), and `gitmemory.index.when(hit)` reads `ts` as a +UTC `datetime`. A validity window, where everything before a date you name is out of date, is one such rule: @@ -140,14 +254,32 @@ def since_reversal(hits): $ PYTHONPATH=. gitmemory recall "upload retries" --policy myrules:since_reversal ``` +Leaving subagents' turns out when you read, rather than never capturing them +([watching.md](watching.md#main-sessions-only-claude-code)), is another rule +for the same file: + +```python +def main_only(hits): + """Drop every subagent's turn, then order as the default does.""" + return latest_user([h for h in hits if not h.is_sidechain]) +``` + +A policy that drops hits prints fewer than `-k` lines, because it sees only the +top `-k`. Raise `-k` to make up for what it drops: + +```console +$ PYTHONPATH=. gitmemory recall "upload retries" -k 30 --policy myrules:main_only +``` + The module is imported and runs with your permissions, so name only code you would run anyway. A name that does not resolve is an error, not a silent fallback to the default. The offset is the point: it takes you back to the exact bytes, not to a -paraphrase of them. The raw segments live under -`raw///g/`, named by the byte range they hold, so a turn at -offset 288 is in the segment whose range contains 288. +paraphrase of them. [`show`](#read-the-whole-turn) finds them for you. To check +it by hand, the raw segments live under `raw///g/`, named +by the byte range they hold, so a turn at offset 288 is in the segment whose +range contains 288. ## Put it back in front of the agent @@ -171,7 +303,24 @@ decision or a rule from before a compaction or from an earlier session, or that you run with a topic: `/gitmemory:recall retry loop`. It runs `recall`, and `index` when there is no index yet or the one there is older than what you are asking about. It quotes each line with its address and says when a hit comes -from another session. It writes nothing but the index. +from another session. It writes nothing but the index. It runs +`gitmemory --version` first and says when the CLI is older than the plugin; +the [hook's README](../hook/README.md#as-a-claude-code-plugin) has how to hold +the two at one release. To have it mask credentials +([above](#masking-credentials-in-what-it-prints)), set the variable in Claude +Code's own environment, in `~/.claude/settings.json`: + +```json +{ + "env": { + "GITMEMORY_REDACT": "1" + } +} +``` + +The skill then passes `--redact` to every `recall` and `show` it runs, so a +CLI too old to know the flag stops with an error instead of printing unmasked, +and it reads nothing under the store by hand. ## Derive key ideas and decisions @@ -221,6 +370,11 @@ finding, and there is no override flag. It does not send yet. If a credential has already landed in the store, [SECURITY.md](../SECURITY.md#if-a-credential-lands-in-the-store) says what to do. +`recall` and `show` output that an agent reads goes to that agent's model +provider, and no gate stands in that path. `--redact` masks the gate's shapes +in it ([above](#masking-credentials-in-what-it-prints)); it masks, and it +never refuses. + ## What it does not do - **It does not summarise for you.** There is no LLM in the product. `recall` diff --git a/docs/reviews/install-feedback-round.md b/docs/reviews/install-feedback-round.md new file mode 100644 index 0000000..d0517be --- /dev/null +++ b/docs/reviews/install-feedback-round.md @@ -0,0 +1,274 @@ +# Install-feedback round: the Kimi K3 cross-review + +The round answered ten items of feedback from a first install, on one branch +off `main`, one signed-off commit per item and then the 0.2.0 release commit. +Two channels reviewed it: + +- **An independent Opus reviewer for each item commit**, with no part in + writing it. Its findings were fixed in that item's commit before the item was + integrated, so they are not repeated here. +- **Kimi K3, as a pair programmer, on each item commit.** One pack per item, + `git diff -U8 main...`, and one review per pack. The redaction item was + split into a code pack and a docs pack to fit. + +**Kimi saw each commit alone**, against `main` and before integration, and +some of its findings come from exactly that. It called the redaction docs +unbacked by code, and the code unbacked by docs, because each half of the +split pack held what the other said was missing. It read a fourteen-test jump +in `docs/REPRODUCE.md` as unexplained, when `main`'s copy was already ten +behind the README. It could not see `main()`'s exception handler, the +`## Echoes` heading or `tests/conftest.py`, because none of them was in the +hunk. Each is recorded below with what it turned out to be, not dropped. + +Every finding was re-checked against the integrated branch at `7a10f22` before +anything changed. The lead refuted the first batch from a code read or a live +probe. A skeptic then took each of the rest and tried to break it, with a probe +wherever one could run. One finding, install-3, was on neither list, and was +checked while the fixes were applied. The fixes landed in one commit on top, +the one that adds this file. + +## The ten items + +| Item | What | Commit | +|---|---|---| +| 1 | Every install command names `--python 3.13` | `59865e1` | +| 7c | `gitmemory --version`; the recall skill recognises an older CLI | `56327cf` | +| 5 | latest-user stops taking a subagent's prompt for the user's words | `656136c` | +| 2 | `gitmemory init ` writes the standard `[[watch]]` table | `beffcf2` | +| 3a | `gitmemory service install` writes a user unit or LaunchAgent | `b39449a` | +| 3b | `recall` says when hook records have waited an hour for a watcher | `5c6a874` | +| 4 | `gitmemory show` prints the turn at a recall address | `dba5db7` | +| 6 | `recall --redact` and `show --redact` | `b9a5f57` | +| 7a | Captured sizes for people, with the exact count kept | `3bb0aff` | +| 7b | How much disk the store takes, and why | `0434189` | +| | Release 0.2.0 | `7a10f22` | + +## Status + +**53 findings: 22 fixed, 6 declined, 25 refuted.** *Fixed* means something +landed: code, test or text. *Declined* means the claim is true and not worth +the change, with the reason written down. *Refuted* means the claim does not +hold on the integrated branch. + +| Severity | Raised | Fixed | Declined | Refuted | +|---|---|---|---|---| +| High | 5 | — | — | 5 | +| Medium | 13 | 7 | — | 6 | +| Low | 35 | 15 | 6 | 14 | +| **Total** | **53** | **22** | **6** | **25** | + +All five Highs are artifacts of the commit seen alone: four from the split +redaction pack, and one (the detector count, redact-docs-4) a correction of a +number `main`'s DESIGN.md had wrong. Fifteen of the 31 that changed nothing +rest on something outside the hunk Kimi was given. service-3 was refuted here +and is counted fixed: Kimi raised it again in its final pass, and the +[final gate](#final-gate) took the reload. + +One change of behaviour came out of the round: two `gitmemory init`s run at +once could lose a table silently (init-1). It carries a test that failed +first and a new negative control, which brings the index to **658**. The rest +are one reworded message (stale-1), one help string (redact-code-3), tests, +wording and pinned numbers. + +## Findings + +| ID | Sev | Claim | Disposition | Evidence or reason | +|---|---|---|---|---| +| install-1 | med | The floor test skips `` `uv tool install` git+… ``, a command split across spans | fixed | Probed: such a line in hook/README.md passed. A span holding the tool alone now reads on when the next word is a source or a flag. Kimi's "rest of the line" rule would have failed three prose lines in hook/README.md | +| install-2 | low | A `#` ends a command, so a `#egg=` fragment hides a later `--python` | declined | Fails loud, with a message naming the line. Documented in the test. No scanned command has a fragment | +| install-3 | low | The `derive` install hint in derive.py has no mutation row | declined | The floor test scans every tracked file: with `--python 3.13` removed from the hint, it fails on `derive.py:142`. Rows are for fixes, and item 1 was docs | +| install-4 | low | A trailing `\` joins prose lines into one command | declined | Documented in both docstrings. Every Markdown line ending in `\` is a fenced shell continuation, and a join can only add a failure, never hide one | +| version-1 | med | The skill's "step 1" does not exist | refuted | The numbered setup list is in SKILL.md; step 1 is the install line (lead) | +| version-2 | med | Rerunning `uv tool install` does not move an install forward | refuted | Live probe, uv 0.11.3: rerunning the unpinned line replaced the install's commit with the remote's tip; `uv tool upgrade` said "Nothing to upgrade" (lead) | +| version-3 | low | A fix with no mutation row | refuted | A `feat:` adding one flag; rows are required for fixes. Deleting the flag fails two skill tests, measured | +| subagents-1 | med | A doc test writes into the real `~/.claude/projects` | refuted | conftest's autouse `_no_real_home` points HOME at `tmp_path` for every test (lead) | +| subagents-2 | low | REPRODUCE's counts jump by 14 for 4 tests | fixed | The jump was `main` already ten behind the README. The real gap is that nothing pinned the guide, so a test now holds it to the README | +| subagents-3 | low | The skill never says what to do when `subagent:` lines crowd the results | fixed | Probed: 20 subagents hid the user's rule at `-k 10` and `-k 30`, and it came first at `-k 50`. The skill now says to search again with `-k 50`. `main_only` is a user-written recipe, so the skill does not point at it | +| subagents-4 | low | A docstring claims "byte-for-byte" and the test checks one field | fixed | Probed: a patched address and text still passed. The docstring now says what is asserted | +| init-1 | med | Read, then rename: an edit between them is lost | fixed | The racer that hits it is a second `init`: 46 of 60 concurrent pairs lost a table, both saying "added". `init` now re-reads under a lock before the rename and refuses, exit 1, if the file moved. 0 losses in 60 since. An editor takes no lock and is not covered, as watching.md now says | +| init-2 | low | The temp-name test checks a typed name, not the writer's | fixed | Probed: it passed with the writer's temp renamed. The test now asks `git check-ignore` about the temp the writer really made, while it exists | +| init-3 | low | The invalid-TOML refusal prints the raw exception | refuted | `push` and the watcher print the same error the same way. tomllib quotes key names and single characters, never a value, and the credential is a value | +| init-4 | low | The refusal could print an empty reason | declined | Every skip a written table can reach logs with the prefix the filter wants. A fallback would only guard a future watcher bug | +| service-1 | med | Other control characters make an invalid plist and unpasteable commands | refuted | plistlib refuses them before anything is written (exit 2). The legal ones pass `plutil -lint`. The printed commands hold no store path | +| service-2 | low | USAGE says the unit keeps the watcher running across logins | fixed | Without linger, systemd stops user units at the last logout. Reworded to "at every login", with the `loginctl enable-linger` line for SSH-only machines | +| service-3 | low | The removal line lacks `daemon-reload` | fixed | On the second report (final gate, F2). First refuted from systemd's documentation, not a probe: `disable` reloads, and a stopped, disabled unit is garbage-collected. The reload is harmless and conventional, so the printed removal now ends with one | +| service-4 | low | `main()` may not map ValueError and RuntimeError to exit 2 | refuted | It does (lead) | +| stale-1 | med | "N records uncollected for " gives the oldest's age to all N | fixed | Probed: a record from seconds earlier was reported 3 days old. Now `N hook record(s) uncollected, the oldest for `, in the code, the skill, USAGE and watching.md | +| stale-2 | low | REPRODUCE's counts jump by 13 for 3 tests | fixed | As subagents-2. One fix serves both; see below | +| stale-3 | low | The skill test assumes `resolve_home` is the identity on a relative path | refuted | Every sibling message also goes through `resolve_home`, and pytest's `tmp_path` is already resolved | +| stale-4 | low | The `_ago` boundaries and the `>= STALE_RECORD` edge are unpinned | fixed | The 2-hour and 2-day edges are now pinned 30 s either side. Probed: four mutants of them that had survived are caught. The `>=` edge is left: it differs only at exactly 3600.0 s, and pinning it needs an injected clock | +| show-1 | med | Store-read errors escape `show`'s exit contract | refuted | `main()` turns OSError, ValueError, RuntimeError and sqlite3.Error into a masked `error:`, exit 2 (lead) | +| show-2 | low | mutate_index's docstrings keep stale counts | fixed | "the other 567", "955-test", "1280-test". The number is gone, and the timings are dated as taken when the suites were 955 and 1280 tests, not re-pinned to today's | +| show-3 | low | Role and kind are printed uncollapsed | refuted | Both adapters write only literal roles and kinds (lead) | +| show-4 | low | The skill's echo rule also skips a real turn that quotes an address | fixed | Probed: read literally, it skipped the user's newer reversal. Now the header's shape, marker then address then date, and a test pins the wording | +| show-5 | low | The `## Echoes` heading may not exist | refuted | It is on `main` (lead) | +| show-6 | low | `g5` and `@0007` are accepted | declined | Probed: same output as the canonical address, and the header prints the canonical one. The address never becomes a path | +| show-7 | low | The success path prints a credential-shaped session id unmasked | refuted | It does by design: masking is opt-in, and `--redact` masks it, except an upper-case shape the store has lowercased. This row first said that residue was documented at the call site and in USAGE. It was only at the call site; the final gate found the gap (A5, A6, A9), and USAGE's list of what masking does not guarantee now names it | +| redact-code-1 | high | No SKILL.md change ships | refuted | It was in the docs half of the split pack (lead) | +| redact-code-2 | med | The linearity test asserts on wall clock | fixed | The page of headers is now counted: one END search, against 2000 under the mutant. The never-closing END line is one `re` search, which cannot be counted, so it stays timed, as the suite's other regex-cost tests are | +| redact-code-3 | low | The `--redact` help says "set and not 0", but empty is off too | fixed | Now "set to anything but empty or 0" | +| redact-code-4 | low | `safe_path` now widens a PEM header to the end of a diagnostic | refuted | The owner's decision, pinned by a test and a mutation row. It only ever masks more | +| redact-code-5 | low | The flag is registered only for `recall` | refuted | Integrated, `show` registers it too, and those are the two commands that print transcript text | +| redact-code-6 | low | The docs for the env var are absent | refuted | They were in the docs half of the split pack (lead) | +| redact-docs-1 | high | No `--redact` in the CLI | refuted | In the code half (lead) | +| redact-docs-2 | high | No `mask()`, no `safe_path` delegation | refuted | In the code half (lead) | +| redact-docs-3 | high | Counts bumped with no tests or rows | refuted | The tests and rows were in the code half (lead) | +| redact-docs-4 | high | Detector counts edited to twelve and four with no code change | refuted | redact.py has 12 HIGH and 4 SUSPECT on `main` and here. `main`'s DESIGN.md said nine and three, which was stale | +| redact-docs-5 | med | The skill's env check may differ from the CLI's | refuted | Same predicate: unset, empty or `0` is off. Probed with `00`, `false` and padded values: both read them as on | +| redact-docs-6 | med | A masked address can be opened neither with `show` nor as a manifest | fixed | The first half was already fixed in integration. The skill now says the manifest cannot be found from one either, and not to list `/sessions/`, whose names are the ids unmasked | +| redact-docs-7 | med | The PEM edges are untested | fixed | Three of the four were already pinned. The fourth, a second key inside the body, had no test, and `_merged`'s `max(` held it by accident: dropping it leaked the body's tail. It now has an assertion | +| redact-docs-8 | low | "Measured examples" should be pinned | refuted | Re-run: all ten examples still print unmasked. No rule says a measurement must be pinned, and drift could only understate coverage | +| redact-docs-9 | low | "Keeps its columns" overclaims | fixed | Alignment is not kept; the field count is. Now "stays one field of a `recall` line or a `show` header" | +| sizes-1 | low | mutate_index's docstring keeps "1280" and "955" | fixed | As show-2 | +| sizes-2 | low | `#set-it-up-once` may not exist | refuted | USAGE.md has `## Set it up once` (lead) | +| sizes-3 | low | The skill or hook README may show old `+NB` output | refuted | Neither has an output sample. The only `+NB` left is the commit subject, kept by decision | +| disk-1 | med | The test never reads which directory the section says to delete | fixed | Probed: "delete `sessions/`" passed. Every directory a sentence says to delete must now be one the store's `.gitignore` ignores | +| disk-2 | low | The CHANGELOG's link is outside the anchor loop | fixed | Added. A broken anchor there now fails | +| disk-3 | low | "About twice" sits beside a table of 2.3x and 3.4x | fixed | Bounded in watching.md, the README and DESIGN.md: "more if your transcripts compress poorly" | +| disk-4 | low | Three behavioural claims are unverified | refuted | Each is true and pinned: recall and dashboard refuse without an index, `show` does not need one, a rebuild renames into place, `gc --auto` runs | +| disk-5 | low | Only column-0 backtick fences are stripped | declined | No tracked Markdown has a `~~~` fence. An indented fence indents its `#` lines too, and the heading pattern reads column 0 only | + +## Decided while applying + +**One pin for two findings.** subagents-2 and stale-2 each proposed holding +`docs/REPRODUCE.md` to the counts: the first by adding assertions to the four +count tests, against the collected numbers, and the second with one new test +holding the guide to the README. The second is the one that landed. The README +is already held to the suite, so it needs no second collection, and it runs on +a machine without the corpora too. The first would have held two of the +guide's four numbers only where the corpora are cloned. The cost is one test. + +**The re-read made an old control blind.** init-1's second read, under the +lock, raises on a file it cannot read by itself. So the row "init takes a +config.toml it cannot read for a missing one", which mutates the first read's +`except FileNotFoundError` to `except OSError`, survived: the second read +still refused. Both reads now go through one helper, the row's anchor moved +into it, and it is caught again. + +## Two findings from outside Kimi's table + +| ID | Claim | Disposition | Evidence or reason | +|---|---|---|---| +| opus-item1 | The README's `uv python install 3.13` advice is wrong where uv may not fetch Python | refuted | Item 1 had already limited it to `python-downloads = "manual"`, and says to put a 3.13 on `PATH` under `"never"`. Probed both, offline, uv 0.11.3 | +| daemon-tmp-x | A test on `main` uses `/tmp/x` as a watch root | fixed | Probed: a stray regular file there empties the result and fails the test. The root is now under `tmp_path`, never created, which also pins that a root that does not exist yet is kept | + +## Verified + +At the state this file describes: + +| | | +|---|---| +| Full suite, corpora off | **1114 passed, 17 skipped**, 1 deselected: 1131, the README's number | +| Full suite, corpora on | **1455 passed, 1 skipped**, 1 deselected | +| Collected | 1131 off and 1456 on, 328 conformance cases: the README and §5b of REPRODUCE.md | +| `ruff check` | clean | +| Mutation rows | **658**. The 23 rows that name a test this commit changed or re-pinned, the new one included: **23 of 23 caught**. Three of them, the README-count rows, bite only with the corpora present, which is what they check; they were run with them | + +## Final gate + +Two more passes read the branch at `b7e3521`, and one commit on top answers +both, the one that adds this section. + +- **Kimi K3, three packs:** the code half of the fix commit, its docs half, + and SKILL.md as it now reads whole. On the code half its verdict was ship, + with no findings. +- **An Opus gate of five lenses** over the whole branch: security, end to end, + docs, CI from a fresh clone, and simplicity. Two skeptics took each finding + and tried to refute it by default, with a probe wherever one could run. 18 + survived both, and 2 were refuted. + +**33 findings: 18 fixed, 4 documented, 4 declined, 1 pre-existing and out of +scope, 6 refuted.** *Documented* means the behaviour stays and the docs now +say so. Four are changes of behaviour, A1, A4, A7 and F2, each with a test that +failed first and a new negative control. + +### Kimi K3 + +| ID | Sev | Claim | Disposition | Evidence or reason | +|---|---|---|---|---| +| F1 | high | The fix commit re-ran only the rows whose test it changed, not the rows anchored in the hunks it changed | declined | Answered by a full pass over every row instead of a better selection. A missing anchor already fails: `test_every_mutation_row_anchors_exactly_once` on CI, and `SKIP` in a run. This commit also re-ran each row anchored in or beside a hunk it touched | +| F2 | med | service-3's refutation is wrong: after the `rm`, the unit stays loaded until a `daemon-reload` | fixed | service-3's second report, and its refutation rested on systemd's documentation, not a probe. The reload is harmless and conventional: the printed removal ends `; systemctl --user daemon-reload`, in the code, its test, a new row and watching.md | +| F3 | med | The CHANGELOG's list of what `service install` prints has no `loginctl enable-linger` line, which USAGE says it prints | fixed | It prints one on Linux, and the CHANGELOG now lists it | +| F4 | low | Pin redact-docs-8's measured examples | declined | Each is a shape `--redact` does not mask, so a pin would hold a weakness in place | +| F5 | low | A reflow left one overlong line in USAGE's `--redact` paragraph | fixed | Rewrapped, with A4's new order | +| S1 | med | Rerunning step 1 does not update the CLI | refuted | Live probe, uv 0.11.3, as version-2: rerunning the unpinned line replaced the install's commit with the remote's tip | +| S2 | med | A CLI from before `--version` does not print `the following arguments are required: cmd` | refuted | Measured on `c352a40`: it prints exactly that | +| S3 | med | A topic that starts with `-` is read as a flag, and the error reads as an older CLI | fixed | Probed: `recall "--bogus"` exits 2, `the following arguments are required: query`. The skill now puts `--` before such a topic, after any flags; `recall -k 30 --policy latest --redact -- "--bogus"` parses | +| S4 | low | Drop the `echo "${GITMEMORY_REDACT:-}"` check, since the CLI already defaults to masking | declined | The owner's decision: an explicit `--redact` makes a CLI too old for the flag fail loudly, where the default would print unmasked | +| S5 | low | `${CLAUDE_SESSION_ID}` may not exist | refuted | It is Claude Code's documented substitution in a skill, and the sentence predates this round | +| S6 | low | A masked address cannot occur | refuted | `capture --session-id` takes a credential-shaped id, and `recall --redact` masks it (A5) | +| S7 | low | Split the by-hand `raw/` lookup into numbered steps | declined | It is the fallback for a CLI older than `show`, and is left as it is | +| S8 | low | Rules are repeated: masking three times, the subagent cautions twice, and `-k 30` beside `-k 50` | fixed | Each is stated once: masking in the opening paragraph, what `subagent:` means in the field table, and one retry, `-k 50`, with its reason, that the order applies to the top `-k` matches only | + +### The Opus gate + +| ID | Lens | Sev | Claim | Disposition | Evidence or reason | +|---|---|---|---|---|---| +| A1 | security | med | `show --redact` stops, exit 2, partway through a turn that holds a lone surrogate | fixed | `mask` encoded with `surrogateescape`, which round-trips only U+DC80-U+DCFF; a JSON `\ud83d`, half an emoji, is any other. It encodes with `surrogatepass` now, as `records` hashes. Tested at the address and in a `-C` neighbour | +| A2 | security | med | The push gate misses a `\b`-anchored key right after a JSON `\n` or `\t` escape | pre-existing, out of scope | Reproduced. The HIGH table is `main`'s, and `push` does not send yet. Left for its own issue; the owner's unpushed egress branch adds a scan of the decoded text that targets this class | +| A3 | security | low | `show --redact` prints a private key's body when its BEGIN line is in an earlier block of the turn | documented | No span is carried across blocks. USAGE's list says a key split across two blocks is masked to the end of the first | +| A4 | security | low | `recall --redact` prints a SUSPECT value with a Unicode space in it, which `show --redact` and the gate flag | fixed | `str.split()` broke the value at a no-break space before the mask saw it. recall now masks as `show` does, with the mask's own collapse, then flattens and cuts | +| A5 | security | low | The address's case-fold residue is not in USAGE, so show-7's evidence is wrong | documented | A bullet in USAGE's list; show-7 corrected above | +| A6 | e2e | low | An AWS or Google key used as the session id prints unmasked, though USAGE, DESIGN and the CHANGELOG say the address is masked | documented | As A5. USAGE and the CHANGELOG now make the exception for a session id, and the DESIGN amendment no longer lists what is masked | +| A7 | e2e | low | `show`'s refusal of a masked address says `as recall prints it` about what recall printed | fixed | A one-line refusal of its own, `a masked address: ...`, still exit 2 | +| A8 | e2e | low | USAGE's `--redact` sample cannot come from the transcript every sample on the page comes from | fixed | It comes from a second transcript now, with its own session, and says so | +| A9 | docs | med | `--redact` leaves a credential-shaped session id unmasked, because ids are stored lowercased; show-7 cites USAGE text that does not exist | documented | As A5 and A6. The ruling was to document it and leave the code | +| A10 | docs | low | While masking, the skill still reads a hit's manifest, whose `source_path` is unmasked and in its original case | fixed | While masking, the skill reads nothing under the store by hand and goes by the address alone | +| A11 | docs | low | USAGE's `--redact` sample puts a different turn at another sample's address | fixed | As A8 | +| A12 | docs | low | The CHANGELOG and SECURITY say the skill passes the flag when the variable is "set and not `0`"; empty is off too | fixed | "set to anything but empty or `0`" in both, as redact-code-3 did for the help | +| A13 | docs | low | The CHANGELOG misses the reworded skip message and keeps the unbounded "about twice" | fixed | Both added | +| A14 | docs | low | The `--redact` caveats and the `settings.json` snippet are kept twice, and the copies differ; some passages run long | fixed | The caveat list and the snippet are in USAGE only; SECURITY and hook/README link them. watching.md's init paragraph and the DESIGN amendment are cut to what a reader acts on | +| A15 | docs | low | The docs index describes watching.md as configuration and flags only | fixed | The row names the service and the disk section | +| A16 | ci | low | REPRODUCE §5b's count is two low on an account named `ubuntu`, `runner` or `root` | fixed | Its commands set `HOME` as the suite's conftest does, and a paragraph says why | +| A17 | simplicity | low | init's join check cannot fire once the appended config parses | fixed | It refuses in the `except`; `have` is gone, and the new table's number is the length of the parsed `watch` array | +| A18 | simplicity | low | `uncollected`'s injected clock has no caller | fixed | Dropped. The `waiting and` the finding called optional is left: it changes nothing, and two rows anchor on it | +| R1 | docs | — | "Nothing leaves the machine" and "the only way anything leaves is `push`" contradict text this branch added | refuted | The tension was on `main` already, where SECURITY.md said the skill's quotes reach the provider unscanned; the added text puts that egress in the agent's path, not in anything gitmemory sends | +| R2 | simplicity | — | `test_recall_redact_keeps_a_subagent_turn_marked` repeats `test_recall_marks_a_subagent_turn` | refuted | By mutation: dropping the mark under `--redact` only fails this test and no other | + +### Decided while applying + +**A4 moves where recall masks.** The owner's decision put the mask after the +whitespace collapse, so that it would see a PEM header a line break split. But +`str.split()` also breaks at a no-break space, and that collapse was what cut +the value. recall now does as `show` does: the mask's own collapse, of ASCII +whitespace, finds the split header, and recall flattens and cuts after it. The +decision's purpose holds, and recall without `--redact` prints what it did. + +**What A1 costs.** In a block that also holds a mask, a lone surrogate prints +as replacement characters, where the same block unmasked prints `?`. + +**Rows.** One new row for each change of behaviour brings the index to +**662**. A4 moved three rows' anchors: the two on recall's head line, and +`show`'s collapse row, whose anchor recall's new line repeated. Every row +anchored in or beside a hunk this commit touched was run, the new ones +included: **45 of 45 caught** by their own test. + +| | | +|---|---| +| Full suite, corpora off | **1117 passed, 17 skipped**, 1 deselected: 1134, the README's number | +| Full suite, corpora on | **1458 passed, 1 skipped**, 1 deselected | +| Collected | 1134 off and 1459 on, 328 conformance cases: the README and §5b of REPRODUCE.md | +| `ruff check` | clean | + +### The full mutation pass + +F1 asked for every row, not a better choice of rows, so every row was run: +twice over, on two tips, each split six ways across fresh clones, with no +corpora. + +| Tip | Rows | Caught by their own test | Not | +|---|---|---|---| +| `b7e3521`, the Kimi fix commit | 658 | 642 | 16 MISSED, 0 SURVIVED, 0 SKIP | +| this commit | 662 | 646 | 15 of the same 16, MISSED or SURVIVED; 0 SKIP | + +None of the sixteen is this branch's. Each was run again on `main`, under the +same two setups the passes used, and each gave the verdict it gave here, row +for row. They are rows whose test cannot run offline here: the README and +conformance counts need the corpora, the dashboard's need the `serve` extra, +and the rest fail the same way on `main`. The sixteenth, the suite-wide home +isolation, is caught on this tip with the real `HOME`: it misses only when the +pass itself overrides `HOME`, so that verdict was the harness's, not the row's. +Every row this branch added or moved is caught by its own test. + +The sixteen are a finding about `main`, and they are left for a round of +their own. diff --git a/docs/watching.md b/docs/watching.md index d91ce5c..c1fe6ec 100644 --- a/docs/watching.md +++ b/docs/watching.md @@ -14,7 +14,21 @@ Watch roots have no default, deliberately. A memory tool whose first run scans your home directory is a tool you have to audit before you can trust it, so gitmemory watches exactly what you name and nothing else. -`$GITMEMORY_HOME/config.toml`: +`gitmemory init claude-code` (or `gitmemory init pi`) writes the table for +you, with the root [agents.md](agents.md) gives for that agent. That is still +not a default: the root is written because you named the agent, and the watcher +reads only the file. `init` appends to an existing `config.toml` and never +changes a byte already there, since the file also holds any `[remote.*]` push +opt-in, and it does nothing if the table is already there. When it cannot add +the table safely it writes nothing, prints why, and exits 1: a root another +agent watches or the watcher would skip (listed below); a file that is not +valid TOML, has a `watch` of another shape, is a symlink or has a second hard +link; or one another `init` changed while it ran, which a second run settles. +An editor saving the file at that moment is not covered. A `config.toml` it +cannot read is an error, exit 2. A running watcher picks the new table up on +its next pass. + +By hand, or for any other agent or root, `$GITMEMORY_HOME/config.toml`: ```toml [[watch]] @@ -26,13 +40,59 @@ roots = ["~/.claude/projects"] |---|---|---| | `agent` | yes | Which adapter finds compaction boundaries. See below. | | `roots` | yes | Directories to tail. `~` is expanded; symlinks are resolved. | -| `pattern` | no | Glob under each root. Defaults to `**/*.jsonl`. | +| `pattern` | no | Glob under each root, with `pathlib` semantics: `*` never crosses a `/`, and `**` spans any number of directories. Defaults to `**/*.jsonl`. | Repeat the table for each agent you run. Absent, unreadable, or malformed config means the watcher watches nothing — and says so on every pass, naming the file and the reason, because "nothing configured" and "misconfigured" must not look alike. +### Main sessions only (Claude Code) + +The default pattern takes every transcript under the root, and under +`~/.claude/projects` most of them are not a conversation you had. Claude Code +writes each subagent to `//subagents/agent-.jsonl`, and +each workflow agent, beside the workflow's `journal.jsonl`, under +`//subagents/workflows//`. A pattern one directory deep +takes only the main sessions, `/.jsonl`: + +```toml +[[watch]] +agent = "claude-code" +roots = ["~/.claude/projects"] +pattern = "*/*.jsonl" +``` + +Put the `pattern` line in the `claude-code` table you already have, not in a +second table beside it. The watcher takes the files of every table, so another +table on the same root with the default pattern still finds every subagent. + +A hook doorbell for a subagent's file then captures nothing: the spool only +hurries a file the pattern already found. + +Prefer the default unless the store's size matters more than what this costs. +`recall` marks a subagent's turns `subagent:` (see +[USAGE.md](USAGE.md#find-something)), so they can be told apart without +leaving them out, and a policy can drop them from what it prints +([USAGE.md](USAGE.md#your-own-rule)). What leaving them out costs: + +- **Those turns are never captured.** Once Claude Code cleans up old + transcripts, they are gone. Until then this is reversible: put the default + back, and the next pass captures each subagent file from its first byte. +- **The dashboard's spend undercounts.** It counts only what was captured, and + subagents are where much of the spend goes: one measured session had 78% of + its cache-read tokens in `subagents/**` ([DESIGN.md §2.2](DESIGN.md#22-claude-code-adapter--the-traps-each-a-test-case)). +- **What was sent survives, and usually what came back.** The parent session + keeps the tool call that started a subagent or a workflow, with its + instructions. A subagent's report comes back as that call's result. A + background agent or workflow gets only a launch receipt there; its report + comes later, as a task-notification turn in the parent, or not at all if the + session ended first. What is lost is the subagents' own working turns, and a + workflow's `journal.jsonl`, which holds each of its agents' results. +- Older Claude Code wrote a subagent's turns into the parent's own file, each + line marked. A pattern cannot leave those out; the `subagent:` mark still + labels them. + ### What gets refused, and why A root is skipped, loudly, when it: @@ -87,10 +147,79 @@ Each pass prints what it did, or nothing when nothing happened. Errors are repeated only when they change or once an interval, so a permanently broken root does not drown the log it belongs in. -Ctrl-C exits 130. There is no daemonisation, no pidfile, and no installer: -`gitmemory watch` is a foreground process, and how you keep it alive — a user -LaunchAgent, a systemd user unit, a terminal you leave open — is yours to -choose. Nothing here needs root. +Ctrl-C exits 130. There is no daemonisation and no pidfile: `gitmemory watch` +is a foreground process, and how you keep it alive — a terminal you leave open, +or a user service, which `gitmemory service install` writes for you +([below](#keep-it-running)) — is yours to choose. Nothing here needs root. + +## Keep it running + +`gitmemory service install` writes one file for your user's service manager, +and starts nothing: + +| Platform | File | Manager | +|---|---|---| +| Linux | `$XDG_CONFIG_HOME/systemd/user/gitmemory.service`, or under `~/.config` when that is unset or not an absolute path | systemd, as your user | +| macOS | `~/Library/LaunchAgents/io.github.doronp.gitmemory.plist` | launchd, in your login session | + +It prints the file it wrote, the command line in it, and the commands below, +and runs none of them: no `systemctl`, no `launchctl`, no `loginctl`. Starting +it is yours to do, as installing the hook is. If the file is already there it +refuses, exit 1, and leaves it alone; the refusal ends with the line that +removes it, so paste that and install again. Any other platform exits 2, and so +does a store path with a line break in it. + +A service starts with none of your shell's state, so the file carries it: + +- **This Python, as it is,** with `-P -m gitmemory`. `-P` because a systemd + user unit starts in `$HOME`, and `python -m` would run a `~/gitmemory.py` in + place of the package. Not the interpreter's real path either, which is + outside the virtual environment and cannot import gitmemory. +- **The store, as `--home `.** A service does not read your shell's rc + files, so an exported `$GITMEMORY_HOME` never reaches it, and the watcher + would read `~/.gitmemory` while the hook writes somewhere else. If you move + the store, remove the file and install again. +- **`PATH`**: the directory `git` was found in, then `/usr/bin:/bin`. No git on + your `PATH` exits 2 and writes nothing. + +On Linux: + +```sh +systemctl --user daemon-reload && systemctl --user enable --now gitmemory.service # start, now and at every login +loginctl enable-linger # keep it running after you log out +systemctl --user status gitmemory.service # check it +journalctl --user -u gitmemory.service -f # its log +systemctl --user restart gitmemory.service # after upgrading gitmemory +systemctl --user disable --now gitmemory.service; rm ~/.config/systemd/user/gitmemory.service; systemctl --user daemon-reload +``` + +On macOS: + +```sh +launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/io.github.doronp.gitmemory.plist # start, now and at every login +launchctl print gui/$(id -u)/io.github.doronp.gitmemory # check it +tail -f ~/Library/Logs/gitmemory.log # its log +launchctl kickstart -k gui/$(id -u)/io.github.doronp.gitmemory # after upgrading gitmemory +launchctl bootout gui/$(id -u)/io.github.doronp.gitmemory; rm ~/Library/LaunchAgents/io.github.doronp.gitmemory.plist +``` + +- **Linger** matters on a machine you reach only over SSH: without it, systemd + stops your user services when your last session ends. `loginctl + enable-linger` with no user name is for yourself, which stock systemd's + polkit policy allows without a password; a distribution can change that. +- **It is restarted after a failure, 30 seconds later** (`Restart=on-failure`; + `KeepAlive` on an unsuccessful exit). A watcher that cannot start — a store + inside a git work tree, say — logs one line every 30 seconds until fixed. +- **Editing `config.toml` needs no restart;** the watcher re-reads it every + pass. Upgrading gitmemory does, because the running process keeps the old + code. +- **The macOS log is outside the store,** which commits everything in it. + Nothing rotates it. On Linux the log is the journal. +- **Removing it stops it, then removes the file whether or not that worked.** + A file written and never started has nothing to stop: `launchctl bootout` + says `No such process` and exits 3, and the `rm` after the `;` runs anyway. + On Linux a `daemon-reload` after the `rm` has systemd drop the unit it had + loaded. ## Check it @@ -105,8 +234,9 @@ anywhere useful. The two failures worth watching for are different shapes: | Symptom | Likely cause | |---|---| -| `no .../config.toml; watching nothing` | The file is not where the watcher is looking. Check `$GITMEMORY_HOME`. | -| Watcher quiet, store empty, no complaint | A root that **does not exist** is kept, not refused — before an agent has run for the first time that is the normal state, and `run` re-reads the config every pass, so it starts working the moment the directory appears. The cost is that a typo'd root and a not-yet-used one look identical. Check the path by hand. | +| `no .../config.toml; watching nothing` | The file is not where the watcher is looking. Check `$GITMEMORY_HOME`, or run `gitmemory init ` to write it there. | +| Watcher quiet, store empty, no complaint | A root that **does not exist** is kept, not refused — before an agent has run for the first time that is the normal state, and `run` re-reads the config every pass, so it starts working the moment the directory appears. The cost is that a typo'd root and a not-yet-used one look identical. Check the path by hand; `gitmemory init` says so when the root it writes does not exist yet. | +| `recall` says `N hook record(s) uncollected, the oldest for ` | No watcher has run a pass on that store for that long: it is stopped, it watches a store other than the hook's `$GITMEMORY_HOME` (a service's store is the `--home` written into it), or it runs a pass less often than once an hour (`--poll` of 3600 or more, or a supervisor re-running `--once` that seldom). Start `gitmemory watch` with the hook's `$GITMEMORY_HOME` and keep it running with `gitmemory service install` ([above](#keep-it-running)), or run passes more often. A service that is installed but not started, or failing, shows here too; its check and log lines are above. A pass drains the spool even when no watch covers the record, so silence here means a pass ran, not that this agent is captured. | | `compact_boundaries: []` everywhere | Adapter missing or `--no-parse`. The bytes are intact; only the boundaries are absent. | | `source changed while parsing` | The transcript was rewritten between the parse and the copy — a compaction landing mid-pass. The bytes are captured; that one generation has no boundaries. Nothing to do. | | `was captured from , not ; pass a distinct --session-id` | Two transcripts collided on one session name, and the store refuses to mix them rather than duplicating both for ever. The advice in the message is for the CLI and there is no `--session-id` to pass here: rename or move either file — the name is derived from the path — and the next pass captures both. Until then the second one is **not being captured**, so this is not a warning to leave running. | @@ -114,3 +244,55 @@ anywhere useful. The two failures worth watching for are different shapes: There is no discovery counter: a pass prints only when it captured something, so "found 12 transcripts, all unchanged" and "found nothing" are the same silence. Listing the roots yourself is currently the way to tell them apart. + +## How much disk it takes + +Budget about twice what you capture, and more if your transcripts compress +poorly: the synthetic corpora below came to 2.3x and 3.4x. The store keeps the +same bytes three ways: + +- **`raw/`** holds each captured byte once, so it is the size of the capture. + Each segment fills whole filesystem blocks, so a store grown in many small + passes runs a little over. +- **`.git/`** holds them again, zlib-compressed and packed by `git gc --auto`. + That is roughly the capture divided by what gzip gets out of your + transcripts. For a store that holds what is under `~/.claude/projects` now, + `find ~/.claude/projects -name '*.jsonl' -exec cat {} + | gzip | wc -c` + estimates it, a little low. That count leaves out bytes the store captured + before a transcript was rewritten or deleted, and any other root you watch. +- **`index/`** holds the searchable text a third time, uncompressed, beside its + FTS5 postings. Git ignores it and `gitmemory index` rebuilds it from the + store, so it is the one to delete when you need the space back. `recall` + and `dashboard` refuse until you rebuild it; `show` reads the store and + does not need it. A rebuild writes the new database beside the old one and + renames it into place when it is done, so while it runs `index/` takes + about twice its usual space. When space is tight, delete `index/` first, + then rebuild. + +`sessions/` is one small manifest per generation. `derived/`, if you run +`derive`, adds a little more: 0.09x on the first corpus below. + +Measured with `du -sk` (APFS, 4 KiB blocks) on synthetic Claude-Code-shaped +corpora of 470 transcripts, each captured in one pass and then indexed, as +multiples of the bytes captured: + +| Captured | `raw/` | `.git/` | `index/` | `sessions/` | Total | +|---|---|---|---|---|---| +| 43 MB of prose and source code (gzip 4.6x) | 1.02 | 0.24 | 1.02 | 0.04 | 2.3x | +| 163 MB, the same mix | 1.01 | 0.22 | 1.03 | 0.01 | 2.3x | +| 44 MB of base64 tool output (gzip 2.6x) | 1.02 | 0.40 | 1.94 | 0.04 | 3.4x | + +The part you cannot rebuild, `raw/`, `.git/` and `sessions/`, came to 1.2-1.5x +in each. Captured in ten passes instead of one, that part of the first corpus +came to 1.6x: smaller segments round up to more blocks, and git held ten packs +it had not yet merged. + +Only the text of a turn is indexed. The JSON around it is not, and an image goes +in as its shape, never its payload, so the more of that your transcripts carry, +the smaller `index/` is next to them. One real store was reported at 1.75x, +684 MB for 392 MB captured. Yours depends on how well your transcripts +compress, how much of them is text, and when git last packed. + +To capture less in the first place, [main sessions only](#main-sessions-only-claude-code) +keeps subagent and workflow transcripts out of the store, at the cost that +those turns are never captured and the dashboard's spend undercounts. diff --git a/hook/README.md b/hook/README.md index 4d63361..7f7d8b3 100644 --- a/hook/README.md +++ b/hook/README.md @@ -93,22 +93,44 @@ The plugin installs the hook and one skill, `/gitmemory:recall`, which has the agent run `gitmemory recall` and quote what it returns ([USAGE.md](../docs/USAGE.md#put-it-back-in-front-of-the-agent)). Neither captures anything. **Nothing is captured until the watcher is installed and -running** (`uv tool install git+https://github.com/doronp/gitmemory`, a watch -root in `config.toml`, then `gitmemory watch`, as in the +running** (`uv tool install --python 3.13 git+https://github.com/doronp/gitmemory`, +`gitmemory init claude-code`, then `gitmemory watch`, as in the [Quickstart](../README.md#quickstart)), and gitmemory is correct without the plugin, as it is without the hook. The hook and the skill run with Claude Code's environment, so a `GITMEMORY_HOME` you set for the watcher has to be set there too, and be absolute, as above. What the skill reads from the store goes to the model provider with the rest of the conversation, and no redaction gate stands in that path ([SECURITY.md](../SECURITY.md#if-a-credential-lands-in-the-store)). - -An install made before the skill was added does not have it. Claude Code keeps -a plugin's cached copy until its version changes, and the skill was added -without a version bump, so `claude plugin update gitmemory@gitmemory` reports -that it is already at 0.1.0. The next release, which bumps the version, brings -the skill; until then, `claude plugin uninstall gitmemory@gitmemory` and then +To have `recall` and `show` mask the credential shapes the gate knows in what +they print, set `GITMEMORY_REDACT=1` in that environment, in +`~/.claude/settings.json` as +[USAGE.md](../docs/USAGE.md#put-it-back-in-front-of-the-agent) shows. The +skill then passes `--redact` to every `recall` and `show` it runs. It masks; it +is not a gate, and +[USAGE.md](../docs/USAGE.md#what-it-does-not-guarantee) lists what it does not +cover. + +An install made from 0.1.0 does not have the skill. Claude Code keeps a +plugin's cached copy until its version changes, and the skill was added after +0.1.0 without a version bump, so until 0.2.0 an update reported that the +plugin was already at 0.1.0. Release 0.2.0 bumps the version, so +`claude plugin update gitmemory@gitmemory` brings the skill; if it still says +0.1.0, `claude plugin uninstall gitmemory@gitmemory` and then `claude plugin install gitmemory@gitmemory` fetch the current copy. +The plugin and the CLI are installed separately, and nothing makes them one +release: `uv tool install` takes the commit you name, or the head of main, and +the plugin is the commit its marketplace was cloned at, kept until its version +changes. The recall skill runs `gitmemory --version` first and tells you when +the CLI is older than it, and when the CLI lacks a subcommand or a flag the +skill needs, it stops and says which rather than retrying without it. To hold +both at one release, pin both to its tag, `v`: +`uv tool install --python 3.13 git+https://github.com/doronp/gitmemory@v` +and `/plugin marketplace add doronp/gitmemory#v` (a marketplace pins a +branch or a tag, not a bare commit). To move the CLI forward, run the +`uv tool install` line again without the `@` part; `uv tool upgrade gitmemory` +leaves a pinned install where it is. + This keeps the rule at the top of [Install](#install): gitmemory never edits another program's configuration. You opt in with `/plugin`, and Claude Code records the change in its own settings. `/plugin uninstall gitmemory@gitmemory` @@ -193,6 +215,11 @@ it and then re-derives everything — how many bytes are new, where the boundary is, whether the file was rewritten — from the transcript itself. A record naming a path outside the configured watch roots is dropped. +Every pass removes every complete record, dropped or not, so a record older +than an hour means no watcher has run a pass on this store since it arrived, +and `gitmemory recall` says so on stderr. That is also how a hook and a watcher +with different `$GITMEMORY_HOME`s show up: the message names the store. + ## Latency The shim is in the agent's critical path, so it is measured rather than asserted. diff --git a/pyproject.toml b/pyproject.toml index 41f13b6..612462d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "gitmemory" -version = "0.1.0" +version = "0.2.0" description = "Git-versioned, contiguity-checked memory for agents that keep a local, append-only, line-delimited transcript" readme = "README.md" requires-python = ">=3.13" diff --git a/skills/recall/SKILL.md b/skills/recall/SKILL.md index f35d1f7..203ec91 100644 --- a/skills/recall/SKILL.md +++ b/skills/recall/SKILL.md @@ -11,17 +11,41 @@ byte, and `gitmemory index` builds a search index from that record. This skill searches it with the `gitmemory` CLI and quotes the words as they were written. It reads the store and rebuilds the index when it has to; it captures and configures nothing. It sends nothing anywhere itself, but what it reads enters -this conversation and goes to the model provider with the rest of it, past no -redaction gate: that gate stands at `gitmemory push`. +this conversation and goes to the model provider with the rest of it. The +redaction gate stands at `gitmemory push`, not here. `--redact` masks the +credential shapes that gate knows in what `recall` and `show` print, as +`sk-a…[103 bytes]`. Pass it to both whenever `echo "${GITMEMORY_REDACT:-}"` +prints anything but nothing or `0`, and whenever the user asks you to keep +credentials out of the conversation. Masking covers only what those two print, +so while you are masking, read nothing under the store by hand: `raw/`, the +manifests under `sessions/` and the index all hold it unmasked. If either +command rejects the flag, do not run it without it. The store is `$GITMEMORY_HOME`, or `~/.gitmemory` when that is unset; the CLI reads it, so leave it as this session has it. If the user keeps the store somewhere else, pass their path before the subcommand, as in -`gitmemory --home recall ""`, and the same for `index`. `` -below means that directory: the path the user gave, or what +`gitmemory --home recall ""`, and the same for `index` and +`show`. `` below means that directory: the path the user gave, or what `echo "${GITMEMORY_HOME:-$HOME/.gitmemory}"` prints. Do not type `~/.gitmemory` in its place from memory. +## The CLI's version + +The plugin and the CLI are installed separately, so they can come from +different releases. This skill is written for gitmemory 0.2.0. Once per +session, before the first search, run: + +```sh +gitmemory --version +``` + +If it prints that release or a later one, go on. If it prints an earlier one, +or exits 2 with `the following arguments are required: cmd` (a CLI from before +`--version` existed), the CLI is older than this skill: tell the user so in +one line, with the fix under [When it fails](#when-it-fails), and go on. If +what `recall` then prints is not laid out as below, say so, and do not judge +which of two lines is the current decision from their order. + ## Search The topic is the skill's argument if it was given one, otherwise the thing the @@ -29,9 +53,14 @@ user is referring to, in the words they would have used. `recall` ranks by BM25: it matches words, not meaning. ```sh -gitmemory recall "" # 10 lines by default; -k N for more or fewer +gitmemory recall "" # 10 lines by default; -k N for more or fewer +gitmemory recall --redact "" # the same, masked ``` +If the topic starts with `-`, put `--` before it, after any flags, as in +`gitmemory recall --redact -- ""`. Otherwise `recall` reads it as a +flag, and the error that follows says nothing about the CLI's age. + One line per turn: what the user said first, newest first, then everything else newest first: @@ -46,6 +75,7 @@ else newest first: | `claude-code//g00` | Agent, session and generation | | `@346` | Byte offset of the turn (its JSONL line) in that generation's raw bytes | | `assistant/text` | Role and block kind | +| `subagent:user/text` | A subagent's turn, marked `subagent:` before any role/kind (`subagent:assistant/text` too, and `subagent:user` in a `show` header). Its user text is the instructions the parent agent gave it, not the user's words | | the rest | The block's first 160 characters, whitespace collapsed | `no matches` on stderr, exit 0, is an answer. Try once more with other words @@ -54,8 +84,11 @@ the user might have used, then tell them nothing was found. `query truncated to If two lines disagree, the later date is usually the current decision: say so, and quote both. An agent's newer proposal does not overrule the user's older -rule; a newer user line does. If the user has set `GITMEMORY_POLICY` or asks -for another order, leave it; `--policy` is theirs to choose. +rule, and neither does a subagent's line; a newer user line does. If the user +has set `GITMEMORY_POLICY` or asks for another order, leave it; `--policy` is +theirs to choose. The order is applied to the top `-k` matches only, so if +echoes (below) or `subagent:` lines crowd out the rest, search again with +`-k 50`. ## Echoes @@ -66,12 +99,26 @@ recalls match the same words. Skip these hits; they are not sources: - `/gitmemory:recall` and its arguments, however the transcript wraps them; - a `gitmemory recall ...` tool call and its result, lines that start with a score, `N generation(s)` or `no matches`; +- a `gitmemory --version` tool call and the one line it printed; +- a `gitmemory show ...` tool call and its result, text that starts `> ` or + `- ` immediately followed by an `//gNN@` address + and a date, which is `show`'s header. It holds the whole turn, dated later + than the turn was: the source is the turn at the address in its `> ` + header, not the echo's own address; - an earlier answer to the same question, and the lines it quoted. -If they crowd out the rest, search again with `-k 30`. Quote the original turn. +Quote the original turn. ## When it fails +- A `gitmemory` command exits 2, what it prints starts `usage: gitmemory`, and + its last line says `invalid choice`, or `unrecognized arguments` naming a + flag (a word starting with `-`) that is not your topic: the CLI is older + than this skill and lacks that subcommand or flag. Do not run it again + without the flag or under another name. Tell the user which one is missing, and that running step 1 + below again updates the CLI, a pinned install included. If + `unrecognized arguments` names only plain words, the CLI is not the problem: + the topic went in unquoted. Put it in one pair of quotes and search again. - ``no index at ; run `gitmemory index` first``, exit 2: run `gitmemory index`, then search again. `index` rebuilds the index from the store in full. It is safe to rerun, and the index is never the source of @@ -79,15 +126,26 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. - The watcher captures; it does not index. If a search misses something the user says was said recently, the index may be older than the capture: run `gitmemory index` and search once more. +- `` hook record(s) uncollected, the oldest for : is `gitmemory watch` running for ? These results may be missing everything said since.`` + on stderr, before the results, exit 0: the hook has rung and no watcher has + run a pass on that store since, so nothing said after that is being captured + there. Answer from the results, say they may be missing everything since, and + tell the user to start `gitmemory watch` with the `GITMEMORY_HOME` this + session has, and to keep it running with `gitmemory service install` (step 3 + below). If they have installed a service already, it is not started, it is + failing (its log says why), or it watches another store. Do not run any of + it yourself. Reindexing cannot bring back what was said since; if the hits + miss something older, the bullet above still applies. - The shell cannot find `gitmemory`, or `index` prints `not a store: `, or it reports `0 generation(s)`: nothing has been captured there. Tell the user how it gets set up, and stop. Do none of it yourself: - 1. `uv tool install git+https://github.com/doronp/gitmemory` - 2. a `[[watch]]` table in `config.toml` in the store (`$GITMEMORY_HOME`, by - default `~/.gitmemory`) naming the agent and where its transcripts live; - for Claude Code, `agent = "claude-code"` and - `roots = ["~/.claude/projects"]` - 3. `gitmemory watch`, left running. + 1. `uv tool install --python 3.13 git+https://github.com/doronp/gitmemory` + 2. `gitmemory init claude-code`, which adds the `[[watch]]` table for + `~/.claude/projects` to `config.toml` in the store (`$GITMEMORY_HOME`, by + default `~/.gitmemory`) + 3. `gitmemory service install`, which writes a user service that keeps + `gitmemory watch` running and prints the line that starts it; or + `gitmemory watch` in a terminal left open. The [Quickstart](https://github.com/doronp/gitmemory#quickstart) has the rest. @@ -97,26 +155,42 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. - Quote the text field verbatim, with its address copied whole, never shortened. A paraphrase is not a quote: if you summarise, say that you are, and keep the address beside it. -- The text stops at 160 characters. If the question needs the rest, the turn - is in `/raw///g/`. Segment file names are byte - ranges, start included, end excluded. In the one whose range holds the - offset, the turn's JSON line starts `offset - start` bytes in, and the block - is inside it. If the file ends before the line does, the line goes on at the - start of the next segment, whose range starts where this one ends. -- This session is `${CLAUDE_SESSION_ID}`. A hit's manifest, - `/sessions///g.json`, has its transcript's - `source_path`: `/.jsonl`, or for a subagent (a session +- The text stops at 160 characters. If the question needs the rest, run + `gitmemory show
` with the address copied whole, or + `gitmemory show --context 2
` for the two turns either side too, + and `gitmemory show --redact
` if you passed `--redact` to + `recall`. A masked address (one with `…[` in it) is not one `show` can + look up: say the line stops there and give the address as printed. A + header starts `>` for the turn at the address and `-` for a neighbour, and + only headers start in column 0; quote the text under one without its + indent. If the CLI is older than `show` (exit 2, `invalid choice: 'show'`), + tell the user as [When it fails](#when-it-fails) says. If you are masking, + say the line stops at 160 characters and give the address. Otherwise read + the turn by hand: it is the JSON line that starts `offset - start` bytes + into the segment under `/raw///g/` whose name, + the byte range `start-end` with the end excluded, holds the offset, and it + may run on into the next segment. +- This session is `${CLAUDE_SESSION_ID}`. Unless you are masking, a hit's + manifest, `/sessions///g.json`, has its + transcript's `source_path`: `/.jsonl`, or for a subagent (a session named `agent--...`) `//subagents/.../agent-.jsonl`. Both belong to ``. `` is the working directory, with `/` - and other punctuation turned into `-`. Say so when a hit is from another - agent, another session or another project. + and other punctuation turned into `-`. While masking, go by the address + alone. Say so when a hit is from another agent, another session or another + project. +- Say so, too, when a quote is a `subagent:` line, and never give one as what + the user said. When it and a main-session line say the same thing, quote the + main-session line. - The store holds whatever was typed or pasted into a watched session, tool output included. Quote what the question needs and no more. ## Read-only -Run `gitmemory recall` and `gitmemory index`, and read files under ``. -`index` writes only the derived index under `/index/`. Do not run -`gitmemory capture`, `gitmemory watch`, `gitmemory derive`, `gitmemory push` -or `gitmemory dashboard`, and never `gitmemory dashboard --expose`. Do not -edit `config.toml` or anything else in the store. +Run `gitmemory --version`, `gitmemory recall`, `gitmemory show` and +`gitmemory index`, and, when you are not masking, read files under +``. `index` writes only the derived index under `/index/`. Do +not run `gitmemory init`, `gitmemory capture`, `gitmemory watch`, +`gitmemory service`, `gitmemory derive`, `gitmemory push` or +`gitmemory dashboard`, and never `gitmemory dashboard --expose`. Do not edit +`config.toml` or anything else in the store. Leave `GITMEMORY_REDACT` as this +session has it. diff --git a/src/gitmemory/__init__.py b/src/gitmemory/__init__.py index fe8ef51..37e1562 100644 --- a/src/gitmemory/__init__.py +++ b/src/gitmemory/__init__.py @@ -3,4 +3,4 @@ """gitmemory — git-versioned, contiguity-checked memory for agents that keep a local, append-only, line-delimited transcript.""" -__version__ = "0.1.0" +__version__ = "0.2.0" diff --git a/src/gitmemory/__main__.py b/src/gitmemory/__main__.py index f036882..90749f4 100644 --- a/src/gitmemory/__main__.py +++ b/src/gitmemory/__main__.py @@ -12,13 +12,18 @@ import argparse import builtins +import functools import ipaddress import os +import re +import shlex import socket import sqlite3 +import stat import sys +import tomllib -from . import daemon, dashboard, derive, gitrepo, index, redact, store +from . import __version__, daemon, dashboard, derive, gitrepo, index, redact, service, store from .records import safe_text @@ -44,13 +49,107 @@ def _safe_exc(exc: BaseException) -> str: Not in `print` itself, deliberately. `recall` prints transcript content because that is what it is for, and a store on the owner's own disk is not - an egress; masking there would make the tool withhold the user's own bytes - from them. Egress is `push` and `serve`, and an exception is neither — it is - the third case, and it is the one that leaks by accident. + an egress; masking there by default would make the tool withhold the user's + own bytes from them. `--redact` on `recall` and `show` is the owner choosing + to withhold them, because the output is going on: through the plugin's + skill, to the model provider. Egress is `push` and `serve`, and an exception + is neither — it is the third case, and it is the one that leaks by accident. """ return redact.safe_path(str(exc)) +def _refuse(why: str) -> int: + print(f"refusing: {why}", file=sys.stderr) + return 1 + + +def _config_bytes(path: str) -> bytes: + """`config.toml` as it is now, or nothing when there is no file. One it + cannot read is not a missing one: taking it for absent would replace it, + push opt-ins and all, so that is main's `error:` and exit 2.""" + try: + with open(path, "rb") as fh: + return fh.read() + except FileNotFoundError: + return b"" + + +def _init(args) -> int: + """Add the standard `[[watch]]` table for one agent to `config.toml`. + + Watch roots still have no default: this writes one only because the user + named the agent. It appends and never changes a byte already there, because + the same file holds the `[remote.*]` push opt-ins; it does nothing when the + table is already there; and it refuses, writing nothing, a table the watcher + would skip. + """ + home = store.resolve_home(args.home) + path = os.path.join(home, "config.toml") + root = daemon.ROOTS[args.agent] + real = os.path.realpath(os.path.expanduser(root)) + try: + # `lstat`: `os.replace` over a symlink swaps the link for a file, so the + # user's dotfile would silently stop being the config. And a FIFO hangs. + st = os.lstat(path) + if not stat.S_ISREG(st.st_mode): + return _refuse(f"{path} is not a regular file; add the table by hand") + # A hard link is the same trap without the `l`: the rename gives the + # name a new inode, and the other name keeps the old bytes. + if st.st_nlink > 1: + return _refuse(f"{path} has another hard link; add the table by hand") + except FileNotFoundError: + pass # no file yet, so the table is the whole of it + old = _config_bytes(path) + try: + before = tomllib.loads(old.decode("utf-8")) + except ValueError as exc: # `TOMLDecodeError`, and `UnicodeDecodeError` on a bad byte + return _refuse(f"{path} is not valid TOML ({exc}); fix it first") + table = ( + f"# added by `gitmemory init {args.agent}`\n" + f'[[watch]]\nagent = "{args.agent}"\nroots = ["{root}"]\n' + ) + sep = b"" if not old else b"\n" if old.endswith(b"\n") else b"\n\n" + new = old + sep + table.encode() + # Re-parsed whole, because appending is not always joining: after a plain + # `[watch]` or an inline `watch = [...]` the result does not parse at all. + # When it parses, the header added one table, last in the `watch` array. + try: + after = tomllib.loads(new.decode("utf-8")) + except ValueError: + return _refuse(f"{path} has a `watch` a [[watch]] table cannot join; add it by hand") + for w in daemon.watches_from(before, path, home, lambda _m: None): + if real in w.roots: + if w.agent == args.agent: + print(f"{path} already watches {root} for {args.agent}; nothing to do") + return 0 + # `discover` gives each file under a shared root to the first watch + # in the file, so a table appended after it gets nothing it takes. + return _refuse(f"{path} already watches {root} as {w.agent!r}") + # The watcher's own rules and its own sentence, so the two cannot disagree + # about why: a table it would skip on every pass is not written at all. + notes: list[str] = [] + if not any( + w.agent == args.agent and real in w.roots + for w in daemon.watches_from(after, path, home, notes.append) + ): + mine = f"{path} [[watch]] #{len(after['watch'])}:" + return _refuse("; ".join(n for n in notes if n.startswith(mine))) + # Two inits at once both read the same bytes, and the later rename dropped + # the other's table while both said "added". Each holds this across a second + # read and the rename, so the later one sees the file move and says so. Not + # taken earlier: it makes `.locks/`, and a refusal writes nothing. An editor + # saving in the same instant takes no lock, and is not covered. + with store._lockfile(os.path.join(home, ".locks", "config.toml.lock")): + if _config_bytes(path) != old: + return _refuse(f"{path} changed while init was reading it; run it again") + store._write_atomic(path, new) # 0600, and a running watcher reads old or new + print(f"{'added to' if old else 'wrote'} {path}: watch {root} for {args.agent}") + if not os.path.isdir(real): + print(f"note: {root} does not exist yet; it is watched once it does", file=sys.stderr) + print("next: run `gitmemory watch`; a watcher already running reads this on its next pass") + return 0 + + def _capture(args) -> int: home = store.resolve_home(args.home) cap = daemon.capture_one( @@ -63,7 +162,7 @@ def _capture(args) -> int: ) # Said before the result line and on stderr, beside `diverged`, because it # is the same kind of fact: something about this store is not the ordinary - # case. A capture that reclaims an orphan reports `+0B`, so without this the + # case. A capture that reclaims an orphan reports `+0 B`, so without this the # loudest thing about the pass is how quiet it was. [E7 fs-F4] if cap.adopted: print( @@ -74,7 +173,8 @@ def _capture(args) -> int: if cap.diverged: print(f"diverged: {cap.diverged}", file=sys.stderr) print(f"sealed generation {cap.generation - 1}; now writing g{cap.generation:02d}") - print(f"{cap.manifest_path} +{cap.appended}B size={cap.size} g{cap.generation:02d}") + added = daemon.size_text(cap.appended) + print(f"{cap.manifest_path} +{added} size={cap.size} g{cap.generation:02d}") return 0 @@ -126,6 +226,21 @@ def _positive_int(text: str) -> int: return value +def _count(text: str) -> int: + """A non-negative integer, for `show --context`, whose default is 0. + + Plain `type=int` took `-C -1`, and the slice that asked for was empty, so + `show` printed nothing and exited 0. + """ + try: + value = int(text) + except ValueError as err: + raise argparse.ArgumentTypeError(f"invalid int value: {text!r}") from err + if value < 0: + raise argparse.ArgumentTypeError(f"cannot be negative: {text}") + return value + + def _watch(args) -> int: home = store.resolve_home(args.home) # The emptiness check used to live here, once, before the loop — so it @@ -142,6 +257,20 @@ def _watch(args) -> int: ) +def _service(args) -> int: + home = store.resolve_home(args.home) + target, body, steps = service.render(sys.platform, home) + if not service.write(target, body): + # One line, with the way out last on it so it pastes whole: the + # removal step, which does not wait on stopping a job never started. + way_out = steps[-1][1] + return _refuse(f"{target} exists; to replace it, run this and install again: {way_out}") + print(f"wrote {target}\nit runs: {shlex.join(service.argv(home))}") + for what, command in steps: + print(f"{what}:\n {command}") + return 0 + + def _not_a_store(home: str | None) -> int | None: """Exit 2 and say so, rather than letting an empty glob read as a clean store. @@ -300,6 +429,15 @@ def _derive(args) -> int: return 0 +def _ago(seconds: float) -> str: + """Minutes, hours or days. Said only past `STALE_RECORD`, so never one.""" + if seconds < 7200: + return f"{int(seconds // 60)} minutes" + if seconds < 172800: + return f"{int(seconds // 3600)} hours" + return f"{int(seconds // 86400)} days" + + def _recall(args) -> int: # First, so a mistyped policy is the error you see, with or without an index. resolve = index.policy(args.policy) @@ -307,6 +445,19 @@ def _recall(args) -> int: if not os.path.exists(path): print(f"no index at {path}; run `gitmemory index` first", file=sys.stderr) return 2 + # Nothing here said the watcher had stopped, and the time since the last + # capture cannot: a running one on an idle machine captures nothing either. + # A record no pass has drained can. Not under `--db`: an index names no store. + if args.db is None: + home = store.resolve_home(args.home) + waiting, age = daemon.uncollected(home) + if waiting and age >= daemon.STALE_RECORD: + print( + f"{waiting} hook record(s) uncollected, the oldest for {_ago(age)}: is " + f"`gitmemory watch` running for {home}? These results may be missing " + "everything said since.", + file=sys.stderr, + ) db = index.open_db(path) try: hits = index.search(db, args.query, k=args.k) @@ -324,13 +475,28 @@ def _recall(args) -> int: shown = list(resolve(hits)) except Exception as exc: # a custom policy is the user's code: report it, no traceback raise ValueError(f"policy {args.policy!r} failed: {exc!r}") from exc + # After the policy, so a custom one cannot route around it. Before the cut: + # a key that straddles character 160 leaves a prefix no detector matches. + # Before the collapse too, with the mask's own, as `show`'s: it joins a PEM + # header a newline split, where `str.split()` also broke on a no-break + # space, which the patterns read as part of a token, and cut the value under + # its floor. `graph._label` measured the cut and the split header. The + # address too: a session id can be a key. + shown_as = functools.partial(redact.mask, collapse=True) if args.redact else str for h in shown: - head = " ".join(h.text.split())[:160] - t = index.when(h) + head = " ".join(shown_as(h.text).split())[:160] # The date on every line, whatever the policy: it is what lets the # reader see that two disagreeing turns are three weeks apart. - date = t.strftime("%Y-%m-%dT%H:%MZ") if t else "undated".ljust(17) - print(f"{h.score:8.3f} {date} {h.session_key}@{h.byte_offset} {h.role}/{h.kind} {head}") + date = _date(h) + # One column still: a session key has no space in it (`store._SAFE_RE`), + # and the one a mask's marker adds would split the address in two. + where = shown_as(f"{h.session_key}@{h.byte_offset}").replace(" ", "") + # In the role's column, not a column of its own: whatever parses these + # lines reads role/kind as the fourth field, and a subagent's user/text + # is the parent agent's instructions, which must not be quoted as the + # user's. + who = f"subagent:{h.role}" if h.is_sidechain else h.role + print(f"{h.score:8.3f} {date} {where} {who}/{h.kind} {head}") if not hits: print("no matches", file=sys.stderr) elif not shown: @@ -340,13 +506,135 @@ def _recall(args) -> int: return 0 +def _date(item) -> str: + """When a hit or a turn was written, as `recall` and `show` both print it: + to the minute in UTC, or `undated` padded to the same width.""" + t = index.when(item) + return t.strftime("%Y-%m-%dT%H:%MZ") if t else "undated".ljust(17) + + +# `//gNN@`, as `recall` prints it. `[0-9]`, not `\d`: +# `\d` is Unicode, and `int()` reads an Arabic-Indic digit as an offset. +_ADDRESS = re.compile(r"([^/]+)/([^/]+)/g([0-9]+)@([0-9]+)\Z") + + +def _show(args) -> int: + """The turn at an address `recall` printed, read from the store. + + The store, not the index: the watcher does not index, and a stale or + hostile index would choose what this prints, when this is how a reader + checks what `recall` said. The parse is `index`'s own, so a turn cut by a + segment boundary comes back whole. + + The address never becomes a path. Its names go through `store.identity`, + the rule `capture` filed them under, so `..`, a dotfile or a newline is + refused before anything is read, and the result is only compared with what + `sessions()` found, which already keeps manifests and segments inside the + store. + """ + m = _ADDRESS.match(args.address) + try: + agent, session_id = store.identity(m[1], m[2]) if m else ("", "") + except ValueError: + m = None + if not m: + # The marker fits `_ADDRESS` and fails `identity`: the user copied it + # whole, from a recall that was masking, and did not mistype it. + if "…[" in args.address: + hint = "recall prints one show can look up when it is not masking" + print(f"a masked address: {redact.safe_path(args.address)!r}; {hint}", file=sys.stderr) + return 2 + want = "expected //gNN@, as recall prints it" + print(f"not an address: {redact.safe_path(args.address)!r}; {want}", file=sys.stderr) + return 2 + if (code := _not_a_store(args.home)) is not None: + return code + gen, offset = int(m[3]), int(m[4]) + key = f"{agent}/{session_id}/g{gen:02d}" + # Each error below names the address as it was typed, masked. A session id + # is whatever `capture --session-id` was handed, so it can be a token, and + # `key` is lowercased, which hides an AWS key id from its detector. + said = redact.safe_path(f"{m[1]}/{m[2]}/g{gen:02d}") + found = [ + s + for s in store.sessions(store.resolve_home(args.home)) + if (s.agent, s.session_id, s.generation) == (agent, session_id, gen) + ] + if len(found) != 1: + # Two is a manifest declaring an identity its path does not have, which + # `verify` reports. Picking one would be choosing the answer. + why = "not in the store" + if found: + why = f"claimed by {len(found)} manifests; run `gitmemory verify`" + print(f"{said}: {why}", file=sys.stderr) + return 1 + turns = index.parse_generation(found[0]).turns + # An offset inside a turn is that turn, as a bench gold offset is; the + # header then gives its start. + at = next( + (i for i, t in enumerate(turns) if t.byte_offset <= offset < t.byte_offset + t.byte_len), + None, + ) + if at is None: + print(f"{said}: no turn at byte {offset}", file=sys.stderr) + return 1 + # Only a header starts in column 0, and `safe_text` leaves "\n" the only + # line break that reaches the terminal, so no transcript can print a line + # that reads as one. Two ways past that were reproduced and are closed + # here: a tool name is any string the transcript chose, so it is collapsed, + # and `splitlines()` broke on "\r", "\x85" and U+2028 before `safe_text` + # could escape them. + # + # `--redact` masks as recall's does, on each block whole, before the split: + # a private key's body is the lines under its header. These lines keep + # their breaks, so only the mask's own collapse joins `-----BEGIN RSA` and + # `PRIVATE KEY-----` across one. + shown_as = functools.partial(redact.mask, collapse=True) if args.redact else str + lo = max(0, at - args.context) + for i, t in enumerate(turns[lo : at + args.context + 1], start=lo): + # A subagent's user turn is the brief its parent wrote, not the user. + who = f"subagent:{t.role}" if t.is_sidechain else t.role + # One field, as in recall: the marker's space would split the address. + where = shown_as(f"{key}@{t.byte_offset}").replace(" ", "") + print(f"{'>' if i == at else '-'} {where} {_date(t)} {who}") + for b in t.blocks: + print(" " + shown_as(" ".join([b.kind, *(b.tool_name or "").split()]))) + for line in shown_as(b.text).split("\n") if b.text else (): + print(f" {line}") + return 0 + + +def _redact_flag(parser: argparse.ArgumentParser) -> None: + """`--redact`, defined once for each command that prints transcript text. + + The flag can only turn masking on. A standing `GITMEMORY_REDACT` is undone + for one call by `GITMEMORY_REDACT=0`, and any value but empty or `0` means + on, so a mistyped one fails toward masking. + """ + parser.add_argument( + "--redact", + action="store_true", + default=os.environ.get("GITMEMORY_REDACT", "") not in ("", "0"), + help="mask the credential shapes the push gate detects in what is printed " + "(default on when $GITMEMORY_REDACT is set to anything but empty or 0)", + ) + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(prog="gitmemory") + # The plugin and the CLI are installed apart and can be releases apart, so + # the recall skill asks this before it trusts `recall`'s columns. argparse + # prints it past the escaping `print` above: fine for a source constant. + ap.add_argument("--version", action="version", version=f"%(prog)s {__version__}") ap.add_argument( "--home", default=None, help="store root (default $GITMEMORY_HOME or ~/.gitmemory)" ) sub = ap.add_subparsers(dest="cmd", required=True) + ini = sub.add_parser("init", help="add an agent's standard [[watch]] table to config.toml") + ini.add_argument("agent", choices=sorted(daemon.ROOTS), help="whose transcript root to watch") + ini.set_defaults(fn=_init) + cap = sub.add_parser("capture", help="copy out bytes appended since the last capture") cap.add_argument("source") cap.add_argument("--agent", default="claude-code") @@ -371,6 +659,13 @@ def main(argv: list[str] | None = None) -> int: wat.add_argument("--no-parse", action="store_true", help="skip adapter; record bytes only") wat.set_defaults(fn=_watch) + svc = sub.add_parser("service", help="keep `watch` running under your user service manager") + act = svc.add_subparsers(dest="action", required=True) + act.add_parser( + "install", + help="write a systemd user unit (Linux) or a LaunchAgent (macOS); starts nothing", + ).set_defaults(fn=_service) + ver = sub.add_parser("verify", help="check every manifest's contiguity proof") ver.set_defaults(fn=_verify) @@ -415,8 +710,17 @@ def main(argv: list[str] | None = None) -> int: help="which hit goes first when they disagree: latest-user, latest, relevance, or " "module:function (default $GITMEMORY_POLICY or %(default)s)", ) + _redact_flag(rec) rec.set_defaults(fn=_recall) + sho = sub.add_parser("show", help="print the turn at an address recall printed, from the store") + sho.add_argument("address", help="//gNN@, as recall prints it") + sho.add_argument( + "-C", "--context", type=_count, default=0, metavar="N", help="also N turns either side" + ) + _redact_flag(sho) + sho.set_defaults(fn=_show) + args = ap.parse_args(argv) try: return args.fn(args) diff --git a/src/gitmemory/daemon.py b/src/gitmemory/daemon.py index a3be2a5..dc6c56e 100644 --- a/src/gitmemory/daemon.py +++ b/src/gitmemory/daemon.py @@ -16,7 +16,8 @@ `config.toml` names. A product whose first run scans `~/.claude/projects` ingests its owner's entire history on install, which is the one thing this project promises not to do. The adapter documents the conventional path; the - config has to name it. + config has to name it. `gitmemory init ` writes it there when asked + by name, and nothing reads `ROOTS` as a default. - **A spool record is a doorbell, not data.** It says something happened to a transcript. Everything else — how many bytes, where the boundary is, whether the file was rewritten — is re-derived from the source file. A record naming a @@ -57,6 +58,11 @@ ERROR_REPEAT = 3600.0 POLL = 5.0 # seconds between ticks STALE_TMP = 3600.0 # a `.tmp-` this old is a killed hook, not a live one +# A complete spool record this old means no watcher pass has run on this store +# since it arrived: every pass drains every record, whatever it names. 720 +# default polls, and no further behind than `INTERVAL` already leaves a growing +# session. Read by `recall`. Its own constant, for `ERROR_REPEAT`'s reason. +STALE_RECORD = 3600.0 # A module constant, not `Watch.pattern`: on a `slots=True` dataclass the class # attribute is the slot's `member_descriptor`, so reading the default off the @@ -112,11 +118,20 @@ class Tick: # Segments this pass attested to without copying a byte — reclaimed from the # generation directory rather than read out of the source. The manifest # records them permanently; this is so the operator hears about it at the - # time, since a repair that shows up only as "+0B" reads like a quiet pass. + # time, since a repair that shows up only as a "+0B" commit reads like a + # quiet pass: with nothing captured, the log has no `captured` line at all. # Not an error: after a killed capture it is the correct outcome. [E7 fs-F4] adopted: list[str] = field(default_factory=list) +# Where each shipped adapter's agent writes its transcripts, as docs/agents.md +# lists them. Not a default: `load_watches` never reads this. Only `gitmemory +# init ` does, and it writes the table into `config.toml` because the +# user named the agent, so the config still names every root that is watched. +# Here and not on the adapters, whose contract is two symbols and nothing else. +ROOTS = {"claude-code": "~/.claude/projects", "pi": "~/.pi/agent/sessions"} + + def load_watches(home: str, log=None) -> list[Watch]: """`[[watch]]` tables from `config.toml`. No config means watch nothing. @@ -168,6 +183,14 @@ def load_watches(home: str, log=None) -> list[Watch]: # this and the specific message is the better one. [E4, review: daemon 3] say(f"{path} is not readable as text ({exc.__class__.__name__}); watching nothing") return [] + return watches_from(cfg, path, home, say) + + +def watches_from(cfg: dict, path: str, home: str, say) -> list[Watch]: + """The `[[watch]]` tables of a config already parsed, with every refusal + `load_watches` applies. Split out so `gitmemory init` asks the watcher's own + question of the table it is about to write, rather than keep a second copy + of these rules that drifts from the first.""" out = [] for n, entry in enumerate(cfg.get("watch") or []): where = f"{path} [[watch]] #{n + 1}" @@ -254,7 +277,13 @@ def load_watches(home: str, log=None) -> list[Watch]: # `~/.gitmemory` is safe only because `_hits` skips dotted # components, which is why that skip is preserved rather than # inherited. [E4, review: CLI 3] - say(f"{where}: root {r} contains the store itself; skipped") + # + # Two directions, two sentences: `gitmemory init` prints this as + # its reason, and a root under the store does not contain it. + how = "contains the store itself" + if not _inside(resolved, home): + how = "is inside the store" + say(f"{where}: root {r} {how}; skipped") continue kept.append(resolved) if kept: @@ -466,6 +495,29 @@ def drain_spool( return wanted, tick +def uncollected(home: str) -> tuple[int, float]: + """`(complete records in the spool, age of the oldest in seconds)`. + + `drain_spool`'s rule for what a record is: a dot-name is a hook still + writing. A directory is not one either: no pass can unlink it, so counting + it warned for ever with a watcher running. Arrival is the record's own + mtime (hook/README.md). Read-only and total, because `recall` asks: + anything it cannot list is `(0, 0.0)`. + """ + now = time.time() + ages = [] + try: + with os.scandir(os.path.join(home, SPOOL)) as entries: + for entry in entries: + if not entry.name.startswith("."): + with contextlib.suppress(OSError): + if not entry.is_dir(follow_symlinks=False): + ages.append(now - entry.stat(follow_symlinks=False).st_mtime) + except OSError: + return 0, 0.0 + return len(ages), max(ages, default=0.0) + + def _file_key(path: str) -> tuple[int, int] | None: """`(st_dev, st_ino)` — the identity two spellings of one file agree on.""" try: @@ -964,6 +1016,25 @@ def tick( return result +def size_text(n: int) -> str: + """How much a pass captured, for a person: `392.0 MB (391995999 B)`. + + `+391995999B` made the reader count digits. The exact count stays beside the + rounded one for anyone doing byte arithmetic, and the units are decimal so + the two can be checked against each other by moving a point. Amounts only: + `size=` and every offset are coordinates, and stay plain bytes. + """ + if n < 1000: + return f"{n} B" + x = float(n) + for unit in ("kB", "MB", "GB", "TB"): + x /= 1000 + # 999.95 and not 1000: `.1f` would round 999.96 kB up to "1000.0 kB". + if x < 999.95 or unit == "TB": + break + return f"{x:.1f} {unit} ({n} B)" + + def run( home: str | None = None, *, @@ -1125,7 +1196,7 @@ def run( else "" ) log( - f"captured {len(result.captured)} +{result.appended}B " + f"captured {len(result.captured)} +{size_text(result.appended)} " f"commit={result.commit}{dropped}" ) if once: diff --git a/src/gitmemory/derive.py b/src/gitmemory/derive.py index 0a39123..173eca1 100644 --- a/src/gitmemory/derive.py +++ b/src/gitmemory/derive.py @@ -139,8 +139,8 @@ def _sumy(): from sumy.utils import get_stop_words except ImportError as exc: # pragma: no cover - exercised by hand, not in CI raise RuntimeError( - "derivation needs the `derive` extra: " - 'uv tool install "gitmemory[derive] @ git+https://github.com/doronp/gitmemory"' + "derivation needs the `derive` extra: uv tool install --python 3.13 " + '"gitmemory[derive] @ git+https://github.com/doronp/gitmemory"' " (from a clone: uv sync --extra derive)" ) from exc diff --git a/src/gitmemory/gitrepo.py b/src/gitmemory/gitrepo.py index 62c680c..fba4891 100644 --- a/src/gitmemory/gitrepo.py +++ b/src/gitmemory/gitrepo.py @@ -79,6 +79,10 @@ # about to push to that same remote. `push_allowed` now refuses an inline # credential outright; this line is the belt to that brace, and it also keeps a # store's remote list out of a repository other people can read. [E7] +# +# And its temp. `gitmemory init` publishes the file by rename, so an init killed +# between the write and the `os.replace` leaves a whole copy of it beside it. +/config.toml.tmp.* /config.toml """ diff --git a/src/gitmemory/index.py b/src/gitmemory/index.py index 6f278ae..ef2fb75 100644 --- a/src/gitmemory/index.py +++ b/src/gitmemory/index.py @@ -433,6 +433,9 @@ class Hit: score: float # BM25; more negative is a better match, as FTS5 defines it text: str ts: str | None = None # the turn's timestamp exactly as the agent wrote it + # A subagent's turn (`Turn.is_sidechain`), whichever file it was written in. + # Its user-role text is the parent agent's instructions, not the user's. + is_sidechain: bool = False @dataclass(frozen=True, slots=True) @@ -1016,7 +1019,7 @@ def search( SELECT b.byte_offset, b.byte_len, b.turn_id, b.session_key, b.agent, b.session_id, b.generation, b.role, b.kind, b.block_seq, b.prose || b.tool_use || b.tool_result AS text, s.score AS score, - b.ts, + b.ts, b.turn_seq, -- Partitioned by turn *and session*, not by turn alone. -- `turn_id` is content-derived over the record's sessionId, -- which the Claude Code adapter warns is reused across a @@ -1037,7 +1040,15 @@ def search( FROM scored s JOIN blocks b ON b.rowid = s.rid ) SELECT byte_offset, byte_len, turn_id, session_key, agent, - session_id, generation, role, kind, text, score, ts + session_id, generation, role, kind, text, score, ts, + -- Only `turns` knows whose turn it was. A lookup on its primary + -- key, not a join, so the ORDER BY's names stay unambiguous; on + -- 120k turns, against queries that match most of them, it timed + -- within noise of no lookup at all. + COALESCE(( + SELECT t.is_sidechain FROM turns t + WHERE t.session_key = ranked.session_key AND t.seq = ranked.turn_seq + ), 0) AS is_sidechain FROM ranked WHERE rn = 1 -- Ties are broken by position, never by rowid: the answer must not -- depend on the order generations happened to be indexed in. @@ -1060,6 +1071,7 @@ def search( score=r["score"], text=r["text"], ts=r["ts"], + is_sidechain=bool(r["is_sidechain"]), ) for r in rows ) @@ -1131,9 +1143,14 @@ def latest_user(hits: list[Hit]) -> list[Hit]: an earlier session, which is newer than every turn it quotes. A user who reverses themselves still wins here, because their reversal is user text. - "The user's own words" is the admission rule `derive` uses for prose: a - `text` block in the user role that no program injected. A tool result is a - user-role line in Claude Code and is not something the user said. + "The user's own words" is the admission rule `derive` uses for prose, a + `text` block in the user role that no program injected, less a subagent's + turn. A tool result is a user-role line in Claude Code and is not something + the user said, and neither is a subagent's user text: that is the parent + agent's instructions, its dispatch prompt first, which led the default as + the user's word while it was newer than the rule it contradicted. + `derive._prose` still admits a subagent's text; excluding it there too would + move the secondary-set numbers, so that waits for a re-measurement. ponytail: a skill's loaded instructions and a slash command's expansion are user text with no marker, so they pass as the user's words — the same gap @@ -1142,7 +1159,12 @@ def latest_user(hits: list[Hit]) -> list[Hit]: from .derive import _injected # lazy: derive imports this module def typed(h: Hit) -> bool: - return h.role == "user" and h.kind == "text" and not _injected(h.text.strip()) + return ( + h.role == "user" + and h.kind == "text" + and not h.is_sidechain + and not _injected(h.text.strip()) + ) ordered = latest(hits) return [h for h in ordered if typed(h)] + [h for h in ordered if not typed(h)] diff --git a/src/gitmemory/redact.py b/src/gitmemory/redact.py index 8836f0b..2489e49 100644 --- a/src/gitmemory/redact.py +++ b/src/gitmemory/redact.py @@ -6,7 +6,8 @@ would make the bytes we commit not the bytes the agent wrote (DESIGN.md §2.5a records agentcairn losing exactly that way). So redaction lives at the two boundaries where bytes leave: anything written to `derived/`, and anything -pushed to a remote. +pushed to a remote. `mask` also serves `recall --redact` and `show --redact`, +which mask what they print when the owner asks them to, and block nothing. Two tiers, because a one-tier gate is either useless or unpassable: @@ -30,18 +31,20 @@ from __future__ import annotations +import bisect import os import re import stat import tomllib import urllib.parse from collections.abc import Iterable -from dataclasses import dataclass +from dataclasses import dataclass, replace __all__ = [ "Finding", "contents", "gate", + "mask", "push_allowed", "safe_path", "scan_bytes", @@ -172,18 +175,107 @@ def safe_path(path: str) -> str: append-only. Masking the span rather than the whole string keeps the finding useful: it still names the directory and the file. [E7] """ - data = path.encode("utf-8", "surrogateescape") - found = scan_bytes(data) - if not found: - return path - out, cursor = b"", 0 + return mask(path) + + +# Where a PEM block ends. Not the detector with BEGIN swapped for END: the gate +# refuses on the header alone, and this only has to say where the key stops. +# Bounded, because `[A-Z0-9 ]*KEY[A-Z ]*` retried every `KEY` in a `-----END +# KEY KEY …` line to the end of it: 125 KB took 4.1 s. The longest real END +# line has eighteen characters before `KEY` and six after; one longer than the +# bounds is not found, and the mask runs to the end of the text instead. +_PEM_END = re.compile(rb"-----END [A-Z0-9 ]{0,40}KEY[A-Z ]{0,10}-----") + +# How `records.safe_text` spells a control or an invisible character, which is +# how the index stores one: `\x7f`, `\u200b`, and `\ue0041` for a tag character. +_SPELLED = re.compile(rb"\\x[0-9a-f]{2}|\\u(?:e00[0-7][0-9a-f]|[0-9a-f]{4})") + +# A whitespace run that `" ".join(text.split())` narrows: two or more, or one +# that is not a space. Greedy and unanchored on either side, so it stays linear. +_RUN = re.compile(rb"\s{2,}|[\t\n\r\f\v]") + + +def mask(text: str, *, collapse: bool = False) -> str: + """`text` with every match of both tiers masked, the rest left readable. + + One span is widened. `private_key_block` matches the BEGIN line, and the + secret is the base64 under it: masking the match alone printed the header + as `----…[31 bytes]` and then every byte of the key. So that span runs to + the next END line, or to the end of `text` when there is none. + + Text holding a spelled-out control is scanned twice. Every spelling ends in + a word character, so `\\ufeffAKIA…` has no `\\b` before the key, where the + push gate, reading the raw bytes, sees a BOM and a boundary. The second scan + reads a copy with each spelling blanked to as many spaces, so its offsets + are the text's. The first stays, because a spelling can sit inside a match + (`password=abc\\x01defghijklmn`) that the blanked copy cuts short. + + `collapse` is for a caller that masks before it flattens, or never does. + `-----BEGIN RSA\\nPRIVATE KEY-----` is a header only once flattened, so such + a caller asks for a third scan, of the blanked copy with every whitespace + run one space. `recall` and `show` both do: flattening first with + `str.split()` broke on a no-break space, which is a token byte here. Its END + line is still looked for as written, so one a break split is not found, and + the mask runs to the end of the text. + """ + # `surrogatepass`, as `records` hashes: `show` masks the parsed text, where + # a JSON `\ud83d` is a lone surrogate, and `surrogateescape` raised on any + # outside U+DC80-U+DCFF. The patterns are bytes; the result decodes lossily. + data = text.encode("utf-8", "surrogatepass") + plain = _SPELLED.sub(lambda m: b" " * len(m[0]), data) + found = scan_bytes(data) + (scan_bytes(plain) if plain != data else []) + if collapse: + found += _found_collapsed(plain) + widened, reach = [], -1 + for f in sorted(found, key=lambda f: f.offset): + # Only a header no earlier span already covers. Each one searching for + # its own END line made a page of headers with none cost headers x + # length: 16,000 of them, 437 KB, took 2.4 s. + if f.detector == "private_key_block" and f.offset >= reach: + end = _PEM_END.search(plain, f.offset + f.length) + reach = end.end() if end else len(data) + f = replace(f, length=reach - f.offset) + widened.append(f) + if not widened: + return text + out, cursor = [], 0 # Merged spans, because two detectors matching the same key (`sk-ant-…` # trips both `anthropic_api_key` and `openai_api_key`) would otherwise mask - # it twice and mangle the rest of the path with it. - for start, end in _merged(found): - out += data[cursor:start] + _mask(data[start:end]).encode("utf-8") + # it twice and mangle the rest of the text with it. Joined once at the end: + # `bytes +=` copies everything so far for every span. + for start, end in _merged(widened): + out += (data[cursor:start], _mask(data[start:end]).encode("utf-8")) cursor = end - return (out + data[cursor:]).decode("utf-8", "replace") + out.append(data[cursor:]) + return b"".join(out).decode("utf-8", "replace") + + +def _found_collapsed(data: bytes) -> list[Finding]: + """What a scan finds in `data` with each whitespace run one space, at the + offsets of `data`: a match that starts or ends on a run's space covers the + whole run.""" + pieces, spaces, ends, cursor, shift = [], [], [], 0, 0 + for m in _RUN.finditer(data): + pieces += (data[cursor : m.start()], b" ") + # Where the run's one space lands in the copy, and where the run ends. + spaces.append(m.start() - shift) + ends.append(m.end()) + shift += len(m[0]) - 1 + cursor = m.end() + if not spaces: + return [] + pieces.append(data[cursor:]) + + def back(x: int) -> int: + # The last run whose space is before `x`; past it, bytes map one to one. + i = bisect.bisect_left(spaces, x) - 1 + return x if i < 0 else ends[i] + x - spaces[i] - 1 + + found = [] + for f in scan_bytes(b"".join(pieces)): + start = back(f.offset) + found.append(replace(f, offset=start, length=back(f.offset + f.length) - start)) + return found def _merged(findings: list[Finding]) -> list[tuple[int, int]]: diff --git a/src/gitmemory/service.py b/src/gitmemory/service.py new file mode 100644 index 0000000..d68592c --- /dev/null +++ b/src/gitmemory/service.py @@ -0,0 +1,162 @@ +# SPDX-FileCopyrightText: 2026 The gitmemory authors +# SPDX-License-Identifier: Apache-2.0 +"""`gitmemory service install`: the file that keeps `watch` running. + +A systemd user unit on Linux, a LaunchAgent on macOS. Written, never enabled: +the command prints the line that starts it and runs no `systemctl`, +`launchctl` or `loginctl` itself. Installing it is the user's action, which is +the hook's rule too (hook/README.md), and a service manager's failures read +better in the user's own terminal than translated here. + +What it is for is the facts only the running process has. A user service gets +none of the shell's state — not its working directory, not an exported +`$GITMEMORY_HOME`, not its `PATH` — so a hand-written unit can start a +different Python, watch a different store, or not find git. Each of those was +reproduced; each is written into the file instead of inherited. +""" + +from __future__ import annotations + +import os +import plistlib +import shlex +import shutil +import sys + +LABEL = "io.github.doronp.gitmemory" +UNIT = "gitmemory.service" +DOCS = "https://github.com/doronp/gitmemory/blob/main/docs/watching.md#keep-it-running" + + +def argv(home: str) -> list[str]: + """The watcher's command line, for a manager that inherits nothing. + + `sys.executable` as given: its `realpath` is the base interpreter outside + the venv, which cannot import gitmemory. `-P`, because a systemd user unit + starts in `$HOME` and `python -m` puts that first on `sys.path`, so a stray + `~/gitmemory.py` ran in place of the package. `--home` always, resolved + now: the manager never sees the shell's `$GITMEMORY_HOME`, and the watcher + would otherwise read a store the hook does not write to. + """ + if not sys.executable: + # Empty or None when Python cannot say where it is; a unit naming "" + # cannot start. + raise RuntimeError( + "cannot tell which Python is running; run this from an installed gitmemory" + ) + return [sys.executable, "-P", "-m", "gitmemory", "--home", home, "watch"] + + +def _word(text: str) -> str: + """One double-quoted word in a unit file (systemd.syntax(7)): `\\` and `"` + escaped, `%` doubled so a store path spelling `%h` is not a specifier.""" + return '"' + text.replace("\\", "\\\\").replace('"', '\\"').replace("%", "%%") + '"' + + +def render(platform: str, home: str) -> tuple[str, bytes, list[tuple[str, str]]]: + """`(path, contents, steps)`: where the file goes, its bytes, and the + `(what, command)` lines the user runs, the removal last.""" + if platform not in ("linux", "darwin"): + raise ValueError( + f"no user service manager known for {platform}; " + "run `gitmemory watch` under your own supervisor" + ) + git = shutil.which("git") + if git is None: + raise RuntimeError("git is not on PATH; the watcher needs it to commit") + # gitrepo finds git through PATH, and a manager's PATH is not the shell's: + # a git from Homebrew or nix is on one and not the other. Absolute, since a + # relative PATH entry would resolve against the service's working + # directory; not realpath, so a symlinked git keeps the shell's spelling. + path = f"{os.path.abspath(os.path.dirname(git))}:/usr/bin:/bin" + words = argv(home) + # Checked once for both: a unit file would end the line there, and plistlib + # turns `\r` into `\n`, so a plist would name a path that does not exist. + for word in (*words, path): + if "\n" in word or "\r" in word: + raise ValueError(f"a service file cannot hold a line break: {word!r}") + if platform == "linux": + # The XDG rule: a relative value is invalid and is ignored, and a unit + # written relative to wherever this ran is one no user manager reads. + base = os.environ.get("XDG_CONFIG_HOME", "") + if not os.path.isabs(base): + base = os.path.expanduser("~/.config") + target = os.path.join(base, "systemd", "user", UNIT) + # `$` is expanded on a command line and has no meaning in Environment=, + # so it is doubled in one and not the other. No WorkingDirectory: a + # user unit's default is `$HOME`, which is what the plist below sets. + body = ( + "[Unit]\n" + "Description=gitmemory watcher\n" + f"Documentation={DOCS}\n" + "\n" + "[Service]\n" + f"ExecStart={' '.join(_word(w.replace('$', '$$')) for w in words)}\n" + f"Environment={_word('PATH=' + path)}\n" + "Restart=on-failure\n" + "RestartSec=30\n" + "\n" + "[Install]\n" + "WantedBy=default.target\n" + ).encode() + ctl = "systemctl --user" + steps = [ + ("Nothing is running yet. Start it now and at every login", + f"{ctl} daemon-reload && {ctl} enable --now {UNIT}"), + ("Keep it running after you log out, on a machine you reach over SSH", + "loginctl enable-linger"), + ("Check it", f"{ctl} status {UNIT}"), + ("Its log", f"journalctl --user -u {UNIT} -f"), + ("After upgrading gitmemory", f"{ctl} restart {UNIT}"), + # `;`: the file goes even when the manager cannot be reached. The + # reload after it is so the manager drops the unit it had loaded. + ("Remove it", + f"{ctl} disable --now {UNIT}; rm {shlex.quote(target)}; {ctl} daemon-reload"), + ] + return target, body, steps + target = os.path.expanduser(f"~/Library/LaunchAgents/{LABEL}.plist") + # Not in the store: the watcher commits it with `git add --all`. + log = os.path.expanduser("~/Library/Logs/gitmemory.log") + body = plistlib.dumps( + { + "Label": LABEL, + "ProgramArguments": words, + "EnvironmentVariables": {"PATH": path}, + # launchd starts a job in `/`; a relative watch root should resolve + # where it does under systemd. + "WorkingDirectory": os.path.expanduser("~"), + "RunAtLoad": True, + "KeepAlive": {"SuccessfulExit": False}, + "ThrottleInterval": 30, + "StandardOutPath": log, + "StandardErrorPath": log, + } + ) + job = f"gui/$(id -u)/{LABEL}" + # `;`, not `&&`, in the removal: `bootout` of a job that was written and + # never bootstrapped exits 3, and the file has to go all the same. + steps = [ + ("Nothing is running yet. Start it now and at every login", + f"launchctl bootstrap gui/$(id -u) {shlex.quote(target)}"), + ("Check it", f"launchctl print {job}"), + ("Its log", f"tail -f {shlex.quote(log)}"), + ("After upgrading gitmemory", f"launchctl kickstart -k {job}"), + ("Remove it", f"launchctl bootout {job}; rm {shlex.quote(target)}"), + ] + return target, body, steps + + +def write(path: str, body: bytes) -> bool: + """Create `path` with `body`, or return False and touch nothing if it exists. + + `O_EXCL`, not a check then a write: a file already there may be one the + user edited, and a second install is not a reason to lose it. + """ + os.makedirs(os.path.dirname(path), exist_ok=True) + try: + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) + except FileExistsError: + return False + with os.fdopen(fd, "wb") as fh: + fh.write(body) + return True diff --git a/tests/conftest.py b/tests/conftest.py index 9a04c23..d95705f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,6 +35,10 @@ def _no_real_home(tmp_path, monkeypatch): home.mkdir(exist_ok=True) monkeypatch.setenv("HOME", str(home)) monkeypatch.delenv("GITMEMORY_HOME", raising=False) + # The docs tell the owner to make masking standing in `settings.json`, and a + # Claude Code session running this suite inherits that: every test that + # reads recall's default output would read the masked one instead. + monkeypatch.delenv("GITMEMORY_REDACT", raising=False) return home diff --git a/tests/mutate_index.py b/tests/mutate_index.py index 958e5fb..5aa1498 100644 --- a/tests/mutate_index.py +++ b/tests/mutate_index.py @@ -3852,6 +3852,205 @@ " if False:", "test_recall_says_a_query_was_truncated_before_it_says_no_matches", ), + # --- install feedback: the hook rang and no watcher answered ----------- # + ( + "recall says nothing when the hook rang and no watcher collected it", + "__main__.py", + " if waiting and age >= daemon.STALE_RECORD:", + " if False:", + "test_recall_says_the_hook_rang_and_no_watcher_answered", + ), + ( + "recall reports a spool record a watcher could still be draining", + "__main__.py", + " if waiting and age >= daemon.STALE_RECORD:", + " if waiting:", + "test_recall_says_nothing_about_a_spool_a_watcher_could_still_be_draining", + ), + ( + "a hook's temp file counts as an uncollected record", + "daemon.py", + ' if not entry.name.startswith("."):', + " if True:", + "test_recall_says_nothing_about_a_spool_a_watcher_could_still_be_draining", + ), + ( + "a directory no pass can unlink counts as an uncollected record", + "daemon.py", + " if not entry.is_dir(follow_symlinks=False):", + " if True:", + "test_one_watcher_pass_is_what_clears_it", + ), + ( + "recall reads a store's spool for an index named by --db", + "__main__.py", + " if args.db is None:", + " if True:", + "test_recall_says_nothing_about_a_spool_a_watcher_could_still_be_draining", + ), + ( + "the uncollected age is the newest record's, not the oldest's", + "daemon.py", + "max(ages, default=0.0)", + "min(ages, default=0.0)", + "test_recall_says_the_hook_rang_and_no_watcher_answered", + ), + ( + "a watcher pass leaves the records it consumed in the spool", + "daemon.py", + " _unlink(path)\n tick.spool_consumed += 1", + " tick.spool_consumed += 1", + "test_one_watcher_pass_is_what_clears_it", + ), + # --- recall --redact: masked where the detectors can still see the key -- # + ( + "recall --redact masks the head after the 160-character cut", + "__main__.py", + ' head = " ".join(shown_as(h.text).split())[:160]', + ' head = shown_as(" ".join(h.text.split())[:160])', + "test_recall_redact_masks_a_key_the_160_character_cut_would_have_halved", + ), + ( + # The mask joins a header a line break split itself, as show's does. + "recall --redact masks the head before the collapse assembles a header", + "__main__.py", + " shown_as = functools.partial(redact.mask, collapse=True) if args.redact else str\n" + " for h in shown:", + " shown_as = redact.mask if args.redact else str\n for h in shown:", + "test_recall_redact_masks_a_header_that_only_the_collapse_assembles", + ), + ( + # `str.split()` breaks on a no-break space; the bytes patterns do not. + "recall --redact collapses a Unicode space out of a value before it masks", + "__main__.py", + ' head = " ".join(shown_as(h.text).split())[:160]', + ' head = shown_as(" ".join(h.text.split()))[:160]', + "test_recall_redact_masks_a_value_with_a_unicode_space_in_it", + ), + ( + "recall --redact masks a private key's header and prints its body", + "redact.py", + " f = replace(f, length=reach - f.offset)\n", + "", + "test_recall_redact_masks_a_private_key_body_and_not_only_its_header", + ), + ( + "recall --redact prints a session id that is itself a credential", + "__main__.py", + ' where = shown_as(f"{h.session_key}@{h.byte_offset}").replace(" ", "")', + ' where = f"{h.session_key}@{h.byte_offset}".replace(" ", "")', + "test_recall_redact_masks_a_session_id_that_is_itself_a_credential", + ), + ( + # The marker is `ghp_…[40 bytes]`, and its space moved role/kind to [4]. + "recall --redact splits a masked address into two columns", + "__main__.py", + ' where = shown_as(f"{h.session_key}@{h.byte_offset}").replace(" ", "")', + ' where = shown_as(f"{h.session_key}@{h.byte_offset}")', + "test_recall_redact_masks_a_session_id_that_is_itself_a_credential", + ), + ( + # The obvious spelling, and it reads `GITMEMORY_REDACT=0` as on. + "recall --redact reads GITMEMORY_REDACT=0 as on", + "__main__.py", + ' default=os.environ.get("GITMEMORY_REDACT", "") not in ("", "0"),', + ' default=bool(os.environ.get("GITMEMORY_REDACT")),', + "test_gitmemory_redact_makes_masking_standing_and_0_turns_it_off", + ), + ( + # The index spells a BOM `\ufeff`, and its `f` takes away the key's `\b`. + "recall --redact prints a key behind a control the index spells out", + "redact.py", + ' plain = _SPELLED.sub(lambda m: b" " * len(m[0]), data)', + " plain = data", + "test_recall_redact_masks_a_key_behind_a_character_the_index_spells_out", + ), + ( + "recall --redact scans only the copy with the spellings blanked", + "redact.py", + " found = scan_bytes(data) + (scan_bytes(plain) if plain != data else [])", + " found = scan_bytes(plain)", + "test_recall_redact_masks_a_key_behind_a_character_the_index_spells_out", + ), + ( + "every private key header searches for its own END line again", + "redact.py", + ' if f.detector == "private_key_block" and f.offset >= reach:', + ' if f.detector == "private_key_block":', + "test_mask_stays_linear_on_a_page_of_key_headers_and_an_end_line_that_never_closes", + ), + ( + "the END line pattern's runs are unbounded again", + "redact.py", + "{0,40}KEY[A-Z ]{0,10}-----", + "*KEY[A-Z ]*-----", + "test_mask_stays_linear_on_a_page_of_key_headers_and_an_end_line_that_never_closes", + ), + # --- show --redact: the whole turn, masked as recall's line is ---------- # + ( + # A private key's body is the lines under its header, and no detector + # matches a line of base64 on its own. + "show --redact masks each line apart and prints a private key's body", + "__main__.py", + ' for line in shown_as(b.text).split("\\n") if b.text else ():', + ' for line in map(shown_as, b.text.split("\\n")) if b.text else ():', + "test_show_redact_masks_a_whole_turn_and_the_default_prints_it", + ), + ( + "show --redact prints a session id that is itself a credential", + "__main__.py", + ' where = shown_as(f"{key}@{t.byte_offset}").replace(" ", "")', + ' where = f"{key}@{t.byte_offset}"', + "test_show_redact_masks_a_whole_turn_and_the_default_prints_it", + ), + ( + "show --redact splits a masked address into two fields", + "__main__.py", + ' where = shown_as(f"{key}@{t.byte_offset}").replace(" ", "")', + ' where = shown_as(f"{key}@{t.byte_offset}")', + "test_show_redact_masks_a_whole_turn_and_the_default_prints_it", + ), + ( + "show --redact prints a tool's name unmasked", + "__main__.py", + ' print(" " + shown_as(" ".join([b.kind, *(b.tool_name or "").split()])))', + ' print(" " + " ".join([b.kind, *(b.tool_name or "").split()]))', + "test_show_redact_masks_a_whole_turn_and_the_default_prints_it", + ), + ( + # show prints the line breaks, so only the mask's own collapse joins one. + "show --redact masks without joining a key header a line break split", + "__main__.py", + "functools.partial(redact.mask, collapse=True) if args.redact else str\n" + " lo = max(0, at - args.context)", + "functools.partial(redact.mask, collapse=False) if args.redact else str\n" + " lo = max(0, at - args.context)", + "test_show_redact_joins_a_key_header_a_line_break_split", + ), + ( + # `surrogateescape` round-trips U+DC80-U+DCFF only; `\ud83d` raised. + "show --redact stops on a lone surrogate the mask cannot encode", + "redact.py", + ' data = text.encode("utf-8", "surrogatepass")', + ' data = text.encode("utf-8", "surrogateescape")', + "test_show_redact_prints_a_turn_that_holds_a_lone_surrogate", + ), + ( + "show refuses a masked address as one recall never printed", + "__main__.py", + ' if "…[" in args.address:', + " if False:", + "test_a_masked_address_is_refused_as_masked_not_as_mistyped", + ), + ( + # The copy is shorter by every run it narrowed, so its offsets are not + # the text's: unmapped, the mask eats the bytes before the key. + "the collapsed scan masks at the copy's offsets, not the text's", + "redact.py", + " return x if i < 0 else ends[i] + x - spaces[i] - 1", + " return x", + "test_show_redact_joins_a_key_header_a_line_break_split", + ), # --- E7 index-F8: the caps bound the shape of the input, not its cost -- # ( "sumy's own quadratic idf is used again", @@ -4597,7 +4796,7 @@ # anchor appear twice and the harness would skip itself. [E7] "the docstring's row count drifts away from the index again", "tests/mutate_index.py", - "a full pass is 6" + "06 mutants", + "a full pass is 6" + "62 mutants", "a full pass is 14" + "6 mutants", "test_the_row_count_in_the_docstring_is_the_row_count", ), @@ -5426,6 +5625,63 @@ " if False:\n", "test_a_non_positive_count_is_refused_at_the_command_line", ), + # `gitmemory show`: the address is input, and the output is a terminal's. + ( + "show takes an address's names without the store's path rule", + "__main__.py", + ' agent, session_id = store.identity(m[1], m[2]) if m else ("", "")', + ' agent, session_id = (m[1], m[2]) if m else ("", "")', + "test_a_malformed_or_escaping_address_is_not_an_address", + ), + ( + "show reads an Arabic-Indic digit as an offset", + "__main__.py", + '_ADDRESS = re.compile(r"([^/]+)/([^/]+)/g([0-9]+)@([0-9]+)\\Z")', + '_ADDRESS = re.compile(r"([^/]+)/([^/]+)/g(\\d+)@(\\d+)\\Z")', + "test_a_malformed_or_escaping_address_is_not_an_address", + ), + ( + "show picks one of two manifests that claim an address", + "__main__.py", + " if len(found) != 1:", + " if not found:", + "test_two_manifests_claiming_one_address_are_refused", + ), + ( + "show prints a credential-shaped session id in its error", + "__main__.py", + ' said = redact.safe_path(f"{m[1]}/{m[2]}/g{gen:02d}")', + " said = key", + "test_a_credential_in_an_address_is_masked_in_the_error", + ), + ( + "show masks the lowercased key, which hides an AWS key id", + "__main__.py", + ' said = redact.safe_path(f"{m[1]}/{m[2]}/g{gen:02d}")', + " said = redact.safe_path(key)", + "test_a_credential_in_an_address_is_masked_in_the_error", + ), + ( + "show prints a tool name's newline, so a transcript forges a header", + "__main__.py", + ' print(" " + shown_as(" ".join([b.kind, *(b.tool_name or "").split()])))', + ' print(" " + shown_as(" ".join([b.kind, *filter(None, [b.tool_name])])))', + "test_a_transcript_cannot_forge_a_show_header_or_reach_the_terminal", + ), + ( + "show splits block text before a carriage return is escaped", + "__main__.py", + ' for line in shown_as(b.text).split("\\n") if b.text else ():', + " for line in shown_as(b.text).splitlines() if b.text else ():", + "test_a_transcript_cannot_forge_a_show_header_or_reach_the_terminal", + ), + ( + "show takes a negative --context and prints nothing", + "__main__.py", + '"-C", "--context", type=_count,', + '"-C", "--context", type=int,', + "test_show_on_no_store_and_a_negative_context_exit_2", + ), ( "every ssh login name but git is refused as a credential", "redact.py", @@ -5500,6 +5756,179 @@ " except ZeroDivisionError as e:", "test_a_call_that_fails_every_retry_scores_wrong_and_is_counted", ), + # --- a subagent's dispatch prompt led `latest-user` as the user's word --- + ( + "latest-user takes a subagent's dispatch prompt for the user's word", + "index.py", + " and not h.is_sidechain\n", + "", + "test_a_subagent_prompt_is_not_the_users_word", + ), + ( + "search reads every turn as a main session's", + "index.py", + " COALESCE((\n" + " SELECT t.is_sidechain FROM turns t\n" + " WHERE t.session_key = ranked.session_key AND t.seq = ranked.turn_seq\n" + " ), 0) AS is_sidechain", + " 0 AS is_sidechain", + "test_a_sidechain_line_inside_a_main_file_is_marked", + ), + ( + "recall prints a subagent's turn with the user's role", + "__main__.py", + ' who = f"subagent:{h.role}" if h.is_sidechain else h.role', + " who = h.role", + "test_recall_marks_a_subagent_turn", + ), + # --- `gitmemory init `: the config is appended to, or left alone --- + ( + # `config.toml` also holds `[remote.*] allow_push`, so a rewrite revokes + # a push opt-in nobody asked to revoke. + "init overwrites config.toml instead of appending to it", + "__main__.py", + " new = old + sep + table.encode()", + " new = table.encode()", + "test_cli_init_appends_and_keeps_every_byte_already_there", + ), + ( + "init follows a symlinked config.toml and replaces the link with a file", + "__main__.py", + " st = os.lstat(path)", + " st = os.stat(path)", + "test_cli_init_does_not_replace_a_symlinked_config", + ), + ( + # The rename gives the name a new inode; the dotfile keeps the old bytes. + "init replaces a hard-linked config.toml and splits it from the other name", + "__main__.py", + " if st.st_nlink > 1:", + " if False:", + "test_cli_init_does_not_split_a_hard_linked_config", + ), + ( + # Unreadable taken for absent: the file is replaced by the table alone. + "init takes a config.toml it cannot read for a missing one", + "__main__.py", + ' except FileNotFoundError:\n return b""', + ' except OSError:\n return b""', + "test_cli_init_says_error_and_exits_2_when_it_could_not_look", + ), + ( + # init prints this as its reason; the store contains the root, not + # the other way round. + "a root inside the store is said to contain the store", + "daemon.py", + ' how = "is inside the store"', + ' how = "contains the store itself"', + "test_cli_init_refuses_a_table_the_watcher_would_skip_and_writes_nothing", + ), + ( + # The table is written, and then skipped by the watcher on every pass. + "init writes a table the watcher would skip", + "__main__.py", + " if not any(\n w.agent == args.agent and real in w.roots\n", + " if False and not any(\n w.agent == args.agent and real in w.roots\n", + "test_cli_init_refuses_a_table_the_watcher_would_skip_and_writes_nothing", + ), + ( + # `discover` gives a root to the first watch naming it: dead on arrival. + "init adds a second watch on a root another agent already watches", + "__main__.py", + ' return _refuse(f"{path} already watches {root} as {w.agent!r}")', + " continue", + "test_cli_init_refuses_a_root_another_agent_already_watches", + ), + ( + # Two inits read the same bytes; the later rename drops the other's + # table, and both say "added". + "init renames over a config another init wrote after it read it", + "__main__.py", + " if _config_bytes(path) != old:", + " if False:", + "test_cli_init_says_so_when_another_init_wrote_first", + ), + ( + "a killed init's temp copy of config.toml is committed", + "gitrepo.py", + "/config.toml.tmp.*\n", + "", + "test_cli_init_leaves_no_temp_the_watcher_would_commit", + ), + # --- `gitmemory service install`: the file a user service manager runs --- + ( + "service install overwrites a unit file that is already there", + "service.py", + "os.O_WRONLY | os.O_CREAT | os.O_EXCL", + "os.O_WRONLY | os.O_CREAT | os.O_TRUNC", + "test_service_install_refuses_to_overwrite_an_existing_file", + ), + ( + "the service runs python -m without -P, so a gitmemory.py in $HOME shadows it", + "service.py", + 'sys.executable, "-P", "-m", "gitmemory"', + 'sys.executable, "-m", "gitmemory"', + "test_the_installed_service_runs_the_watcher_from_its_own_working_directory", + ), + ( + "the service resolves sys.executable to an interpreter outside the venv", + "service.py", + "return [sys.executable, ", + "return [os.path.realpath(sys.executable), ", + "test_the_installed_service_runs_the_watcher_from_its_own_working_directory", + ), + ( + # The quoting test's decoder used to turn `%%` into `%`, which let an + # undoubled `%h` through as itself: this mutant survived. [review] + "the unit leaves % undoubled, so a store path spelling %h names the home directory", + "service.py", + '.replace("%", "%%")', + '.replace("%", "%")', + "test_a_unit_survives_a_home_with_quotes_percent_and_dollar_in_it", + ), + ( + "ExecStart leaves $ undoubled, so a store path spelling $HOME is expanded", + "service.py", + "_word(w.replace('$', '$$'))", + "_word(w)", + "test_a_unit_survives_a_home_with_quotes_percent_and_dollar_in_it", + ), + ( + "the printed removal waits on bootout, so a plist never bootstrapped is never removed", + "service.py", + 'f"launchctl bootout {job}; rm {shlex.quote(target)}"', + 'f"launchctl bootout {job} && rm {shlex.quote(target)}"', + "test_the_printed_removal_works_when_the_job_was_never_started", + ), + ( + "the printed systemd removal leaves the manager holding the unit it removed", + "service.py", + "; rm {shlex.quote(target)}; {ctl} daemon-reload", + "; rm {shlex.quote(target)}", + "test_service_install_writes_a_systemd_user_unit_and_starts_nothing", + ), + # --- a captured amount said for a person --- + ( + "the watch line goes back to a bare byte count", + "daemon.py", + "+{size_text(result.appended)} ", + "+{result.appended}B ", + "test_the_watch_line_says_how_much_in_units_a_person_reads", + ), + ( + "capture's result line goes back to a bare byte count", + "__main__.py", + " added = daemon.size_text(cap.appended)", + ' added = f"{cap.appended}B"', + "test_cli_capture_says_how_much_in_units_and_where_in_bytes", + ), + ( + "a size just under the next unit rounds up to 1000.0 of this one", + "daemon.py", + ' if x < 999.95 or unit == "TB":', + ' if x < 1000 or unit == "TB":', + "test_a_byte_count_is_said_for_a_person_and_exactly", + ), ] @@ -5530,12 +5959,13 @@ def run(args: list[str]) -> int: figure depends on something the docstring used to get wrong. `addopts` carries `-m 'not corpus'`, but only one test is marked `corpus`; the ~328 conformance cases are gated on the `.conformance/` clones being present, not - on a marker. So this is the 955-test offline suite on a machine without the - clones and the 1280-test suite on a machine with them — **measured**, 81 s - for the former on the author's laptop. It said "about a second" from E3 - until somebody read it next to the "about a minute" nine lines down and - noticed the same call described twice, an order of magnitude apart. - [E3; review: docs, claims] + on a marker. So this is the offline suite on a machine without the clones + and the corpus-on suite on a machine with them — **measured**, 81 s for the + former on the author's laptop, when the two were 955 and 1280 tests; both + have grown since and it has not been re-timed. It said "about a second" + from E3 until somebody read it next to the "about a minute" nine lines + down and noticed the same call described twice, an order of magnitude + apart. [E3; review: docs, claims] `ANCHOR_TEST` as well, for the reason written where it is defined: it is the one test in the suite that is *supposed* to fail while a mutant is applied, @@ -5652,19 +6082,20 @@ def _select(test: str) -> list[str]: def main() -> int: """Run every mutant, or only those whose name contains an argument. - The filter exists because a full pass is 606 mutants × two suite runs — + The filter exists because a full pass is 662 mutants × two suite runs — long enough that adding one row and checking it used to mean either waiting - for the other 567 or trusting the new one untested. + for every other row or trusting the new one untested. Cost: **measured**, three rows timed end to end at 3.7 s, 17.1 s and 17.2 s, on a machine with the `.conformance/` clones — so against the - 1280-test suite, not the 955-test offline one (`run` explains why the two - differ). **Estimated** from those, a full pass is roughly one to two and a - half hours. The spread is not noise and does not average away — the first - of the two runs is `-x`, so a row costs however long the suite takes to - *reach* the test the mutant breaks, and that is a position in collection - order, not a property of the row. A row the suite misses entirely pays for - the whole suite instead. + corpus-on suite, not the offline one (`run` explains why the two differ), + when they were 1280 and 955 tests. **Estimated** from those, a full pass + was then roughly one to two and a half hours; the suite and the index have + both grown since, and neither figure has been re-timed. The spread is not + noise and does not average away — the first of the two runs is `-x`, so a + row costs however long the suite takes to *reach* the test the mutant + breaks, and that is a position in collection order, not a property of the + row. A row the suite misses entirely pays for the whole suite instead. Every figure in this paragraph had drifted: "41 s a row", "1,126-test", "71 verdicts in 49 minutes" and "about six hours" were all carried forward diff --git a/tests/test_daemon.py b/tests/test_daemon.py index bf599ca..e3e22b6 100644 --- a/tests/test_daemon.py +++ b/tests/test_daemon.py @@ -126,11 +126,13 @@ def test_a_malformed_config_means_watch_nothing_not_watch_everything(tmp_path): def test_a_watch_missing_its_roots_is_skipped_and_the_rest_survive(tmp_path): + home = str(tmp_path / "home") + root = tmp_path / "proj" # never created: a root that does not exist yet is kept _write( - os.path.join(str(tmp_path), "config.toml"), - '[[watch]]\nagent = "a"\n\n[[watch]]\nagent = "b"\nroots = ["/tmp/x"]\n', + os.path.join(home, "config.toml"), + f'[[watch]]\nagent = "a"\n\n[[watch]]\nagent = "b"\nroots = [{json.dumps(str(root))}]\n', ) - got = daemon.load_watches(str(tmp_path)) + got = daemon.load_watches(home) assert [w.agent for w in got] == ["b"] @@ -869,6 +871,39 @@ def test_run_reports_a_failing_pass_in_its_exit_code(tmp_path): os.chmod(bad, 0o600) +@pytest.mark.parametrize( + "n, said", + [ + (0, "0 B"), + (999, "999 B"), + (1000, "1.0 kB (1000 B)"), + (999_949, "999.9 kB (999949 B)"), + (999_950, "1.0 MB (999950 B)"), # not "1000.0 kB" + (391_995_999, "392.0 MB (391995999 B)"), + (10**15, "1000.0 TB (1000000000000000 B)"), # the last unit has no next one + ], +) +def test_a_byte_count_is_said_for_a_person_and_exactly(n, said): + assert daemon.size_text(n) == said + + +def test_the_watch_line_says_how_much_in_units_a_person_reads(tmp_path): + """`captured 470 +43025169B` made a person count digits to read the pass. + + The commit subject keeps `+NB`: it is the durable record, the history + already written says it that way, and nobody reads it to size a pass. + """ + home = str(tmp_path / "home") + root = tmp_path / "proj" + _write(str(root / "a.jsonl"), TURN * 40) # 2400 bytes + _config(home, [root]) + lines: list[str] = [] + assert daemon.run(home, once=True, interval=0, log=lines.append) == 0 + said = [line for line in lines if line.startswith("captured ")] + assert len(said) == 1 and said[0].startswith("captured 1 +2.4 kB (2400 B) commit="), lines + assert _subject(home).endswith(" +2400B"), _subject(home) + + # --- the seam the hook writes to ------------------------------------------- @@ -1963,7 +1998,7 @@ def test_a_fifo_with_a_writer_is_not_a_record_even_though_it_reads(tmp_path): def test_an_adoption_only_pass_names_what_it_reclaimed(tmp_path): - """`result.captured` is empty and `+0B` is the headline, so nothing else says it. + """`result.captured` is empty and the commit's `+0B` is the headline, so nothing else says it. Adoption attests to bytes found in the generation directory rather than copied out of the source. The manifest records that permanently; this is so diff --git a/tests/test_docs.py b/tests/test_docs.py index c2e9c26..f7209ce 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -16,6 +16,8 @@ import re import subprocess import sys +import tomllib +from itertools import pairwise import pytest from test_claude_code import CC_FIXTURES, _corpus @@ -183,6 +185,35 @@ def test_the_readme_arithmetic_paragraph_is_the_arithmetic(): assert f"replace three of the {offline}" in text +def test_the_reproduce_guide_states_the_readme_counts(): + """docs/REPRODUCE.md repeats the board's four numbers, and nothing held them. + + So they drifted: main said 1027 tests beside the README's 1,037, and an + install-feedback commit left 1043 beside 1,063. The README is pinned to the + suite by the tests above, so holding the guide to the README holds it to + the suite, with no second collection, and on a machine without the corpora + too. [review] + """ + board = _read("README.md") + guide = " ".join(_read("docs/REPRODUCE.md").split()) + + def number(pattern: str) -> int: + found = re.search(pattern, board) + assert found, pattern + return int(found.group(1).replace(",", "")) + + tests = number(r"\*\*([\d,]+) tests\*\*") + cases = number(r"(\d[\d,]*) conformance cases") + online = number(r"collects (\d+), not \d+\.") + controls = number(r"\*\*([\d,]+)\*\* negative controls") + assert ( + f"**Expected:** `{tests}/{tests + 1} tests collected (1 deselected)`, then " + f"`{online}/{online + 1} tests collected (1 deselected)`, then `{cases}`." + ) in guide, "docs/REPRODUCE.md §5b does not expect the README's counts" + assert f"why {tests:,} + {cases:,} is {online:,} and not {tests + cases:,}" in guide + assert f"**Expected:** `{controls}`." in guide, "§5c does not expect the README's count" + + def test_the_design_document_does_not_promise_a_file_that_is_not_written(): """`manifest.json` was the name in every version of the verification recipe. @@ -214,6 +245,54 @@ def test_the_documented_manifest_fields_are_the_fields(): assert field in design, f"DESIGN.md does not mention the manifest field {field!r}" +def test_the_disk_section_calls_rebuildable_only_what_is_gitignored(): + """ARCHITECTURE.md said the store "holds each byte the agent wrote exactly + once", and DESIGN.md "zero duplication". That is `raw/`. `.git` keeps a + second, compressed copy and `index/` the text a third time, so one store + came to 1.75x what it had captured, and no document said why. + + The ratios are not pinned: they move with the filesystem's block size, the + git version, when `gc --auto` last packed, and the corpus. What is pinned is + the advice. Every directory a sentence of the section tells you to delete + must be one the store's own `.gitignore` ignores, and `raw/` and `sessions/` + must stay tracked, because deleting either loses the only copy. The anchors + pointing at the section are checked against its heading, so renaming it + breaks a test and not just a link. + Headings are read with fenced blocks cut out: a `# comment` in a shell or + TOML recipe is not a heading, and counting it let a link to no heading pass. + """ + from gitmemory import gitrepo + + watching = re.sub(r"^```.*?^```", "", _read("docs/watching.md"), flags=re.S | re.M) + section = re.search(r"^## How much disk it takes\n(.*?)(?=^## |\Z)", watching, re.S | re.M) + assert section, "docs/watching.md has no 'How much disk it takes' section" + body = section.group(1) + assert "`index/`" in body and "`gitmemory index`" in body, "it no longer says what rebuilds it" + + ignored = gitrepo.GITIGNORE.splitlines() + told = [ + d + for sentence in re.split(r"(?<=\.)\s+", " ".join(body.split())) + if re.search(r"\b(?:delete|remove)\b", sentence, re.I) + for d in re.findall(r"`([\w.-]+/)`", sentence) + ] + assert told, "the section no longer names the directory to delete" + for d in told: + assert f"/{d}" in ignored, f"the section says to delete `{d}`, which git does not ignore" + for durable in ("/raw/", "/sessions/"): + assert durable not in ignored, f"{durable} is ignored, so git no longer holds it" + + slugs = { + re.sub(r"[^\w\- ]", "", h.strip().lower()).replace(" ", "-") + for h in re.findall(r"^#+ (.+)$", watching, re.M) + } + for rel in ("README.md", "CHANGELOG.md", "docs/ARCHITECTURE.md", "docs/DESIGN.md"): + anchors = re.findall(r"watching\.md#([\w-]+)", _read(rel)) + assert "how-much-disk-it-takes" in anchors, f"{rel} does not point at the section" + for anchor in anchors: + assert anchor in slugs, f"{rel} links watching.md#{anchor}, which no heading makes" + + def test_the_documented_shim_length_is_the_shim_length(): """ "~20 lines POSIX sh" outlived the sentence by a factor of five. @@ -277,6 +356,122 @@ def test_the_design_document_cites_nothing_in_tmp(): assert "/tmp/" not in line, f"{rel}:{n} cites a scratch path: {line.strip()}" +_INSTALLERS = r"uv tool install|pipx install" +# The tool, then its words up to the end of the code span, a `)`, a `#`, the end +# of the line, or the next tool, so two commands on one line are two commands. +_INSTALL = re.compile(rf"\b({_INSTALLERS})\b([^`)#\n]*?)(?=\b(?:{_INSTALLERS})\b|[`)#]|$)") + + +def _installs(text: str): + """(line number, tool, its words, as written) for each install command in + `text`. A line that ends in a backslash goes on into the next, as in sh. A + tool whose span holds nothing else takes the rest of the line, up to the + next tool, when that starts with a source or a flag: `` `uv tool install` + git+… `` is one command split across spans, not prose.""" + lines, n = text.splitlines(), 0 + while n < len(lines): + start, line = n + 1, lines[n] + while line.endswith("\\") and n + 1 < len(lines): + n += 1 + line = line[:-1] + " " + lines[n] + n += 1 + for m in _INSTALL.finditer(line): + words = m.group(2).split() + if not words: + rest = re.split(_INSTALLERS, line[m.end() :], maxsplit=1)[0] + rest = rest.replace("`", " ").split() + if rest and rest[0].strip("\"'").startswith(("git+", "gitmemory", "-")): + words = rest + yield start, m.group(1), words, " ".join([m.group(1), *words]) + + +def _names_python(tool: str, words: list[str], want: str) -> bool: + """`--python ` or `--python=`, or uv's own `-p `.""" + flags = ("--python", "-p") if tool == "uv tool install" else ("--python",) + words = [w.strip("\"'") for w in words] + return f"--python={want}" in words or any( + a in flags and b == want for a, b in pairwise(words) + ) + + +def test_every_documented_install_names_the_python_floor(): + """The README's install line did not install on Ubuntu 24.04; the same line + with `--python 3.13` did. + + pyproject says `>=3.13`, and the README said so one line above a command + that did not. Left to choose, uv can settle on the machine's own Python + (3.12 there) and refuse the install, which is what it does with Python + downloads off (measured); named, it builds the tool on 3.13, fetching one + if it may, or says to install one. pipx installs into the Python it runs + under, so its line names an interpreter of that version. The floor is read + from pyproject, so the commands cannot drift from it a second time. + + A command is the tool and what follows it, to the end of its code span, a + `)` or a `#`, the end of the line (a trailing backslash carries it on to the + next), or the next command. It names the floor as `--python X.Y`, + `--python=X.Y` or uv's `-p X.Y`; pipx's value is `pythonX.Y`. Prose that + names the tool with nothing after it ("if you installed with + `uv tool install`,") has no source, so it is not one. Prose with words after + the tool and no code span around it reads as a command and fails, so put the + tool in a code span or write the whole command; a span holding the tool + alone still reads on when the next word is a source or a flag + (`` `uv tool install` git+… ``). `tests/` is not scanned, + because its strings are fixtures and this one names the tools; nor are the + records, which quote what was run at the time. [install feedback 1] + """ + with open(os.path.join(ROOT, "pyproject.toml"), "rb") as fh: + spec = tomllib.load(fh)["project"]["requires-python"] + floor = re.fullmatch(r">=(\d+\.\d+)", spec) + assert floor, f"requires-python is {spec!r}; this test knows only a bare lower bound" + v = floor.group(1) + want = {"uv tool install": v, "pipx install": f"python{v}"} + + # The scanner first, on spellings a later line may use. [install review 2, 4] + probe = ( + f"(uv tool install --python {v} git+x, or pipx install git+x)\n" + f"`uv tool install -p {v} git+x` and `uv tool install --python={v} git+x`\n" + f"uv tool install \\\n --python {v} git+x\n" + f"`pipx install --python python{v} git+x` `pipx install -p python{v} git+x`\n" + "if you installed with `uv tool install`,\n" + f"`uv tool install` git+x, then uv tool install `--python {v} git+x`\n" + ) + found = [(n, t, _names_python(t, w, want[t])) for n, t, w, _ in _installs(probe) if w] + assert found == [ + (1, "uv tool install", True), + (1, "pipx install", False), + (2, "uv tool install", True), + (2, "uv tool install", True), + (3, "uv tool install", True), + (5, "pipx install", True), + (5, "pipx install", False), + (7, "uv tool install", False), + (7, "uv tool install", True), + ], found + + records = ("tests/", "docs/reviews/", "docs/tasks/", "docs/DESIGN-v0.md") + tracked = subprocess.run( + ["git", "ls-files", "-z"], cwd=ROOT, capture_output=True, text=True, check=True + ).stdout.split("\0") + seen, wrong = set(), [] + for rel in filter(None, tracked): + if rel.startswith(records): + continue + try: + text = _read(rel) + except (UnicodeDecodeError, FileNotFoundError, IsADirectoryError): + continue # a binary or a submodule; no command in it + for n, tool, words, said in _installs(text): + if not words: + continue + seen.add(rel) + if not _names_python(tool, words, want[tool]): + wrong.append(f"{rel}:{n}: `{said}` lacks --python {want[tool]}") + # The floor, because a scan that matches nothing passes loudest. + named = {"README.md", "hook/README.md", "skills/recall/SKILL.md", ".claude-plugin/plugin.json"} + assert named <= seen, named - seen + assert not wrong, f"requires-python is {spec}:\n" + "\n".join(wrong) + + def test_no_tracked_file_contains_an_invisible_character(): """A character you cannot see is a character nobody can review. @@ -449,11 +644,13 @@ def test_the_plugin_recall_skill_runs_only_commands_the_cli_has(tmp_path, capsys waiting, and it reads as gitmemory being broken. So every `gitmemory ...` in the skill's code is asked for its own `--help`, which argparse answers with exit 0 only for a subcommand that exists, and every flag it names is - looked for in that help: `--home` before the subcommand in the top-level - help, the rest in the subcommand's. The commands it forbids are checked the - same way, because a prohibition on a command that no longer exists guards - nothing. The messages it tells the agent to branch on are the ones the CLI - prints, got by running it on a path that is no store and on an empty one. + looked for in that help: `--home` before the subcommand, and `--version`, + which comes with none, in the top-level help, the rest in the subcommand's. + The commands it forbids are checked the same way, because a prohibition on + a command that no longer exists guards nothing. The messages it tells the + agent to branch on are the ones the CLI prints, got by running it on a path + that is no store, on an empty one, and on one whose spool no watcher has + drained. `skills/` holds this one skill and neither manifest adds a skills path; its name is the one the three documents that mention it tell you to type. """ @@ -486,10 +683,12 @@ def usage(*argv: str) -> str: for m in re.finditer(r"(? tuple[int, str, str]: assert code == 0 and out.startswith("0 generation(s)") and "`0 generation(s)`" in flat code, _, err = run(empty, "recall", "x") assert code == 0 and err == "no matches\n" and "`no matches` on stderr, exit 0" in flat + os.makedirs(os.path.join(empty, "spool")) + with open(os.path.join(empty, "spool", "1-SessionEnd.json"), "w") as fh: + fh.write("{}") + os.utime(fh.name, (0, 0)) + code, _, err = run(empty, "recall", "x") + said = err.splitlines()[0].replace("", "") + said = re.sub(r"for \d+ \w+:", "for :", re.sub(r"^\d+", "", said)) + assert code == 0 and f"``{said}``" in flat, said + + +def test_the_recall_skill_names_the_release_it_was_written_for(capsys): + """The plugin and the CLI are installed separately, and nothing makes them + one release. + + The test above runs the skill against the CLI in this checkout, which says + nothing about an installed copy of each. A skew has shipped already: at + v0.1.0 `recall` printed no date and ranked by relevance, the skill reads + the date to tell which of two lines is current, both said 0.1.0, and + nothing errored. So the skill names its release, pinned here to + `pyproject.toml` and `__version__` (test_hook.py pins `plugin.json`), and + asks `gitmemory --version` once per session. A CLI from before the flag + answers with the first error below (measured on c352a40 under 3.12, 3.13 + and 3.14); one that lacks a subcommand or a flag answers with one of the + other two. The skill branches on those words, so each is got from the CLI + and must be in the skill as the CLI prints it. A topic left unquoted is + rejected in the same words as a missing flag, naming a plain word, so the + skill tells the two apart by the leading `-`. And the store captures the + `--version` call of every session that uses the skill, which outranks real + hits on "version" unless the skill lists it among its echoes. + """ + from gitmemory import __main__ as cli + from gitmemory import __version__ + + with open(os.path.join(ROOT, "pyproject.toml"), "rb") as fh: + release = tomllib.load(fh)["project"]["version"] + assert __version__ == release + skill = " ".join(_read("skills/recall/SKILL.md").split()) + + with pytest.raises(SystemExit) as done: + cli.main(["--version"]) + assert done.value.code == 0 and capsys.readouterr().out == f"gitmemory {release}\n" + assert f"written for gitmemory {release}." in skill, release + + for argv, said in ( + ([], "the following arguments are required: cmd"), + (["no-such-subcommand"], "invalid choice"), + (["recall", "--no-such-flag", "x"], "unrecognized arguments"), + ): + with pytest.raises(SystemExit) as done: + cli.main(argv) + err = capsys.readouterr().err + assert done.value.code == 2 and err.startswith("usage: gitmemory "), err + assert said in err.splitlines()[-1] and f"`{said}`" in skill, err + assert "`usage: gitmemory`" in skill + + with pytest.raises(SystemExit) as done: + cli.main(["recall", "retry", "loop"]) + last = capsys.readouterr().err.splitlines()[-1] + assert done.value.code == 2 and last.endswith("unrecognized arguments: loop"), last + assert "a word starting with `-`" in skill and "went in unquoted" in skill + + echoes = skill.split("## Echoes ", 1)[1].split(" ## ", 1)[0] + assert "`gitmemory --version`" in echoes, echoes + + +def test_the_main_sessions_only_pattern_does_what_watching_md_says(tmp_path): + """`docs/watching.md` offers a pattern that leaves Claude Code's subagents out, + and says a hook cannot bring one back in. Both are claims about the watcher, + so the documented block is the config here, verbatim, over Claude Code's + layout: a Task subagent, a workflow agent and the workflow's journal beside + the main session. The default pattern finds all four, so the three this one + misses are files the watcher would otherwise capture. The page also says the + recipe must replace the default table, not sit beside it.""" + import dataclasses + + from gitmemory import daemon + + doc = _read("docs/watching.md") + section = re.search(r"^### Main sessions only.*?(?=^#)", doc, re.M | re.S) + assert section, "docs/watching.md has no main-sessions-only recipe" + block = re.search(r"^```toml\n(.*?)^```", section.group(0), re.M | re.S) + assert block, section.group(0) + + turn = '{"type":"user","message":{"role":"user","content":"hello"}}\n' + session = os.path.join(os.path.expanduser("~/.claude/projects"), "-proj", "s1") + parent = session + ".jsonl" + workflow = os.path.join(session, "subagents", "workflows", "wf_1") + child = os.path.join(session, "subagents", "agent-a1.jsonl") + others = [child, *(os.path.join(workflow, f) for f in ("agent-b2.jsonl", "journal.jsonl"))] + for path in (parent, *others): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + fh.write(turn) + home = str(tmp_path / "store") + os.makedirs(os.path.join(home, daemon.SPOOL)) + with open(os.path.join(home, "config.toml"), "w", encoding="utf-8") as fh: + fh.write(block.group(1)) + + watches = daemon.load_watches(home) + found = [p for _, p in daemon.discover(watches)] + assert found == [os.path.realpath(parent)] + every = [dataclasses.replace(w, pattern=daemon.PATTERN) for w in watches] + assert len(daemon.discover(every)) == 4 + first = daemon.tick(home, watches, commit=False) + assert [key.split("/")[1].startswith("s1-") for key in first.captured] == [True] + + # A hook firing as the subagent ends rings for a file the pattern left out. + with open(child, "a", encoding="utf-8") as fh: + fh.write(turn) + with open(os.path.join(home, daemon.SPOOL, "1-2-SessionEnd.json"), "w") as fh: + json.dump({"transcript_path": child}, fh) + second = daemon.tick(home, watches, interval=1e9, commit=False) + assert second.captured == [] and second.spool_consumed == 1 + + # Added as a second table beside a default one on the same root, it leaves + # nothing out, which is why the page says to edit the table you have. + default = re.sub(r"^pattern = .*\n", "", block.group(1), flags=re.M) + assert default != block.group(1) + with open(os.path.join(home, "config.toml"), "w", encoding="utf-8") as fh: + fh.write(default + "\n" + block.group(1)) + assert len(daemon.discover(daemon.load_watches(home))) == 4 + + +def test_the_documented_watch_table_is_the_one_init_writes(tmp_path, capsys): + """The README and watching.md show the table and say `gitmemory init` writes + it. Two copies of one config in prose and one in `daemon.ROOTS` is how a doc + ends up telling people to hand-write a table the command no longer writes, + so the block under each section heading is looked for in what init wrote. + By section, because watching.md may hold other `toml` recipes init does + not write; split on `##` because a TOML comment is a `#` at column 0.""" + from gitmemory.__main__ import main + + home = str(tmp_path / "store") + assert main(["--home", home, "init", "claude-code"]) == 0 + capsys.readouterr() + with open(os.path.join(home, "config.toml"), encoding="utf-8") as fh: + written = fh.read() + for rel, heading in [("README.md", "## Quickstart"), ("docs/watching.md", "## Configure")]: + section = _read(rel).split(f"\n{heading}", 1)[1].split("\n##", 1)[0] + block = re.search(r"^```toml\n(.*?)^```", section, re.M | re.S) + assert block, f"{rel} {heading} no longer shows the table" + assert block.group(1) in written, rel diff --git a/tests/test_index.py b/tests/test_index.py index edf1b7f..8cc30a4 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -22,7 +22,7 @@ import pytest -from gitmemory import gitrepo, index, records, store +from gitmemory import gitrepo, index, records, redact, store from gitmemory.__main__ import main @@ -1550,3 +1550,402 @@ def roles(*argv: str) -> list[str]: assert roles() == ["user/text", "user/tool_result", "assistant/text"] assert roles("--policy", "latest") == ["user/tool_result", "assistant/text", "user/text"] + + +def sidechain(uid: str, text: str, ts: str) -> dict: + """A subagent's turn. Claude Code marks it `isSidechain` wherever it is + written, and its first one is user-role text the parent agent wrote: the + dispatch prompt.""" + return dated(uid, text, ts) | {"isSidechain": True, "agentId": "a1"} + + +def with_a_subagent(home: str, tmp_path: Path) -> None: + """The user's rule in a main session, then the opposite a day later in a + dispatch prompt to one of its subagents. Claude Code's own layout: + `/.jsonl` and `//subagents/agent-.jsonl`.""" + parent = tmp_path / "proj" / "s1.jsonl" + child = tmp_path / "proj" / "s1" / "subagents" / "agent-a1.jsonl" + child.parent.mkdir(parents=True) + write( + str(parent), + [ + dated("u1", "Don't add a retry loop around the upload.", "2026-09-28T10:00:00Z"), + assistant("a1", [{"type": "text", "text": "Understood: the upload has no retry loop."}]) + | {"timestamp": "2026-09-28T11:00:00Z"}, + ], + ) + prompt = sidechain("u2", "Add a retry loop around the upload.", "2026-09-29T09:00:00Z") + write(str(child), [prompt]) + store.capture(str(parent), "claude-code", "s1", home=home) + store.capture(str(child), "claude-code", "agent-a1", home=home) + index.build(home) + + +def test_a_subagent_prompt_is_not_the_users_word(home, tmp_path): + """The forgetful agent again, arriving through a dispatch prompt rather than a + proposal. The prompt is user text, newer than the rule and matching it, so + the default led with it as the user's own words. The turn is marked, so it + is ranked with the agent's turns, where the parent's instructions belong.""" + with_a_subagent(home, tmp_path) + db = index.open_db(index.db_path(home)) + try: + hits = index.search(db, "retry loop upload") + finally: + db.close() + ordered = index.latest_user(hits) + assert ordered[0].text == "Don't add a retry loop around the upload." + assert [h.is_sidechain for h in ordered] == [False, True, False] + + +def test_recall_marks_a_subagent_turn(home, tmp_path, capsys): + """A prefix on the role, not a new column: `split()[3]` is role/kind to + whatever parses these lines, this file included, and a main session's + role/kind carries no prefix.""" + with_a_subagent(home, tmp_path) + capsys.readouterr() + assert main(["--home", home, "recall", "retry loop upload"]) == 0 + lines = capsys.readouterr().out.splitlines() + assert [line.split()[3] for line in lines] == [ + "user/text", + "subagent:user/text", + "assistant/text", + ] + assert lines[1].split()[2].startswith("claude-code/agent-a1/") + + +def test_a_sidechain_line_inside_a_main_file_is_marked(home, src): + """Older Claude Code wrote a subagent's turns into the parent's own file, each + line marked. The mark is read per turn, so those are caught too, which no + rule on the file's path can do.""" + db = built( + home, + src, + [ + dated("u1", "keep the upload single-shot", "2026-09-28T10:00:00Z"), + sidechain("u2", "review the upload code", "2026-09-28T10:05:00Z"), + ], + ) + try: + hits = index.search(db, "upload") + finally: + db.close() + assert {h.text: h.is_sidechain for h in hits} == { + "keep the upload single-shot": False, + "review the upload code": True, + } + + +# --------------------------------------------------------------------------- # +# the hook rang and no watcher answered +# --------------------------------------------------------------------------- # + + +def _ring(home: str, age: float, name: str = "4242-SessionEnd.json") -> None: + """A spool record as the shim leaves it, arrived `age` seconds ago.""" + spool = os.path.join(home, "spool") + os.makedirs(spool, exist_ok=True) + path = os.path.join(spool, name) + with open(path, "w") as fh: + fh.write('{"transcript_path": "/nowhere/s.jsonl"}') + then = time.time() - age + os.utime(path, (then, then)) + + +def _indexed(home: str, src: str) -> None: + write(src, [user("u1", "the peculiar marmoset")]) + assert main(["--home", home, "capture", src]) == 0 + assert main(["--home", home, "index"]) == 0 + + +def test_recall_says_the_hook_rang_and_no_watcher_answered(home, src, capsys): + """A stopped watcher was invisible from `recall`: it answered from the store + as it was when the watcher stopped, and said nothing. Time since the last + capture cannot say so, because a running watcher on an idle machine captures + nothing either. A record the watcher would have drained within one poll can. + Before the answer, like the truncation notice. [install feedback]""" + _indexed(home, src) + _ring(home, 3 * 3600 + 5) + _ring(home, 60, "4243-PreCompact.json") + capsys.readouterr() + assert main(["--home", home, "recall", "marmoset"]) == 0 + out, err = capsys.readouterr() + assert "marmoset" in out and "uncollected" not in out, out + assert err == ( + "2 hook record(s) uncollected, the oldest for 3 hours: is `gitmemory watch` " + f"running for {home}? These results may be missing everything said since.\n" + ), err + assert main(["--home", home, "recall", "aardvark"]) == 0 + err = capsys.readouterr().err + assert err.index("uncollected") < err.index("no matches"), err + # Each unit starts at 2, so the number is never 1 and its plural always fits. + for age, said in ( + (90 * 60, "90 minutes"), + (2 * 3600 - 30, "119 minutes"), + (2 * 3600 + 30, "2 hours"), + (2 * 86400 - 30, "47 hours"), + (2 * 86400 + 30, "2 days"), + (3 * 86400, "3 days"), + ): + _ring(home, age) + assert main(["--home", home, "recall", "marmoset"]) == 0 + assert f"the oldest for {said}:" in capsys.readouterr().err + + +def test_recall_says_nothing_about_a_spool_a_watcher_could_still_be_draining( + home, src, capsys +): + """Silent below the threshold, on a hook still writing, and under `--db`, + whose index records no store to look in.""" + _indexed(home, src) + _ring(home, 30 * 60) + _ring(home, 3 * 3600, ".tmp-4242-0") + capsys.readouterr() + assert main(["--home", home, "recall", "marmoset"]) == 0 + assert capsys.readouterr().err == "", "a fresh record or a hook's temp was reported" + _ring(home, 3 * 3600) + assert main(["--home", home, "recall", "marmoset", "--db", index.db_path(home)]) == 0 + assert capsys.readouterr().err == "", "--db names an index, not a store" + + +def test_one_watcher_pass_is_what_clears_it(home, src, capsys): + """The notice is true exactly as long as `drain_spool` has not run: every + pass consumes every complete record, covered or not. A directory in the + spool is no record and no pass can unlink it, so counting it warned for + ever with a watcher running. [review]""" + root = json.dumps(os.path.dirname(src)) + with open(os.path.join(home, "config.toml"), "w") as fh: + fh.write(f'[[watch]]\nagent = "claude-code"\nroots = [{root}]\n') + _indexed(home, src) + _ring(home, 3 * 3600) + stuck = os.path.join(home, "spool", "4244-Stop.json") + os.mkdir(stuck) + os.utime(stuck, (time.time() - 3 * 3600,) * 2) + capsys.readouterr() + assert main(["--home", home, "recall", "marmoset"]) == 0 + err = capsys.readouterr().err + assert err.startswith("1 hook record(s) uncollected, the oldest for 3 hours:"), err + assert main(["--home", home, "watch", "--once"]) == 0 + assert os.path.isdir(stuck), "the pass removed the directory; this test no longer bites" + capsys.readouterr() + assert main(["--home", home, "recall", "marmoset"]) == 0 + assert "uncollected" not in capsys.readouterr().err + + +# --------------------------------------------------------------------------- # +# recall --redact: the owner asks for the gate's shapes to be masked +# --------------------------------------------------------------------------- # + +# Synthetic, right shape, and assembled here so that no credential-shaped +# literal sits in the tree for a push scanner to find. +AKIA = "AKIA" + "ZQ" * 8 +PEM_BODY = "b" * 64 +PEM = ( + "-----BEGIN RSA " + "PRIVATE KEY-----\n" + PEM_BODY + "\n" + PEM_BODY + "\n" + "-----END RSA " + "PRIVATE KEY-----" +) + + +def _recall_out(home: str, capsys, *argv: str) -> str: + assert main(["--home", home, "recall", *argv]) == 0 + return capsys.readouterr().out + + +def test_recall_redact_masks_a_key_the_160_character_cut_would_have_halved(home, src, capsys): + """Masked before the cut. The key starts at character 150, so the cut keeps + ten of its twenty characters, and ten match no detector: masking the + printed head instead would scan a fragment clean and print it.""" + text = "zebra " * 25 + AKIA + " tail" + assert text.index(AKIA) == 150 + built(home, src, [user("u1", text)]).close() + capsys.readouterr() + assert AKIA[:10] in _recall_out(home, capsys, "zebra"), "the default stays verbatim" + out = _recall_out(home, capsys, "zebra", "--redact") + assert "AKIA…[" in out and AKIA[4:10] not in out, out + + +def test_recall_redact_masks_a_private_key_body_and_not_only_its_header(home, src, capsys): + """`private_key_block` matches the BEGIN line, and the key is the base64 + under it. Masking the match alone printed `----…[31 bytes]` followed by + every byte of the key.""" + built(home, src, [user("u1", "pemcase " + PEM + " after the key")]).close() + capsys.readouterr() + assert PEM_BODY[:8] in _recall_out(home, capsys, "pemcase"), "premise: the body is there" + out = _recall_out(home, capsys, "pemcase", "--redact") + assert PEM_BODY[:8] not in out and "PRIVATE KEY" not in out, out + assert "pemcase ----…[" in out and "after the key" in out, out + + +def test_recall_redact_masks_a_header_that_only_the_collapse_assembles(home, src, capsys): + """`-----BEGIN RSA\\nPRIVATE KEY-----` scans clean as written and is a + header once the line is flattened, and the flattened line is what recall + prints. So the mask scans a flattened copy too, as `graph._label` found it + had to, and as `show`'s does. [E7b L3-F1]""" + split = PEM.replace("RSA PRIVATE", "RSA\nPRIVATE", 1) + assert redact.mask(split) == split, "premise: nothing matches before the collapse" + built(home, src, [user("u1", "pemcase " + split)]).close() + capsys.readouterr() + out = _recall_out(home, capsys, "pemcase", "--redact") + assert PEM_BODY[:8] not in out and "PRIVATE KEY" not in out, out + + +def test_recall_redact_masks_a_value_with_a_unicode_space_in_it(home, src, capsys): + """`str.split()` breaks on a no-break space, and the gate's bytes patterns + read one as part of the token. Collapsing first cut the value below the + twelve-character floor, so `recall --redact` printed what `show --redact` + and the push gate both flag. The collapse that joins a split header is the + mask's own, on ASCII whitespace, as `show`'s is.""" + secret = "pass" + "word=abc defghijklmnop" + flat = " ".join(secret.split()) + assert redact.mask(flat) == flat and redact.mask(secret) != secret, "premise" + built(home, src, [user("u1", f"gnu note {secret} end")]).close() + capsys.readouterr() + out = _recall_out(home, capsys, "gnu", "--redact") + assert "defghijklmnop" not in out and "gnu note pass…[" in out, out + + +def test_recall_redact_masks_a_session_id_that_is_itself_a_credential(home, src, capsys): + """The address is printed too, and `_SAFE_RE` accepts a token as a session + id. Lowercase, because the store case-folds ids. + + The masked address stays one column. The mask's marker has a space in it, + and with it `split()[3]` read `bytes]/g00@0` where role/kind belongs.""" + ghp = "ghp_" + "a" * 36 + write(src, [user("u1", "walrus note")]) + store.capture(src, "claude-code", ghp, home=home) + index.build(home) + capsys.readouterr() + assert ghp in _recall_out(home, capsys, "walrus") + out = _recall_out(home, capsys, "walrus", "--redact") + assert ghp not in out and "/ghp_…[40bytes]/g00@0 user/text" in out, out + assert out.split()[3] == "user/text", out + + +def test_recall_redact_masks_a_key_the_raw_bytes_only_spell_escaped(home, src, capsys): + """The push gate reads raw bytes and cannot see `\\u0073k-ant-…`. The index + holds the parsed text, where the escape is already the key, so recall's + mask sees what the push gate does not.""" + key = "sk-ant-api03-" + "A1b2_-" * 15 + with open(src, "ab") as fh: + fh.write( + b'{"type":"user","uuid":"u1","sessionId":"s1","message":{"role":"user",' + b'"content":"escaped \\u0073' + key[1:].encode() + b' unicorn"}}\n' + ) + assert redact.scan_bytes(Path(src).read_bytes()) == [], "premise: the raw bytes scan clean" + store.capture(src, "claude-code", "sess", home=home) + index.build(home) + capsys.readouterr() + out = _recall_out(home, capsys, "unicorn", "--redact") + assert key[4:20] not in out and "sk-a…[103 bytes]" in out, out + + +def test_gitmemory_redact_makes_masking_standing_and_0_turns_it_off( + home, src, capsys, monkeypatch +): + """Unset, empty and `0` are off, and anything else is on, so a mistyped + value fails toward masking. Read when the parser is built, as + `GITMEMORY_POLICY` is.""" + built(home, src, [user("u1", "aws " + AKIA + " done")]).close() + capsys.readouterr() + for value, masked in (("1", True), ("yes", True), ("0", False), ("", False)): + monkeypatch.setenv("GITMEMORY_REDACT", value) + assert (AKIA not in _recall_out(home, capsys, "aws")) is masked, value + monkeypatch.delenv("GITMEMORY_REDACT") + assert AKIA in _recall_out(home, capsys, "aws") + assert AKIA not in _recall_out(home, capsys, "aws", "--redact") + + +def test_mask_returns_clean_text_unchanged_and_keeps_the_words_around_a_key(): + clean = "the lock is held across the await, see a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0" + assert redact.mask(clean) is clean + assert redact.mask(f"use {AKIA} for staging") == "use AKIA…[20 bytes] for staging" + # Each header runs to its own END line, not the last one in the text. + two = redact.mask(f"one {PEM} between {PEM} two") + assert two == f"one ----…[{len(PEM)} bytes] between ----…[{len(PEM)} bytes] two", two + # Cut before its END line, the rest of the text is the key, so the rest goes. + assert redact.mask("x " + PEM[:80]) == "x ----…[80 bytes]" + # A second key in the body is inside the header's span, and must not end it. + inner = PEM.replace(PEM_BODY + "\n", f"{PEM_BODY}\n{AKIA}\n", 1) + assert redact.mask(f"x {inner} y") == f"x ----…[{len(inner)} bytes] y" + + +def test_recall_redact_masks_a_key_behind_a_character_the_index_spells_out(home, src, capsys): + """The push gate reads raw bytes, and a BOM, a zero-width space or a DEL in + front of a key is not a word character there, so the key's `\\b` holds and + the gate finds it. The index holds `records.safe_text`'s spelling of each, + `\\ufeff`, `\\u200b`, `\\x7f`, and every spelling ends in a word character: + the boundary was gone and `recall --redact` printed all three whole. A `cat` + of an `.env` saved with a BOM is the ordinary way in. + + The password has a control inside its value, which a scan of the blanked + copy alone cuts to three characters and misses, so both scans stay.""" + ghp = "ghp_" + "a" * 36 + secret = "pass" + "word=abc" + "\x01" + "defghijklmn" + text = f"marmot \ufeff{AKIA} x\u200b{ghp} y\x7f{AKIA} z {secret} end" + line = {"type": "user", "uuid": "u1", "sessionId": "s1"} + line["message"] = {"role": "user", "content": text} + # Literal UTF-8, as Node's JSON.stringify writes it, which is what the gate reads. + with open(src, "wb") as fh: + fh.write(json.dumps(line, ensure_ascii=False).encode() + b"\n") + gate = [f.detector for f in redact.scan_bytes(Path(src).read_bytes())] + assert gate.count("aws_access_key_id") == 2 and "github_token" in gate, gate + assert "assigned_secret" in gate, gate + store.capture(src, "claude-code", "sess", home=home) + index.build(home) + capsys.readouterr() + assert AKIA in _recall_out(home, capsys, "marmot"), "premise: the default prints them" + out = _recall_out(home, capsys, "marmot", "--redact") + assert AKIA[4:10] not in out and ghp[4:20] not in out and "defghijklmn" not in out, out + assert "\\ufeffAKIA…[20 bytes] x\\u200bghp_…[40 bytes] y\\x7fAKIA…[20 bytes]" in out, out + + +def test_mask_stays_linear_on_a_page_of_key_headers_and_an_end_line_that_never_closes( + monkeypatch, +): + """Two ways the PEM widening went quadratic on text an agent fetched. + + Each header searched for its own END line, so headers with none cost + headers x length: 16,000 of them (437 KB) took 2.4 s where `scan_bytes` + took 0.06 s, and a 2 MB tool_result of them kept `recall --redact` for + 41 s. And the END pattern's unbounded runs retried every `KEY` in a + `-----END KEY KEY …` line to its end: 125 KB took 4.1 s. + + The page of headers is counted, not timed: one header searches, and every + later one sits inside the span that search found. The END line is one + `search` whose cost is inside `re` and cannot be counted, so it is timed, + as `test_a_separator_free_megabyte_does_not_hang_the_build` is: about 10 s + at this size when quadratic, and 1.5 s is several times the linear cost. + """ + header = "-----BEGIN " + "PRIVATE KEY-----\n" + real, searched = redact._PEM_END, [] + + class Counted: + def search(self, data, pos): + searched.append(pos) + return real.search(data, pos) + + with monkeypatch.context() as m: + m.setattr(redact, "_PEM_END", Counted()) + text = header * 2_000 + assert redact.mask(text) == f"----…[{len(text)} bytes]" + assert searched == [len(header) - 1], f"{len(searched)} headers searched for an END line" + + text = header + "-----END " + "KEY " * 50_000 + started = time.monotonic() + out = redact.mask(text) + elapsed = time.monotonic() - started + assert out == f"----…[{len(text)} bytes]", out[:80] + assert elapsed < 1.5, f"the END line pattern went quadratic again: {elapsed:.2f}s" + + +def test_recall_redact_keeps_a_subagent_turn_marked(home, tmp_path, capsys): + """The mask and the subagent mark compose. `--redact` changes what the + address and the text say, never which field role/kind is in or whose turn + a line is.""" + with_a_subagent(home, tmp_path) + capsys.readouterr() + out = _recall_out(home, capsys, "retry loop upload", "--redact") + assert [line.split()[3] for line in out.splitlines()] == [ + "user/text", + "subagent:user/text", + "assistant/text", + ], out diff --git a/tests/test_init.py b/tests/test_init.py new file mode 100644 index 0000000..05a6678 --- /dev/null +++ b/tests/test_init.py @@ -0,0 +1,248 @@ +# SPDX-FileCopyrightText: 2026 The gitmemory authors +# SPDX-License-Identifier: Apache-2.0 +"""`gitmemory init `: the `[[watch]]` table, written because it was asked for. + +Not `gitrepo.init`, which makes the store's repository and is tested in +`test_gitrepo.py`. This one writes `config.toml`, and the properties under test +are the ones that make writing it safe: the file also holds the `[remote.*]` +push opt-ins, so it is appended to and never rewritten; a second run writes +nothing; and a table the watcher would skip is refused before it is written, +with the watcher's own sentence, rather than written and left to fail quietly +on every pass. +""" + +from __future__ import annotations + +import os +import re +import stat +import subprocess + +import pytest + +from gitmemory import daemon, gitrepo, redact +from gitmemory.__main__ import main + + +def _cfg(home: str) -> str: + return os.path.join(home, "config.toml") + + +def test_cli_init_writes_the_table_the_watcher_reads(tmp_path, capsys, _no_real_home): + """No config, one command, and the watcher watches the documented root.""" + home = str(tmp_path / "store") + assert main(["--home", home, "init", "claude-code"]) == 0 + out, err = capsys.readouterr() + assert out.startswith(f"wrote {_cfg(home)}: watch ~/.claude/projects for claude-code\n") + assert "next: run `gitmemory watch`" in out + assert "~/.claude/projects does not exist yet" in err + with open(_cfg(home), encoding="utf-8") as fh: + assert fh.read().startswith("# added by `gitmemory init claude-code`\n[[watch]]\n") + want = os.path.realpath(_no_real_home / ".claude" / "projects") + assert daemon.load_watches(home) == [daemon.Watch("claude-code", (want,))] + assert stat.S_IMODE(os.stat(_cfg(home)).st_mode) == 0o600 + assert stat.S_IMODE(os.stat(home).st_mode) == 0o700 + + +def test_cli_init_twice_is_one_table(tmp_path, capsys): + home = str(tmp_path / "store") + assert main(["--home", home, "init", "pi"]) == 0 + with open(_cfg(home), "rb") as fh: + first = fh.read() + capsys.readouterr() + assert main(["--home", home, "init", "pi"]) == 0 + with open(_cfg(home), "rb") as fh: + assert fh.read() == first + assert "already watches ~/.pi/agent/sessions for pi; nothing to do" in capsys.readouterr().out + + +def test_cli_init_appends_and_keeps_every_byte_already_there(tmp_path): + """`config.toml` is also where a push is opted into. Rewriting it would + revoke the remote; appending has to leave it, comments and all.""" + home = tmp_path / "store" + home.mkdir() + old = b'# mine\n[remote.origin]\nurl = "git@host:me/p.git"\nallow_push = true' # no final \n + (home / "config.toml").write_bytes(old) + assert main(["--home", str(home), "init", "claude-code"]) == 0 + assert main(["--home", str(home), "init", "pi"]) == 0 + assert (home / "config.toml").read_bytes().startswith(old + b"\n") + assert redact.push_allowed(str(home), "origin")[0] + assert [w.agent for w in daemon.load_watches(str(home))] == ["claude-code", "pi"] + + +@pytest.mark.parametrize("case", ["file-root", "store-under-root", "root-under-store"]) +def test_cli_init_refuses_a_table_the_watcher_would_skip_and_writes_nothing( + tmp_path, capsys, _no_real_home, case +): + """Each one would be skipped by the watcher on every pass. The refusal is + the watcher's own sentence, so the two cannot disagree about why, and the + two overlaps are told apart: a root inside the store does not contain it.""" + projects = _no_real_home / ".claude" / "projects" + if case == "file-root": + projects.parent.mkdir() + projects.write_text("one transcript, not a directory") + home, why = str(tmp_path / "store"), "is a file, not a directory" + elif case == "store-under-root": + projects.mkdir(parents=True) + home, why = str(projects / "store"), "contains the store itself" + else: + home, why = str(_no_real_home / ".claude"), "is inside the store" + assert main(["--home", home, "init", "claude-code"]) == 1 + err = capsys.readouterr().err + assert re.search(rf"^refusing: .*\[\[watch\]\] #1: root ~/.claude/projects {why}", err, re.M) + assert not os.path.lexists(home), "a refusal writes nothing, not even the store" + + +@pytest.mark.parametrize( + ("body", "why"), + [ + (b"[[watch]\nnot toml", "is not valid TOML"), + (b"\xff\xfe", "is not valid TOML"), + (b'watch = [{agent = "pi", roots = ["/nowhere"]}]\n', "cannot join"), + (b'[watch]\nagent = "pi"\n', "cannot join"), + (b"watch = 5\n", "cannot join"), + ], + ids=["syntax", "not-utf8", "inline-array", "plain-table", "not-a-list"], +) +def test_cli_init_refuses_a_config_it_cannot_append_to_and_leaves_it( + tmp_path, capsys, body, why +): + """The last three parse, and appending `[[watch]]` to them does not: the + reason is said rather than left to whichever check happens to trip next.""" + home = tmp_path / "store" + home.mkdir() + (home / "config.toml").write_bytes(body) + assert main(["--home", str(home), "init", "claude-code"]) == 1 + err = capsys.readouterr().err + assert err.startswith("refusing: ") and why in err, err + assert (home / "config.toml").read_bytes() == body + + +@pytest.mark.parametrize("live", [True, False], ids=["live", "dangling"]) +def test_cli_init_does_not_replace_a_symlinked_config(tmp_path, capsys, live): + """`os.replace` over a link swaps the link for a file, so the user's dotfile + would silently stop being the config. Refused instead.""" + home = tmp_path / "store" + home.mkdir() + target = tmp_path / "dotfiles.toml" + if live: + target.write_text("") + (home / "config.toml").symlink_to(target) + assert main(["--home", str(home), "init", "claude-code"]) == 1 + assert "is not a regular file" in capsys.readouterr().err + assert os.path.islink(home / "config.toml") + assert (target.read_text() == "") if live else not target.exists() + + +def test_cli_init_does_not_split_a_hard_linked_config(tmp_path, capsys): + """A regular file, and still a link: `os.replace` gives `config.toml` a new + inode, so the dotfile hard-linked to it would keep the old bytes, push + opt-in and all, and silently stop being the config. Refused, like a symlink.""" + home = tmp_path / "store" + home.mkdir() + dotfile = tmp_path / "dotfiles.toml" + body = '[remote.origin]\nurl = "git@host:me/p.git"\nallow_push = true\n' + dotfile.write_text(body) + os.link(dotfile, home / "config.toml") + assert main(["--home", str(home), "init", "claude-code"]) == 1 + assert "has another hard link" in capsys.readouterr().err + assert os.path.samefile(dotfile, home / "config.toml") + assert dotfile.read_text() == body + + +@pytest.mark.parametrize("case", ["home-is-a-file", "unreadable"]) +def test_cli_init_says_error_and_exits_2_when_it_could_not_look(tmp_path, capsys, case): + """Exit 1 is an answer: init looked, and refused. A config it could not + read is not one, and taking it for absent would replace it with the table + alone, push opt-ins gone. So it is main's `error:` and exit 2, and the + file is left as it was.""" + home = tmp_path / "store" + body = '[remote.origin]\nurl = "git@host:me/p.git"\nallow_push = true\n' + if case == "home-is-a-file": + home.write_text(body) + kept = home + else: + if hasattr(os, "geteuid") and os.geteuid() == 0: + pytest.skip("root reads a mode-000 file") + home.mkdir() + kept = home / "config.toml" + kept.write_text(body) + kept.chmod(0) + try: + assert main(["--home", str(home), "init", "pi"]) == 2 + finally: + kept.chmod(0o600) + assert capsys.readouterr().err.startswith("error: ") + assert kept.read_text() == body + + +def test_cli_init_says_so_when_another_init_wrote_first(tmp_path, capsys, monkeypatch): + """Two inits at once read the same bytes, and the later rename dropped the + other's table while both said "added". The other one is run here between + this one's read and its write: this one now sees the file move, and says so.""" + home = str(tmp_path / "store") + watches_from = daemon.watches_from + + def racer(*args): + monkeypatch.setattr(daemon, "watches_from", watches_from) + assert main(["--home", home, "init", "pi"]) == 0 + return watches_from(*args) + + monkeypatch.setattr(daemon, "watches_from", racer) + assert main(["--home", home, "init", "claude-code"]) == 1 + assert "changed while init was reading it; run it again" in capsys.readouterr().err + assert [w.agent for w in daemon.load_watches(home)] == ["pi"] + + +def test_cli_init_refuses_a_root_another_agent_already_watches(tmp_path, capsys, _no_real_home): + """Two watches on one root at the same depth: `discover` keeps the first, so + the table init would add is dead on arrival. Spelled absolute and with a + trailing slash, because the comparison is the watcher's resolved path.""" + home = tmp_path / "store" + home.mkdir() + body = f'[[watch]]\nagent = "pi"\nroots = ["{_no_real_home}/.claude/projects/"]\n' + (home / "config.toml").write_text(body) + assert main(["--home", str(home), "init", "claude-code"]) == 1 + assert "already watches ~/.claude/projects as 'pi'" in capsys.readouterr().err + assert (home / "config.toml").read_text() == body + + +def test_cli_init_knows_only_adapters_that_ship_and_roots_the_docs_name(): + """`ROOTS` is a third copy of a path the adapter's docstring and agents.md + already state, so it is checked against both. An agent with no documented + root is not offered: argparse refuses it as a usage error.""" + from gitmemory.adapters import ADAPTERS + + here = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + with open(os.path.join(here, "docs", "agents.md"), encoding="utf-8") as fh: + agents = fh.read() + for agent, root in daemon.ROOTS.items(): + assert agent in ADAPTERS + assert f"`{root}/**/*.jsonl`" in agents, root + assert ADAPTERS[agent].__doc__.split(":")[1].strip().startswith(root), agent + with pytest.raises(SystemExit) as done: + main(["init", "kimi-code"]) + assert done.value.code == 2 + + +def test_cli_init_leaves_no_temp_the_watcher_would_commit(tmp_path, monkeypatch): + """`init` publishes `config.toml` by rename, so an init killed between the + write and the `os.replace` leaves a full copy of the file — push credential + included — beside it, and the watcher's `git add --all` would commit it. + Asked of the temp the writer really made, while it exists: a name typed into + the test passed against a writer that spelled its temp any other way.""" + home = str(tmp_path / "store") + gitrepo.init(home) + asked = [] + rename = os.replace + + def spy(src, dst): + if os.path.basename(dst) == "config.toml": + rel = os.path.relpath(src, home) + ask = subprocess.run(["git", "-C", home, "check-ignore", "-q", rel], check=False) + asked.append((rel, ask.returncode)) + return rename(src, dst) + + monkeypatch.setattr(os, "replace", spy) + assert main(["--home", home, "init", "pi"]) == 0 + assert [code for _, code in asked] == [0], asked diff --git a/tests/test_service.py b/tests/test_service.py new file mode 100644 index 0000000..f60026d --- /dev/null +++ b/tests/test_service.py @@ -0,0 +1,357 @@ +# SPDX-FileCopyrightText: 2026 The gitmemory authors +# SPDX-License-Identifier: Apache-2.0 +"""`gitmemory service install`: one inert file that keeps `watch` running. + +The property under test: what a user service manager starts is the watcher the +user's shell would have run. systemd and launchd hand a job none of the shell's +state — not its working directory, not `$GITMEMORY_HOME`, not its `PATH` — so +everything the watcher needs has to be written into the file, and the only +proof that it was is to run the file's own command line under the manager's +conditions. That is the end-to-end test below; the others pin what each +manager reads and that the command writes, refuses and never starts. + +Nothing here runs `systemctl`, `launchctl` or `loginctl`. HOME comes from +conftest and XDG_CONFIG_HOME is set or unset per test, so every file lands in +tmp_path, and both renderers run on either OS by setting `sys.platform`. The +one test that runs a printed command puts stand-ins for the managers first on +its PATH. +""" + +from __future__ import annotations + +import os +import plistlib +import re +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from gitmemory.__main__ import main + +LABEL = "io.github.doronp.gitmemory" + + +@pytest.fixture +def xdg(tmp_path, monkeypatch): + path = tmp_path / "xdg" + monkeypatch.setenv("XDG_CONFIG_HOME", str(path)) + return path + + +@pytest.fixture +def no_spawn(monkeypatch): + """Fail on any child process: the command writes a file and starts nothing. + + `Popen` is what `run`, `call` and `check_output` all construct, so one patch + covers every way the module could reach `systemctl` or `launchctl`. + """ + + def spawned(*args, **kwargs): + raise AssertionError(f"service install spawned {args!r}") + + monkeypatch.setattr(subprocess, "Popen", spawned) + + +def _install(monkeypatch, platform: str, *argv: str) -> int: + with monkeypatch.context() as m: + m.setattr(sys, "platform", platform) + return main([*argv, "service", "install"]) + + +def _unit(path) -> dict[str, str]: + """`key -> value` for a unit file's assignments; sections are not needed.""" + lines = Path(path).read_text(encoding="utf-8").splitlines() + return dict(line.split("=", 1) for line in lines if "=" in line) + + +_QUOTED = re.compile(r'"((?:[^"\\]|\\.)*)"(?: |$)', re.S) + + +def _words(value: str, *, dollars: bool) -> list[str]: + """Read a unit's words back as systemd.syntax(7) says systemd does, and + fail on anything it would read as something else. + + Strict, because a lenient reader is what made the quoting test decorative: + turning `%%` into `%` also lets an undoubled `%h` through as itself, so the + round trip held whether or not the unit escaped anything. To systemd a `%h` + is a specifier, a lone `$` on a command line is a variable, and `\\z` is an + escape it rejects, so each is a failure here. Order as systemd's: unquote, + then specifiers, then (`ExecStart=` only) variables. The test's reading of + the manual, not systemd's parser; there is no systemd to ask on a Mac. + """ + + def strict(pattern: str, keep: tuple[str, ...], word: str) -> str: + def one(m: re.Match) -> str: + assert m.group(1) in keep, f"systemd reads {m.group(0)!r} in {value!r}" + return m.group(1) + + return re.sub(pattern, one, word, flags=re.S) + + words, pos = [], 0 + while pos < len(value): + m = _QUOTED.match(value, pos) + assert m, f"not a double-quoted word at {pos}: {value!r}" + word = strict(r"%(.?)", ("%",), strict(r"\\(.?)", ("\\", '"'), m.group(1))) + words.append(strict(r"\$(.?)", ("$",), word) if dollars else word) + pos = m.end() + return words + + +def _expected(home) -> list[str]: + return [sys.executable, "-P", "-m", "gitmemory", "--home", os.path.realpath(home), "watch"] + + +def test_service_install_writes_a_systemd_user_unit_and_starts_nothing( + tmp_path, monkeypatch, capsys, xdg, no_spawn +): + home = tmp_path / "store" + assert _install(monkeypatch, "linux", "--home", str(home)) == 0 + path = xdg / "systemd" / "user" / "gitmemory.service" + unit = _unit(path) + assert _words(unit["ExecStart"], dollars=True) == _expected(home) + env = _words(unit["Environment"], dollars=False) + assert env == [f"PATH={os.path.dirname(shutil.which('git'))}:/usr/bin:/bin"], env + assert unit["Restart"] == "on-failure" and unit["RestartSec"] == "30" + assert unit["WantedBy"] == "default.target" + out, err = capsys.readouterr() + assert out.startswith(f"wrote {path}\n") and not err + ctl = "systemctl --user" + assert f"{ctl} daemon-reload && {ctl} enable --now gitmemory.service" in out + assert "loginctl enable-linger" in out + removal = f"{ctl} disable --now gitmemory.service; rm {shlex.quote(str(path))}" + assert f" {removal}; {ctl} daemon-reload\n" in out + + +def test_service_install_writes_a_launch_agent_and_starts_nothing( + tmp_path, monkeypatch, capsys, no_spawn +): + """No `--home`: the store comes from `$GITMEMORY_HOME`, which a LaunchAgent + started at login never sees, so the file has to name it outright.""" + home = tmp_path / "elsewhere" + monkeypatch.setenv("GITMEMORY_HOME", str(home)) + assert _install(monkeypatch, "darwin") == 0 + user = os.environ["HOME"] + path = os.path.join(user, "Library", "LaunchAgents", f"{LABEL}.plist") + with open(path, "rb") as fh: + job = plistlib.load(fh) + assert job["Label"] == LABEL + assert job["ProgramArguments"] == _expected(home) + git = os.path.dirname(shutil.which("git")) + assert job["EnvironmentVariables"] == {"PATH": f"{git}:/usr/bin:/bin"} + assert job["WorkingDirectory"] == user + assert job["RunAtLoad"] is True and job["KeepAlive"] == {"SuccessfulExit": False} + assert job["ThrottleInterval"] == 30 + # Outside the store, which commits everything in it with `git add --all`. + log = os.path.join(user, "Library", "Logs", "gitmemory.log") + assert job["StandardOutPath"] == job["StandardErrorPath"] == log + out, _ = capsys.readouterr() + assert f"launchctl bootstrap gui/$(id -u) {shlex.quote(path)}" in out + assert f"launchctl bootout gui/$(id -u)/{LABEL}; rm {shlex.quote(path)}" in out + + +@pytest.mark.parametrize("platform", ["linux", "darwin"]) +def test_the_installed_service_runs_the_watcher_from_its_own_working_directory( + tmp_path, monkeypatch, capsys, xdg, platform +): + """Run the file's command line the way its manager would, plus `--once`. + + The manager's conditions are: the working directory is `$HOME` (systemd's + default for a user unit, and what the plist sets), the environment is HOME + and the file's PATH and nothing else, and `$HOME` holds a `gitmemory.py`. + Each is a trap a hand-written unit fell into: `realpath(sys.executable)` + leaves the venv and cannot import gitmemory; `python -m` without `-P` puts + the working directory first on `sys.path` and runs the stray file instead; + and without `--home` the watcher reads `~/.gitmemory`, not the store the + user configured, and watches nothing. + """ + home, roots = tmp_path / "store", tmp_path / "roots" + os.makedirs(home) + os.makedirs(roots / "proj") + (roots / "proj" / "s1.jsonl").write_text( + '{"type":"user","message":{"role":"user","content":"hello"}}\n' + ) + (home / "config.toml").write_text(f'[[watch]]\nagent = "claude-code"\nroots = ["{roots}"]\n') + user = os.environ["HOME"] + with open(os.path.join(user, "gitmemory.py"), "w") as fh: + fh.write("raise SystemExit('a gitmemory.py in the working directory ran instead')\n") + assert _install(monkeypatch, platform, "--home", str(home)) == 0 + if platform == "linux": + unit = _unit(xdg / "systemd" / "user" / "gitmemory.service") + argv = _words(unit["ExecStart"], dollars=True) + path = _words(unit["Environment"], dollars=False)[0].removeprefix("PATH=") + cwd = user + else: + with open(os.path.join(user, "Library", "LaunchAgents", f"{LABEL}.plist"), "rb") as fh: + job = plistlib.load(fh) + argv, path = job["ProgramArguments"], job["EnvironmentVariables"]["PATH"] + cwd = job["WorkingDirectory"] + capsys.readouterr() + done = subprocess.run( + [*argv, "--once"], + cwd=cwd, + env={"HOME": user, "PATH": path}, + capture_output=True, + text=True, + timeout=120, + ) + assert done.returncode == 0, done.stderr + assert "captured 1 " in done.stderr, done.stderr + assert main(["--home", str(home), "verify"]) == 0 + + +@pytest.mark.parametrize("platform", ["linux", "darwin"]) +def test_service_install_refuses_to_overwrite_an_existing_file( + tmp_path, monkeypatch, capsys, xdg, no_spawn, platform +): + """Exit 1, the file untouched, and one line that says how to replace it. + + A file already there may be one the user edited; a second install is not a + reason to lose that.""" + if platform == "linux": + path = xdg / "systemd" / "user" / "gitmemory.service" + else: + path = Path(os.environ["HOME"], "Library", "LaunchAgents", f"{LABEL}.plist") + os.makedirs(path.parent) + path.write_bytes(b"the user's own\n") + assert _install(monkeypatch, platform, "--home", str(tmp_path / "store")) == 1 + assert path.read_bytes() == b"the user's own\n" + out, err = capsys.readouterr() + assert not out and err.count("\n") == 1, err + assert err.startswith(f"refusing: {path} exists") and f"rm {shlex.quote(str(path))}" in err + + +def test_a_unit_survives_a_home_with_quotes_percent_and_dollar_in_it( + tmp_path, monkeypatch, capsys, xdg, no_spawn +): + """systemd expands `%h` and `$HOME` inside a quoted word and rejects `\\z`, + so a store whose path spells any of them would be read as some other path, + or the unit not read at all. `_words` fails on each unless it is escaped.""" + home = tmp_path / 'a b %h $HOME "q" \\z' + assert _install(monkeypatch, "linux", "--home", str(home)) == 0 + path = xdg / "systemd" / "user" / "gitmemory.service" + assert _words(_unit(path)["ExecStart"], dollars=True) == _expected(home) + + +@pytest.mark.parametrize("platform", ["linux", "darwin"]) +@pytest.mark.parametrize("brk", ["\n", "\r"], ids=["lf", "cr"]) +def test_a_line_break_in_the_store_path_is_refused_on_either_platform( + tmp_path, monkeypatch, capsys, xdg, no_spawn, platform, brk +): + """Exit 2 and no file. A unit file would end the line there; a plist holds + `\\n`, but plistlib turns `\\r` into `\\n`, so that service would watch a + path that does not exist rather than refuse.""" + assert _install(monkeypatch, platform, "--home", str(tmp_path / f"a{brk}b")) == 2 + assert capsys.readouterr().err.startswith("error: ") + assert not os.path.exists(xdg) and not os.listdir(os.environ["HOME"]) + + +@pytest.mark.parametrize("platform", ["linux", "darwin"]) +def test_the_printed_removal_works_when_the_job_was_never_started( + tmp_path, monkeypatch, capsys, xdg, platform +): + """The refusal's way out, pasted, removes the file even when the stop fails. + + Written and never started is the state a second install meets: the user + reads `it runs:`, sees the wrong store, and installs again. `launchctl + bootout` of a job that was never bootstrapped exits 3 ("No such process"), + and a user manager that cannot be reached fails `systemctl --user` too, so + a removal that waits on the stop leaves the file, and the next install is + refused with the same advice. The stand-ins below fail that way; the real + managers are never on the PATH they run with. + """ + if platform == "linux": + path = xdg / "systemd" / "user" / "gitmemory.service" + else: + path = Path(os.environ["HOME"], "Library", "LaunchAgents", f"{LABEL}.plist") + store = str(tmp_path / "store") + assert _install(monkeypatch, platform, "--home", store) == 0 + out = capsys.readouterr().out + printed = out.split("Remove it:\n ", 1)[1].rstrip("\n") + assert _install(monkeypatch, platform, "--home", store) == 1 + refusal = capsys.readouterr().err + assert refusal.rstrip("\n").endswith(f": {printed}"), refusal + stubs = tmp_path / "stubs" + stubs.mkdir() + fails = "#!/bin/sh\necho 'Boot-out failed: 3: No such process' >&2\nexit 3\n" + for manager in ("systemctl", "launchctl"): + (stubs / manager).write_text(fails) + (stubs / manager).chmod(0o755) + done = subprocess.run( + ["/bin/sh", "-c", printed], + env={"HOME": os.environ["HOME"], "PATH": f"{stubs}:/usr/bin:/bin"}, + capture_output=True, + text=True, + timeout=30, + ) + assert not path.exists(), (printed, done.stderr) + assert _install(monkeypatch, platform, "--home", store) == 0 + + +@pytest.mark.parametrize("value", [None, "", "rel"], ids=["unset", "empty", "relative"]) +def test_a_unit_lands_under_dot_config_without_an_absolute_xdg_config_home( + tmp_path, monkeypatch, capsys, no_spawn, value +): + """The default most Linux users get, and the XDG rule that a relative + `$XDG_CONFIG_HOME` is invalid and ignored: a unit written relative to + wherever install ran is one no user manager looks for.""" + monkeypatch.chdir(tmp_path) + if value is None: + monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) + else: + monkeypatch.setenv("XDG_CONFIG_HOME", value) + assert _install(monkeypatch, "linux", "--home", str(tmp_path / "store")) == 0 + path = os.path.join(os.environ["HOME"], ".config", "systemd", "user", "gitmemory.service") + assert _words(_unit(path)["ExecStart"], dollars=True) == _expected(tmp_path / "store") + assert not (tmp_path / "rel").exists() + out = capsys.readouterr().out + removal = f"; rm {shlex.quote(path)}; systemctl --user daemon-reload\n" + assert out.startswith(f"wrote {path}\n") and removal in out + + +@pytest.mark.parametrize("platform", ["linux", "darwin"]) +def test_git_found_through_a_relative_path_entry_is_written_absolute( + tmp_path, monkeypatch, capsys, xdg, no_spawn, platform +): + """`PATH=relbin:...` makes `which` answer `relbin/git`. Written as it is, + the service would resolve it against its own working directory, `$HOME`, + not the directory install ran in.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(shutil, "which", lambda name, *a, **k: os.path.join("relbin", name)) + assert _install(monkeypatch, platform, "--home", str(tmp_path / "store")) == 0 + if platform == "linux": + unit = _unit(xdg / "systemd" / "user" / "gitmemory.service") + path = _words(unit["Environment"], dollars=False)[0].removeprefix("PATH=") + else: + plist = Path(os.environ["HOME"], "Library", "LaunchAgents", f"{LABEL}.plist") + path = plistlib.loads(plist.read_bytes())["EnvironmentVariables"]["PATH"] + assert path == f"{os.path.join(os.getcwd(), 'relbin')}:/usr/bin:/bin" + + +@pytest.mark.parametrize("exe", ["", None], ids=["empty", "none"]) +def test_service_install_refuses_when_it_cannot_tell_which_python_runs( + tmp_path, monkeypatch, capsys, xdg, no_spawn, exe +): + """`sys.executable` is empty or None when Python cannot say where it is. + A unit naming `""` cannot start, and None was a traceback; exit 2 instead, + with nothing written.""" + monkeypatch.setattr(sys, "executable", exe) + assert _install(monkeypatch, "linux", "--home", str(tmp_path / "store")) == 2 + assert capsys.readouterr().err.startswith("error: ") + assert not os.path.exists(xdg) + + +def test_service_install_without_git_or_on_an_unknown_platform_writes_nothing( + tmp_path, monkeypatch, capsys, xdg, no_spawn +): + """Exit 2 either way: the watcher commits with git, and it would find none.""" + home = str(tmp_path / "store") + assert _install(monkeypatch, "freebsd14", "--home", home) == 2 + assert "freebsd14" in capsys.readouterr().err + monkeypatch.setattr(shutil, "which", lambda name, *a, **k: None) + assert _install(monkeypatch, "linux", "--home", home) == 2 + assert "git" in capsys.readouterr().err + assert not os.path.exists(xdg) and not os.listdir(os.environ["HOME"]) diff --git a/tests/test_show.py b/tests/test_show.py new file mode 100644 index 0000000..fc47ac8 --- /dev/null +++ b/tests/test_show.py @@ -0,0 +1,333 @@ +# SPDX-FileCopyrightText: 2026 The gitmemory authors +# SPDX-License-Identifier: Apache-2.0 +"""`gitmemory show
`: the turn behind a line `recall` printed. + +`recall` cuts a turn to 160 characters and hands back an address. Until `show`, +the rest of the turn was segment arithmetic done by hand under `raw/`. These +tests hold the three things that make the address worth printing: it resolves +from the store and not from the index, it never becomes a path, and nothing in +a transcript can make the output say a different address than the one it is. +""" + +from __future__ import annotations + +import json +import os + +import pytest +from test_index import AKIA, PEM, PEM_BODY, assistant, user, write + +from gitmemory import redact, store +from gitmemory.__main__ import main + + +@pytest.fixture +def home(tmp_path): + h = tmp_path / "store" + h.mkdir() + return str(h) + + +@pytest.fixture +def src(tmp_path): + p = tmp_path / "src" / "sess.jsonl" + p.parent.mkdir() + return str(p) + + +def size(obj: dict) -> int: + """One JSONL line's length, newline included: the next turn's offset.""" + return len(json.dumps(obj).encode()) + 1 + + +def show(home: str, capsys, *argv: str) -> tuple[int, str, str]: + code = main(["--home", home, "show", *argv]) + out, err = capsys.readouterr() + return code, out, err + + +def test_show_prints_the_whole_turn_recall_cut_at_160(home, src, capsys): + long = "the marmoset rule: " + "never retry the upload " * 20 + "END-OF-TURN" + write(src, [user("u1", long) | {"timestamp": "2026-09-28T10:00:00Z"}]) + store.capture(src, "claude-code", "sess", home=home) + assert main(["--home", home, "index"]) == 0 + capsys.readouterr() + assert main(["--home", home, "recall", "marmoset"]) == 0 + out = capsys.readouterr().out + assert "END-OF-TURN" not in out + code, out, err = show(home, capsys, out.split()[2]) + assert (code, err) == (0, "") + assert out == f"> claude-code/sess/g00@0 2026-09-28T10:00Z user\n text\n {long}\n" + + +def test_show_needs_no_index(home, src, capsys): + write(src, [user("u1", "aardvark")]) + store.capture(src, "claude-code", "sess", home=home) + code, out, _ = show(home, capsys, "claude-code/sess/g00@0") + assert code == 0 and " aardvark\n" in out + assert not os.path.exists(os.path.join(home, "index")) + + +def test_show_resolves_a_turn_that_straddles_a_segment_cut(home, src, capsys): + first, second = user("u1", "first"), user("u2", "the peculiar marmoset") + write(src, [first, second]) + with open(src, "rb") as fh: + whole = fh.read() + with open(src, "wb") as fh: + fh.write(whole[: size(first) + 10]) # the cut lands inside the second line + store.capture(src, "claude-code", "sess", home=home) + with open(src, "wb") as fh: + fh.write(whole) + store.capture(src, "claude-code", "sess", home=home) + assert len(store.sessions(home)[0].segments) == 2 + code, out, _ = show(home, capsys, f"claude-code/sess/g00@{size(first)}") + assert code == 0 + assert out.splitlines()[0].startswith(f"> claude-code/sess/g00@{size(first)} ") + assert " the peculiar marmoset" in out.splitlines() + + +def test_show_context_prints_neighbours_in_order_and_marks_the_target(home, src, capsys): + turns = [user(f"u{i}", f"turn-{i}") for i in range(5)] + write(src, turns) + store.capture(src, "claude-code", "sess", home=home) + at = size(turns[0]) + size(turns[1]) + code, out, _ = show(home, capsys, f"claude-code/sess/g00@{at}", "--context", "1") + assert code == 0 + heads = [ln for ln in out.splitlines() if not ln.startswith(" ")] + assert [h[0] for h in heads] == ["-", ">", "-"] + assert [ln.strip() for ln in out.splitlines() if ln.startswith(" ")] == [ + "turn-1", + "turn-2", + "turn-3", + ] + + +def test_an_offset_inside_a_turn_shows_that_turn_at_its_start(home, src, capsys): + write(src, [user("u1", "aardvark"), user("u2", "marmoset")]) + store.capture(src, "claude-code", "sess", home=home) + start = size(user("u1", "aardvark")) + code, out, _ = show(home, capsys, f"claude-code/sess/g00@{start + 5}") + assert code == 0 and out.startswith(f"> claude-code/sess/g00@{start} ") + assert "marmoset" in out and "aardvark" not in out + + +def test_a_subagent_turn_is_headed_subagent_not_user(home, src, capsys): + """A sidechain's user-role text is the parent agent's brief, not the user's + word, so the header says whose it is. Same column, so a reader that splits + the header still finds the role where it was.""" + brief = user("u1", "review the upload code") | {"isSidechain": True} + write(src, [brief, assistant("a2", [{"type": "text", "text": "done"}])]) + store.capture(src, "claude-code", "sess", home=home) + code, out, _ = show(home, capsys, "claude-code/sess/g00@0", "--context", "1") + assert code == 0 + heads = [ln for ln in out.splitlines() if not ln.startswith(" ")] + assert heads == [ + "> claude-code/sess/g00@0 undated subagent:user", + f"- claude-code/sess/g00@{size(brief)} undated assistant", + ] + + +@pytest.mark.parametrize( + "address", + [ + "../../etc/passwd@0", + "claude-code/../g00@0", + "claude-code/.git/g00@0", + "claude-code/sess\n/g00@0", + "claude-code/sess/g00@\u0663", # ARABIC-INDIC DIGIT THREE: int() reads it as 3 + "claude-code/sess/g00", + "claude-code/sess@0", + "/etc/sess/g00@0", + ], +) +def test_a_malformed_or_escaping_address_is_not_an_address(home, src, capsys, address): + write(src, [user("u1", "aardvark")]) + store.capture(src, "claude-code", "sess", home=home) + code, out, err = show(home, capsys, address) + assert (code, out) == (2, "") and err.startswith("not an address: ") + assert err.count("\n") == 1 + + +def test_a_masked_address_is_refused_as_masked_not_as_mistyped(home, src, capsys): + """`recall --redact` masks a credential-shaped session id in the address, + and a masked address is not one `show` can look up. The refusal said + `expected //gNN@, as recall prints it` about the + string recall had just printed, so it now says which it is. Still exit 2.""" + ghp = "ghp_" + "a" * 36 # lowercase, because the store case-folds a session id + write(src, [user("u1", "walrus note")]) + store.capture(src, "claude-code", ghp, home=home) + assert main(["--home", home, "index"]) == 0 + capsys.readouterr() + assert main(["--home", home, "recall", "walrus", "--redact"]) == 0 + address = capsys.readouterr().out.split()[2] + assert address == "claude-code/ghp_…[40bytes]/g00@0", address + code, out, err = show(home, capsys, address) + assert (code, out) == (2, "") and err.count("\n") == 1, err + assert err.startswith("a masked address: ") and "as recall prints it" not in err, err + + +def test_an_address_the_store_does_not_hold_is_an_answer_exit_1(home, src, capsys): + write(src, [user("u1", "aardvark")]) + store.capture(src, "claude-code", "sess", home=home) + code, out, err = show(home, capsys, "claude-code/sess/g01@0") + assert (code, out, err) == (1, "", "claude-code/sess/g01: not in the store\n") + code, out, err = show(home, capsys, "claude-code/sess/g00@99999") + assert (code, out, err) == (1, "", "claude-code/sess/g00: no turn at byte 99999\n") + + +def test_a_credential_in_an_address_is_masked_in_the_error(home, src, capsys): + """A session id is whatever `capture --session-id` was given, and these + pass the path-component rule. The error masks the address as it was typed: + `identity()` lowercases, and an AWS key id lowercased is one no detector + knows, so masking the store's spelling printed it whole.""" + write(src, [user("u1", "aardvark")]) + store.capture(src, "claude-code", "sess", home=home) + token = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8" + code, _, err = show(home, capsys, f"claude-code/{token}/g00@0") + assert code == 1 and "ghp_" in err and token.lower() not in err.lower() + key = "AKIAZZZZQQQQWWWW1234" # synthetic, right shape + code, _, err = show(home, capsys, f"claude-code/{key}/g00@0") + assert code == 1 and "AKIA" in err and key.lower() not in err.lower(), err + + +def test_a_transcript_cannot_forge_a_show_header_or_reach_the_terminal(home, src, capsys): + """Only a header starts in column 0, so a line that looks like one has to + have come from `show`. Two forgeries got past the first version: a tool + name with a newline in it printed a header of its own, and `splitlines()` + broke on a carriage return before `safe_text` could escape it.""" + forged = "> claude-code/forged/g00@0 2026-01-01T00:00Z user" + text = f"needle \x1b[2K\r{forged}\n{forged}\u2028{forged}\x85{forged}" + tool = {"type": "tool_use", "id": "t", "name": f"Bash\n{forged}", "input": {"command": "ls"}} + write(src, [user("u1", text), assistant("a2", [tool])]) + store.capture(src, "claude-code", "sess", home=home) + code, out, _ = show(home, capsys, "claude-code/sess/g00@0", "--context", "1") + assert code == 0 + assert not any(c in out for c in "\x1b\r\u2028\x85") + assert out.split("\n")[2] == " needle \\x1b[2K\\x0d" + forged + heads = [ln for ln in out.splitlines() if not ln.startswith(" ")] + assert len(heads) == 2 and not any("forged" in h for h in heads), heads + + +def test_two_manifests_claiming_one_address_are_refused(home, src, capsys): + """`verify` flags a manifest whose declared identity is not its path. Until + it has been run, which of the two is the turn is not something `show` can + know, so it does not pick one.""" + write(src, [user("u1", "aardvark")]) + store.capture(src, "claude-code", "sess", home=home) + store.capture(src, "claude-code", "other", home=home) + other = os.path.join(home, "sessions", "claude-code", "other", "g00.json") + with open(other) as fh: + manifest = json.load(fh) + manifest["session_id"] = "sess" + with open(other, "w") as fh: + json.dump(manifest, fh) + code, out, err = show(home, capsys, "claude-code/sess/g00@0") + assert (code, out) == (1, "") and "claimed by 2 manifests" in err + + +def test_a_captured_show_result_is_an_echo_the_skill_lists(home, tmp_path, capsys): + """A session that ran `show` was captured too, with the whole turn in a + tool result dated after it, so `recall` finds the turn again under the + echo's own address. The skill tells an agent to run `show`, so it has to + tell it how to know the echo: by the header at the front of its text.""" + first, later = str(tmp_path / "a.jsonl"), str(tmp_path / "b.jsonl") + rule = user("u1", "the marmoset rule: never retry the upload") + write(first, [rule | {"timestamp": "2026-09-01T10:00:00Z"}]) + store.capture(first, "claude-code", "a", home=home) + code, said, _ = show(home, capsys, "claude-code/a/g00@0") + call = {"type": "tool_use", "id": "t1", "name": "Bash", "input": {"command": "gitmemory show"}} + result = {"type": "tool_result", "tool_use_id": "t1", "content": said} + echo = user("u3", "") | {"timestamp": "2026-10-01T10:00:00Z"} + echo["message"]["content"] = [result] + write(later, [assistant("a2", [call]), echo]) + store.capture(later, "claude-code", "b", home=home) + assert code == 0 and main(["--home", home, "index"]) == 0 + capsys.readouterr() + assert main(["--home", home, "recall", "marmoset"]) == 0 + heads = [ln.split(None, 4) for ln in capsys.readouterr().out.splitlines()] + assert [h[2].split("@")[0] for h in heads] == ["claude-code/a/g00", "claude-code/b/g00"] + assert heads[1][4].startswith("> claude-code/a/g00@0 "), heads + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + with open(os.path.join(root, "skills", "recall", "SKILL.md")) as fh: + echoes = fh.read().split("\n## Echoes\n")[1].split("\n## ")[0] + assert "`gitmemory show ...`" in echoes and "`> `" in echoes, echoes + # The header's shape, not "a quote or a bullet that names an address": + # a real turn can open with either and cite one further on. + assert "`- ` immediately followed by an `/" in " ".join(echoes.split()), echoes + + +def test_show_on_no_store_and_a_negative_context_exit_2(tmp_path, capsys): + code, _, err = show(str(tmp_path / "absent"), capsys, "claude-code/sess/g00@0") + assert code == 2 and err.startswith("not a store: ") + with pytest.raises(SystemExit) as done: + main(["--home", str(tmp_path), "show", "claude-code/sess/g00@0", "-C", "-1"]) + assert done.value.code == 2 + + +def test_show_redact_masks_a_whole_turn_and_the_default_prints_it(home, src, capsys): + """What `recall --redact` masked one line of, whole. A private key's body is + the lines under its header, so a block is masked before it is split into + lines, never one line at a time. The address and a tool's name are masked + too, and the address stays one field, as recall's does.""" + ghp = "ghp_" + "a" * 36 # lowercase, because the store case-folds a session id + text = f"the deploy key:\n{PEM}\nand {AKIA} for staging" + tool = {"type": "tool_use", "id": "t", "name": ghp, "input": {}} + write(src, [user("u1", text), assistant("a2", [tool])]) + store.capture(src, "claude-code", ghp, home=home) + address = f"claude-code/{ghp}/g00@0" + code, out, _ = show(home, capsys, address, "-C", "1") + assert code == 0 and PEM_BODY in out and AKIA in out and out.count(ghp) == 3, out + code, out, err = show(home, capsys, address, "-C", "1", "--redact") + assert (code, err) == (0, "") + undated = "undated".ljust(17) + assert out.splitlines() == [ + f"> claude-code/ghp_…[40bytes]/g00@0 {undated} user", + " text", + " the deploy key:", + f" ----…[{len(PEM)} bytes]", + " and AKIA…[20 bytes] for staging", + f"- claude-code/ghp_…[40bytes]/g00@{size(user('u1', text))} {undated} assistant", + " tool_use ghp_…[40 bytes]", + ], out + assert ghp[4:20] not in out and AKIA[4:10] not in out and PEM_BODY[:8] not in out, out + + +def test_show_redact_joins_a_key_header_a_line_break_split(home, src, capsys): + """`-----BEGIN RSA\\nPRIVATE KEY-----` is no header until the line is + flattened. recall flattens a turn before it masks; show keeps the breaks, so + its mask flattens a copy itself and masks what that finds in the text as + written, leaving the lines around the key as they were. [E7b L3-F1]""" + split = PEM.replace("RSA PRIVATE", "RSA\nPRIVATE", 1) + text = f"first words\n\n {split}\nlast words" + assert redact.mask(text) == text, "premise: nothing matches with the break in" + write(src, [user("u1", text)]) + store.capture(src, "claude-code", "sess", home=home) + code, out, _ = show(home, capsys, "claude-code/sess/g00@0", "--redact") + assert code == 0 + assert out.splitlines()[2:] == [ + " first words", + " ", + f" ----…[{len(split)} bytes]", + " last words", + ], out + + +def test_show_redact_prints_a_turn_that_holds_a_lone_surrogate(home, src, capsys): + """Half an emoji is what `JSON.stringify` writes for a tool result cut + mid-pair, `\\ud83d`, and the parse keeps it as a lone surrogate. `show` + printed it as `?`. `show --redact` printed the headers and then stopped, + exit 2, because the mask encoded with `surrogateescape`, which round-trips + only U+DC80-U+DCFF. At the address and in a `-C` neighbour alike, and the + key beside it is still masked.""" + result = {"type": "tool_result", "tool_use_id": "t", "content": f"build ok \ud83d {AKIA}"} + first, second = user("u1", "the first turn"), user("u2", "") + second["message"]["content"] = [result] + write(src, [first, second]) + store.capture(src, "claude-code", "sess", home=home) + at = f"claude-code/sess/g00@{size(first)}" + code, out, err = show(home, capsys, at) + assert (code, err) == (0, "") and f" build ok ? {AKIA}\n" in out, out + for argv in ([at], ["-C", "1", "claude-code/sess/g00@0"]): + code, out, err = show(home, capsys, *argv, "--redact") + assert (code, err) == (0, ""), (argv, err) + assert " build ok " in out and out.endswith(" AKIA…[20 bytes]\n"), (argv, out) diff --git a/tests/test_store.py b/tests/test_store.py index da7a0d8..0e4c4aa 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -577,6 +577,27 @@ def test_cli_capture_records_the_compaction_boundary_the_adapter_found(tmp_path, assert main(["--home", home, "verify"]) == 0 +def test_cli_capture_says_how_much_in_units_and_where_in_bytes(tmp_path, capsys): + """`+` is an amount, so a person gets units; `size=` is a coordinate, so it stays bytes. + + `size=` is the end of `[0, size)`, the range `verify` and every `@offset` + are stated in. The second capture is there so the two numbers differ. + """ + from gitmemory.__main__ import main + + home, src = str(tmp_path / "h"), str(tmp_path / "s.jsonl") + os.mkdir(home) + base = transcript(src, 20) + assert main(["--home", home, "capture", src, "--session-id", "sess"]) == 0 + end = transcript(src, 20, start=20) + assert (base, end) == (1500, 3020) + capsys.readouterr() + + assert main(["--home", home, "capture", src, "--session-id", "sess"]) == 0 + out = capsys.readouterr().out + assert out.endswith(" +1.5 kB (1520 B) size=3020 g00\n"), out + + def test_cli_verify_exits_nonzero_on_a_hole(tmp_path): from gitmemory.__main__ import main @@ -1994,7 +2015,8 @@ def test_a_credential_in_an_exception_message_is_masked_on_the_way_out(home, cap An `OSError` carries the path it failed on. Nobody asked to see that string — it is a diagnostic — and `recall`, which prints transcript content because - that is what it is for, is deliberately not masked. Only exception text is. + that is what it is for, is not masked unless the owner asks (`--redact`). + Exception text always is. """ from gitmemory.__main__ import main @@ -2649,7 +2671,7 @@ def test_a_manifest_whose_adopted_is_not_a_list_is_refused(home, src): def test_cli_capture_says_when_it_only_found_the_bytes(tmp_path, capsys): - """A capture that reclaims an orphan reports `+0B` and is otherwise silent. + """A capture that reclaims an orphan reports `+0 B` and is otherwise silent. The manifest is where the fact lives for ever, but a person running `gitmemory capture` after a crash is not reading manifests — and the @@ -2674,7 +2696,7 @@ def test_cli_capture_says_when_it_only_found_the_bytes(tmp_path, capsys): out = capsys.readouterr() assert f"adopted 1 segment(s) found on disk, not copied from the source: {name}" in out.err - assert "+0B" in out.out, "the line has to be beside a result that looks like nothing happened" + assert "+0 B" in out.out, "the line has to be beside a result that looks like nothing happened" # --- E7 fs-F5: the proof tree reached through a link ------------------------- diff --git a/uv.lock b/uv.lock index fd4cdb2..0df6143 100644 --- a/uv.lock +++ b/uv.lock @@ -433,7 +433,7 @@ wheels = [ [[package]] name = "gitmemory" -version = "0.1.0" +version = "0.2.0" source = { editable = "." } [package.optional-dependencies]