From 59865e10ae2b13afe861182b61f05aeb8e5d6b26 Mon Sep 17 00:00:00 2001 From: doronp <3586743+doronp@users.noreply.github.com> Date: Sun, 18 Oct 2026 13:24:26 +0300 Subject: [PATCH 01/13] docs: name --python 3.13 in every install command The README's install line did not install on Ubuntu 24.04, whose own Python is 3.12, and the same line with `--python 3.13` did. pyproject requires >=3.13, but no install command said so. Left to choose, uv can settle on the older interpreter and refuse (it does with Python downloads off), and pipx always installs into the Python it runs under. Every install command with a source argument now names the floor: the README quickstart and [serve] example, the skill's setup step, hook/README.md, the plugin description, and the derive extra's error hint. pipx survives as a README footnote that names a 3.13 interpreter. Where uv fetches Python only when asked, the README says to run `uv python install 3.13` first; where it never does, to put a 3.13 on PATH, because `uv python install` is refused there too. The floor itself is unchanged; DESIGN.md section 1 stays as locked. test_every_documented_install_names_the_python_floor reads requires-python with tomllib and fails on any tracked install line with a source that lacks the floor, spelled `--python X.Y`, `--python=X.Y` or uv's `-p X.Y` (pipx: `pythonX.Y`). A command ends at its code span, a `)` or `#`, the end of a line not continued by a backslash, or the next command on the same line, and the test checks that scanner on those cases before it scans. Prose with no source, such as hook/README.md's "if you installed with `uv tool install`", is not a command and is skipped. The README's test counts move by one, and docs/REPRODUCE.md's expected counts, stale since 1,027, now read what a checkout collects: 1038 offline, 1363 with the corpora, 328 conformance cases. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QPBLmJD2q1PVxdkWx8C16U Signed-off-by: doronp <3586743+doronp@users.noreply.github.com> --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 8 +++ README.md | 19 +++++-- docs/REPRODUCE.md | 6 +-- hook/README.md | 4 +- skills/recall/SKILL.md | 2 +- src/gitmemory/derive.py | 4 +- tests/test_docs.py | 104 +++++++++++++++++++++++++++++++++++++ 8 files changed, 135 insertions(+), 14 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 973fb0c..d16bd30 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "gitmemory", "version": "0.1.0", - "description": "Registers gitmemory's hook shim on PreCompact and SessionEnd, so a session is captured at the compaction boundary instead of at the watcher's next sweep, and adds a recall skill that has the agent search the captured sessions with gitmemory recall and quote what was said, with the address of each line. The hook is only a doorbell, and the skill writes nothing but the search index: nothing is captured unless the gitmemory watcher is installed (uv tool install git+https://github.com/doronp/gitmemory) and running, and without the hook gitmemory is still correct.", + "description": "Registers gitmemory's hook shim on PreCompact and SessionEnd, so a session is captured at the compaction boundary instead of at the watcher's next sweep, and adds a recall skill that has the agent search the captured sessions with gitmemory recall and quote what was said, with the address of each line. The hook is only a doorbell, and the skill writes nothing but the search index: nothing is captured unless the gitmemory watcher is installed (uv tool install --python 3.13 git+https://github.com/doronp/gitmemory) and running, and without the hook gitmemory is still correct.", "author": { "name": "doronp" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 183f521..9e79463 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,14 @@ tell you. - `recall` now lists the user's own words newest first by default, not best match first. Pass `--policy relevance` for the old order. `index.search` is unchanged. +- **Install commands name Python 3.13.** Every documented `uv tool install` + passes `--python 3.13`, so where the machine's own Python is older (Ubuntu + 24.04 ships 3.12) uv builds the tool on 3.13. Left to choose, uv can settle + on the older one and refuse (it does with Python downloads off). pipx is + down to a footnote: it installs into the Python it runs under, so it needs a + 3.13 interpreter. A test reads the floor from `pyproject.toml` and holds + every install line to it. See + [README.md](README.md#quickstart). ## [0.1.0] - 2026-09-28 diff --git a/README.md b/README.md index 9b8141b..bbfe473 100644 --- a/README.md +++ b/README.md @@ -59,9 +59,18 @@ dependencies. Requires Python 3.13+ and git. ```sh -uv tool install git+https://github.com/doronp/gitmemory # or: pipx install git+https://… +uv tool install --python 3.13 git+https://github.com/doronp/gitmemory ``` +`--python 3.13` has uv build the tool on 3.13, fetching one where the machine's +own Python is older (Ubuntu 24.04 ships 3.12); left to choose, uv can settle on +the older one and refuse. Where uv fetches Python only when asked +(`python-downloads = "manual"`), run `uv python install 3.13` first; where it +never does (`"never"`), put a Python 3.13 on `PATH`.[^pipx] + +[^pipx]: pipx installs into the Python it runs under, so it needs a 3.13 + interpreter: `pipx install --python python3.13 git+https://github.com/doronp/gitmemory`. + Tell it what to watch. Watch roots have no default, and the config lives in `$GITMEMORY_HOME/config.toml` (by default `~/.gitmemory`): @@ -107,7 +116,7 @@ agent after compaction, derived key ideas, the dashboard — is in Optional extras: `hybrid` (what the benchmarks' dense and rerank arms need; `recall` does not use them), `derive` (key ideas and the graph), and `serve` (the dashboard). Example: -`uv tool install "gitmemory[serve] @ git+https://github.com/doronp/gitmemory"`. +`uv tool install --python 3.13 "gitmemory[serve] @ git+https://github.com/doronp/gitmemory"`. ## Results @@ -185,13 +194,13 @@ real text, and that is a measurement rather than a suspicion. | What | How it was measured | Result | |---|---|---| | Hook cost in the agent's critical path | Timed against spawning `true` the same way, three runs of 400 | p50 **7.4 – 7.5 ms**, p99 **10.2 – 11.5 ms** | -| The suite | On a fresh checkout, no downloads | **1,037 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | +| The suite | On a fresh checkout, no downloads | **1,038 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | | Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **606** negative controls | The two suite counts do not add up, and should not. Switching the corpora on -collects 1362, not 1365. Three conformance cases fill parametrisations that +collects 1363, not 1366. Three conformance cases fill parametrisations that collect as one empty placeholder each while the corpora are absent, so they -replace three of the 1037 rather than joining them. Every figure on this board +replace three of the 1038 rather than joining them. Every figure on this board is pinned by a test, which is how it stays true. **What is not measured**, and is shown as a row on the dashboard rather than diff --git a/docs/REPRODUCE.md b/docs/REPRODUCE.md index 98d17a4..d569456 100644 --- a/docs/REPRODUCE.md +++ b/docs/REPRODUCE.md @@ -484,11 +484,11 @@ uv run pytest -q --collect-only -p no:cacheprovider | tail -1 uv run pytest -q --collect-only -p no:cacheprovider | grep -c '\.jsonl\]$' ``` -**Expected:** `1027/1028 tests collected (1 deselected)`, then -`1352/1353 tests collected (1 deselected)`, then `328`. The deselected test is +**Expected:** `1038/1039 tests collected (1 deselected)`, then +`1363/1364 tests collected (1 deselected)`, then `328`. The deselected test is the LongMemEval corpus test, which is opt-in: `uv run pytest -q -m corpus` after `bench/fetch_longmemeval.sh`. The README's -paragraph on why 1,027 + 328 is 1,352 and not 1,355 is the arithmetic of these +paragraph on why 1,038 + 328 is 1,363 and not 1,366 is the arithmetic of these three numbers. All three were run while writing this guide. `tests/test_docs.py` holds the README to the first and third numbers, so a diff --git a/hook/README.md b/hook/README.md index 4d63361..1af85e2 100644 --- a/hook/README.md +++ b/hook/README.md @@ -93,8 +93,8 @@ The plugin installs the hook and one skill, `/gitmemory:recall`, which has the agent run `gitmemory recall` and quote what it returns ([USAGE.md](../docs/USAGE.md#put-it-back-in-front-of-the-agent)). Neither captures anything. **Nothing is captured until the watcher is installed and -running** (`uv tool install git+https://github.com/doronp/gitmemory`, a watch -root in `config.toml`, then `gitmemory watch`, as in the +running** (`uv tool install --python 3.13 git+https://github.com/doronp/gitmemory`, +a watch root in `config.toml`, then `gitmemory watch`, as in the [Quickstart](../README.md#quickstart)), and gitmemory is correct without the plugin, as it is without the hook. The hook and the skill run with Claude Code's environment, so a `GITMEMORY_HOME` you set for the watcher has to be set diff --git a/skills/recall/SKILL.md b/skills/recall/SKILL.md index f35d1f7..14e1461 100644 --- a/skills/recall/SKILL.md +++ b/skills/recall/SKILL.md @@ -82,7 +82,7 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. - The shell cannot find `gitmemory`, or `index` prints `not a store: `, or it reports `0 generation(s)`: nothing has been captured there. Tell the user how it gets set up, and stop. Do none of it yourself: - 1. `uv tool install git+https://github.com/doronp/gitmemory` + 1. `uv tool install --python 3.13 git+https://github.com/doronp/gitmemory` 2. a `[[watch]]` table in `config.toml` in the store (`$GITMEMORY_HOME`, by default `~/.gitmemory`) naming the agent and where its transcripts live; for Claude Code, `agent = "claude-code"` and diff --git a/src/gitmemory/derive.py b/src/gitmemory/derive.py index 0a39123..173eca1 100644 --- a/src/gitmemory/derive.py +++ b/src/gitmemory/derive.py @@ -139,8 +139,8 @@ def _sumy(): from sumy.utils import get_stop_words except ImportError as exc: # pragma: no cover - exercised by hand, not in CI raise RuntimeError( - "derivation needs the `derive` extra: " - 'uv tool install "gitmemory[derive] @ git+https://github.com/doronp/gitmemory"' + "derivation needs the `derive` extra: uv tool install --python 3.13 " + '"gitmemory[derive] @ git+https://github.com/doronp/gitmemory"' " (from a clone: uv sync --extra derive)" ) from exc diff --git a/tests/test_docs.py b/tests/test_docs.py index c2e9c26..d81eb3c 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -16,6 +16,8 @@ import re import subprocess import sys +import tomllib +from itertools import pairwise import pytest from test_claude_code import CC_FIXTURES, _corpus @@ -277,6 +279,108 @@ def test_the_design_document_cites_nothing_in_tmp(): assert "/tmp/" not in line, f"{rel}:{n} cites a scratch path: {line.strip()}" +_INSTALLERS = r"uv tool install|pipx install" +# The tool, then its words up to the end of the code span, a `)`, a `#`, the end +# of the line, or the next tool, so two commands on one line are two commands. +_INSTALL = re.compile(rf"\b({_INSTALLERS})\b([^`)#\n]*?)(?=\b(?:{_INSTALLERS})\b|[`)#]|$)") + + +def _installs(text: str): + """(line number, tool, its words, as written) for each install command in + `text`. A line that ends in a backslash goes on into the next, as in sh.""" + lines, n = text.splitlines(), 0 + while n < len(lines): + start, line = n + 1, lines[n] + while line.endswith("\\") and n + 1 < len(lines): + n += 1 + line = line[:-1] + " " + lines[n] + n += 1 + for m in _INSTALL.finditer(line): + yield start, m.group(1), m.group(2).split(), " ".join(m.group(0).split()) + + +def _names_python(tool: str, words: list[str], want: str) -> bool: + """`--python ` or `--python=`, or uv's own `-p `.""" + flags = ("--python", "-p") if tool == "uv tool install" else ("--python",) + words = [w.strip("\"'") for w in words] + return f"--python={want}" in words or any( + a in flags and b == want for a, b in pairwise(words) + ) + + +def test_every_documented_install_names_the_python_floor(): + """The README's install line did not install on Ubuntu 24.04; the same line + with `--python 3.13` did. + + pyproject says `>=3.13`, and the README said so one line above a command + that did not. Left to choose, uv can settle on the machine's own Python + (3.12 there) and refuse the install, which is what it does with Python + downloads off (measured); named, it builds the tool on 3.13, fetching one + if it may, or says to install one. pipx installs into the Python it runs + under, so its line names an interpreter of that version. The floor is read + from pyproject, so the commands cannot drift from it a second time. + + A command is the tool and what follows it, to the end of its code span, a + `)` or a `#`, the end of the line (a trailing backslash carries it on to the + next), or the next command. It names the floor as `--python X.Y`, + `--python=X.Y` or uv's `-p X.Y`; pipx's value is `pythonX.Y`. Prose that + names the tool with nothing after it ("if you installed with + `uv tool install`,") has no source, so it is not one. Prose with words after + the tool and no code span around it reads as a command and fails, so put the + tool in a code span or write the whole command. `tests/` is not scanned, + because its strings are fixtures and this one names the tools; nor are the + records, which quote what was run at the time. [install feedback 1] + """ + with open(os.path.join(ROOT, "pyproject.toml"), "rb") as fh: + spec = tomllib.load(fh)["project"]["requires-python"] + floor = re.fullmatch(r">=(\d+\.\d+)", spec) + assert floor, f"requires-python is {spec!r}; this test knows only a bare lower bound" + v = floor.group(1) + want = {"uv tool install": v, "pipx install": f"python{v}"} + + # The scanner first, on spellings a later line may use. [install review 2, 4] + probe = ( + f"(uv tool install --python {v} git+x, or pipx install git+x)\n" + f"`uv tool install -p {v} git+x` and `uv tool install --python={v} git+x`\n" + f"uv tool install \\\n --python {v} git+x\n" + f"`pipx install --python python{v} git+x` `pipx install -p python{v} git+x`\n" + "if you installed with `uv tool install`,\n" + ) + found = [(n, t, _names_python(t, w, want[t])) for n, t, w, _ in _installs(probe) if w] + assert found == [ + (1, "uv tool install", True), + (1, "pipx install", False), + (2, "uv tool install", True), + (2, "uv tool install", True), + (3, "uv tool install", True), + (5, "pipx install", True), + (5, "pipx install", False), + ], found + + records = ("tests/", "docs/reviews/", "docs/tasks/", "docs/DESIGN-v0.md") + tracked = subprocess.run( + ["git", "ls-files", "-z"], cwd=ROOT, capture_output=True, text=True, check=True + ).stdout.split("\0") + seen, wrong = set(), [] + for rel in filter(None, tracked): + if rel.startswith(records): + continue + try: + text = _read(rel) + except (UnicodeDecodeError, FileNotFoundError, IsADirectoryError): + continue # a binary or a submodule; no command in it + for n, tool, words, said in _installs(text): + if not words: + continue + seen.add(rel) + if not _names_python(tool, words, want[tool]): + wrong.append(f"{rel}:{n}: `{said}` lacks --python {want[tool]}") + # The floor, because a scan that matches nothing passes loudest. + named = {"README.md", "hook/README.md", "skills/recall/SKILL.md", ".claude-plugin/plugin.json"} + assert named <= seen, named - seen + assert not wrong, f"requires-python is {spec}:\n" + "\n".join(wrong) + + def test_no_tracked_file_contains_an_invisible_character(): """A character you cannot see is a character nobody can review. From 56327cfb1e15a542ead20a30f0e5dd55ae372f4c Mon Sep 17 00:00:00 2001 From: doronp <3586743+doronp@users.noreply.github.com> Date: Sun, 18 Oct 2026 13:35:23 +0300 Subject: [PATCH 02/13] feat: gitmemory --version, and the recall skill recognises an older CLI The plugin and the CLI are installed separately, and nothing makes them one release: Claude Code keeps the plugin at the commit its marketplace was cloned at until its version changes, while `uv tool install` takes whatever commit it is given. A skew has shipped already, silently: v0.1.0's recall prints no date column and ranks by relevance, the skill decides which of two lines is current by date, and both say 0.1.0. - `gitmemory --version` prints `gitmemory <__version__>` through argparse's version action (the constant had no reader). Every CLI before it exits 2 with "the following arguments are required: cmd", measured under Python 3.12, 3.13 and 3.14. - The recall skill names the release it is written for (0.1.0, today's pyproject version), runs --version once per session and says when the CLI is older. Its first "When it fails" bullet reads exit 2 with `invalid choice`, or `unrecognized arguments` naming a flag, as an older CLI: say which subcommand or flag is missing and never retry without it, so a later safety flag cannot degrade into its absence. `unrecognized arguments` naming only plain words is a topic left unquoted, not an older CLI: quote it and search again. --version joins the read-only allow-list, and its call joins the echoes: every session that uses the skill now records one, and on a query about versions those records outrank the real hits at the default -k 10. - The skill-commands test accepts `gitmemory --version` with no subcommand and looks for the flag in the top-level help. A new test pins the release the skill names to pyproject and __version__, the --version output, the argparse phrases the skill quotes, each got from the CLI, the unquoted-topic case, and the --version echo. - hook/README.md says how to pin both to one tag, with the install line naming --python 3.13 like every other. CONTRIBUTING.md gets the bump rule, `uv lock` included: uv.lock records the project version, and `uv lock --check` fails after a pyproject-only bump. It also says to move CHANGELOG's [Unreleased] and to tag the bump `v`, which the pin advice names. - The new test moves the README's counts and docs/REPRODUCE.md's expected counts by one, to what a checkout now collects: 1039 offline, 1364 with the corpora, 328 conformance cases. No version bump here; the release commit makes it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QPBLmJD2q1PVxdkWx8C16U Signed-off-by: doronp <3586743+doronp@users.noreply.github.com> --- CHANGELOG.md | 5 +++ CONTRIBUTING.md | 16 +++++++++ README.md | 6 ++-- docs/REPRODUCE.md | 6 ++-- docs/USAGE.md | 5 ++- hook/README.md | 13 +++++++ skills/recall/SKILL.md | 29 ++++++++++++++- src/gitmemory/__main__.py | 6 +++- tests/test_docs.py | 76 ++++++++++++++++++++++++++++++++++----- 9 files changed, 144 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e79463..874d1c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,11 @@ tell you. version is still 0.1.0, so a copy installed from 0.1.0 does not get the skill from `claude plugin update`; reinstall it, or wait for the next release ([hook/README.md](hook/README.md#as-a-claude-code-plugin)). +- **`gitmemory --version`** prints the installed release. The recall skill + runs it first and tells you when the CLI is older than the plugin; when the + CLI lacks a subcommand or a flag the skill needs, the skill stops and says + which instead of retrying without it. See + [hook/README.md](hook/README.md#as-a-claude-code-plugin). ### Changed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8ab1c87..c601e40 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -81,6 +81,22 @@ is supported when `tests/conformance.py::check_adapter` passes on a public corpus of that agent's transcripts. Say in the PR where the corpus comes from and what its licence is. +## Bumping the version + +Claude Code keeps an installed plugin's copy until the plugin's version +changes, so a change under `skills/`, `hooks/` or `.claude-plugin/`, or to +`hook/gitmemory-hook.sh`, reaches nobody who installed the plugin until the +version is bumped. A bump moves five places in one commit: `version` in +`pyproject.toml`, `__version__` in `src/gitmemory/__init__.py`, `version` in +`.claude-plugin/plugin.json`, the release `skills/recall/SKILL.md` says it is +written for, and `uv.lock`, which records the project's version: run `uv lock` +after editing `pyproject.toml`, or CI's `uv sync --locked` fails. The tests pin +the first four equal. The same commit turns CHANGELOG.md's `[Unreleased]` into +the release's heading, with its date and a compare link from the previous tag, +and starts `[Unreleased]`'s link at the new one. Once the commit is on main, +tag it `v` and push the tag: hook/README.md tells users to pin both +installs to that tag, and a pin to a tag that does not exist installs nothing. + ## Pull requests - Keep each PR to one change. Explain the *why* in the description; the diff diff --git a/README.md b/README.md index bbfe473..e76853f 100644 --- a/README.md +++ b/README.md @@ -194,13 +194,13 @@ real text, and that is a measurement rather than a suspicion. | What | How it was measured | Result | |---|---|---| | Hook cost in the agent's critical path | Timed against spawning `true` the same way, three runs of 400 | p50 **7.4 – 7.5 ms**, p99 **10.2 – 11.5 ms** | -| The suite | On a fresh checkout, no downloads | **1,038 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | +| The suite | On a fresh checkout, no downloads | **1,039 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | | Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **606** negative controls | The two suite counts do not add up, and should not. Switching the corpora on -collects 1363, not 1366. Three conformance cases fill parametrisations that +collects 1364, not 1367. Three conformance cases fill parametrisations that collect as one empty placeholder each while the corpora are absent, so they -replace three of the 1038 rather than joining them. Every figure on this board +replace three of the 1039 rather than joining them. Every figure on this board is pinned by a test, which is how it stays true. **What is not measured**, and is shown as a row on the dashboard rather than diff --git a/docs/REPRODUCE.md b/docs/REPRODUCE.md index d569456..82c607b 100644 --- a/docs/REPRODUCE.md +++ b/docs/REPRODUCE.md @@ -484,11 +484,11 @@ uv run pytest -q --collect-only -p no:cacheprovider | tail -1 uv run pytest -q --collect-only -p no:cacheprovider | grep -c '\.jsonl\]$' ``` -**Expected:** `1038/1039 tests collected (1 deselected)`, then -`1363/1364 tests collected (1 deselected)`, then `328`. The deselected test is +**Expected:** `1039/1040 tests collected (1 deselected)`, then +`1364/1365 tests collected (1 deselected)`, then `328`. The deselected test is the LongMemEval corpus test, which is opt-in: `uv run pytest -q -m corpus` after `bench/fetch_longmemeval.sh`. The README's -paragraph on why 1,038 + 328 is 1,363 and not 1,366 is the arithmetic of these +paragraph on why 1,039 + 328 is 1,364 and not 1,367 is the arithmetic of these three numbers. All three were run while writing this guide. `tests/test_docs.py` holds the README to the first and third numbers, so a diff --git a/docs/USAGE.md b/docs/USAGE.md index be113d2..18643eb 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -171,7 +171,10 @@ decision or a rule from before a compaction or from an earlier session, or that you run with a topic: `/gitmemory:recall retry loop`. It runs `recall`, and `index` when there is no index yet or the one there is older than what you are asking about. It quotes each line with its address and says when a hit comes -from another session. It writes nothing but the index. +from another session. It writes nothing but the index. It runs +`gitmemory --version` first and says when the CLI is older than the plugin; +the [hook's README](../hook/README.md#as-a-claude-code-plugin) has how to hold +the two at one release. ## Derive key ideas and decisions diff --git a/hook/README.md b/hook/README.md index 1af85e2..e857c54 100644 --- a/hook/README.md +++ b/hook/README.md @@ -109,6 +109,19 @@ that it is already at 0.1.0. The next release, which bumps the version, brings the skill; until then, `claude plugin uninstall gitmemory@gitmemory` and then `claude plugin install gitmemory@gitmemory` fetch the current copy. +The plugin and the CLI are installed separately, and nothing makes them one +release: `uv tool install` takes the commit you name, or the head of main, and +the plugin is the commit its marketplace was cloned at, kept until its version +changes. The recall skill runs `gitmemory --version` first and tells you when +the CLI is older than it, and when the CLI lacks a subcommand or a flag the +skill needs, it stops and says which rather than retrying without it. To hold +both at one release, pin both to its tag, `v`: +`uv tool install --python 3.13 git+https://github.com/doronp/gitmemory@v` +and `/plugin marketplace add doronp/gitmemory#v` (a marketplace pins a +branch or a tag, not a bare commit). To move the CLI forward, run the +`uv tool install` line again without the `@` part; `uv tool upgrade gitmemory` +leaves a pinned install where it is. + This keeps the rule at the top of [Install](#install): gitmemory never edits another program's configuration. You opt in with `/plugin`, and Claude Code records the change in its own settings. `/plugin uninstall gitmemory@gitmemory` diff --git a/skills/recall/SKILL.md b/skills/recall/SKILL.md index 14e1461..0a67c15 100644 --- a/skills/recall/SKILL.md +++ b/skills/recall/SKILL.md @@ -22,6 +22,23 @@ below means that directory: the path the user gave, or what `echo "${GITMEMORY_HOME:-$HOME/.gitmemory}"` prints. Do not type `~/.gitmemory` in its place from memory. +## The CLI's version + +The plugin and the CLI are installed separately, so they can come from +different releases. This skill is written for gitmemory 0.1.0. Once per +session, before the first search, run: + +```sh +gitmemory --version +``` + +If it prints that release or a later one, go on. If it prints an earlier one, +or exits 2 with `the following arguments are required: cmd` (a CLI from before +`--version` existed), the CLI is older than this skill: tell the user so in +one line, with the fix under [When it fails](#when-it-fails), and go on. If +what `recall` then prints is not laid out as below, say so, and do not judge +which of two lines is the current decision from their order. + ## Search The topic is the skill's argument if it was given one, otherwise the thing the @@ -66,12 +83,21 @@ recalls match the same words. Skip these hits; they are not sources: - `/gitmemory:recall` and its arguments, however the transcript wraps them; - a `gitmemory recall ...` tool call and its result, lines that start with a score, `N generation(s)` or `no matches`; +- a `gitmemory --version` tool call and the one line it printed; - an earlier answer to the same question, and the lines it quoted. If they crowd out the rest, search again with `-k 30`. Quote the original turn. ## When it fails +- A `gitmemory` command exits 2, what it prints starts `usage: gitmemory`, and + its last line says `invalid choice`, or `unrecognized arguments` naming a + flag (a word starting with `-`): the CLI is older than this skill and lacks + that subcommand or flag. Do not run it again without the flag or under + another name. Tell the user which one is missing, and that running step 1 + below again updates the CLI, a pinned install included. If + `unrecognized arguments` names only plain words, the CLI is not the problem: + the topic went in unquoted. Put it in one pair of quotes and search again. - ``no index at ; run `gitmemory index` first``, exit 2: run `gitmemory index`, then search again. `index` rebuilds the index from the store in full. It is safe to rerun, and the index is never the source of @@ -115,7 +141,8 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. ## Read-only -Run `gitmemory recall` and `gitmemory index`, and read files under ``. +Run `gitmemory --version`, `gitmemory recall` and `gitmemory index`, and read +files under ``. `index` writes only the derived index under `/index/`. Do not run `gitmemory capture`, `gitmemory watch`, `gitmemory derive`, `gitmemory push` or `gitmemory dashboard`, and never `gitmemory dashboard --expose`. Do not diff --git a/src/gitmemory/__main__.py b/src/gitmemory/__main__.py index f036882..b733243 100644 --- a/src/gitmemory/__main__.py +++ b/src/gitmemory/__main__.py @@ -18,7 +18,7 @@ import sqlite3 import sys -from . import daemon, dashboard, derive, gitrepo, index, redact, store +from . import __version__, daemon, dashboard, derive, gitrepo, index, redact, store from .records import safe_text @@ -342,6 +342,10 @@ def _recall(args) -> int: def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(prog="gitmemory") + # The plugin and the CLI are installed apart and can be releases apart, so + # the recall skill asks this before it trusts `recall`'s columns. argparse + # prints it past the escaping `print` above: fine for a source constant. + ap.add_argument("--version", action="version", version=f"%(prog)s {__version__}") ap.add_argument( "--home", default=None, help="store root (default $GITMEMORY_HOME or ~/.gitmemory)" ) diff --git a/tests/test_docs.py b/tests/test_docs.py index d81eb3c..aaca56b 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -553,11 +553,12 @@ def test_the_plugin_recall_skill_runs_only_commands_the_cli_has(tmp_path, capsys waiting, and it reads as gitmemory being broken. So every `gitmemory ...` in the skill's code is asked for its own `--help`, which argparse answers with exit 0 only for a subcommand that exists, and every flag it names is - looked for in that help: `--home` before the subcommand in the top-level - help, the rest in the subcommand's. The commands it forbids are checked the - same way, because a prohibition on a command that no longer exists guards - nothing. The messages it tells the agent to branch on are the ones the CLI - prints, got by running it on a path that is no store and on an empty one. + looked for in that help: `--home` before the subcommand, and `--version`, + which comes with none, in the top-level help, the rest in the subcommand's. + The commands it forbids are checked the same way, because a prohibition on + a command that no longer exists guards nothing. The messages it tells the + agent to branch on are the ones the CLI prints, got by running it on a path + that is no store and on an empty one. `skills/` holds this one skill and neither manifest adds a skills path; its name is the one the three documents that mention it tell you to type. """ @@ -590,10 +591,12 @@ def usage(*argv: str) -> str: for m in re.finditer(r"(? tuple[int, str, str]: assert code == 0 and out.startswith("0 generation(s)") and "`0 generation(s)`" in flat code, _, err = run(empty, "recall", "x") assert code == 0 and err == "no matches\n" and "`no matches` on stderr, exit 0" in flat + + +def test_the_recall_skill_names_the_release_it_was_written_for(capsys): + """The plugin and the CLI are installed separately, and nothing makes them + one release. + + The test above runs the skill against the CLI in this checkout, which says + nothing about an installed copy of each. A skew has shipped already: at + v0.1.0 `recall` printed no date and ranked by relevance, the skill reads + the date to tell which of two lines is current, both said 0.1.0, and + nothing errored. So the skill names its release, pinned here to + `pyproject.toml` and `__version__` (test_hook.py pins `plugin.json`), and + asks `gitmemory --version` once per session. A CLI from before the flag + answers with the first error below (measured on c352a40 under 3.12, 3.13 + and 3.14); one that lacks a subcommand or a flag answers with one of the + other two. The skill branches on those words, so each is got from the CLI + and must be in the skill as the CLI prints it. A topic left unquoted is + rejected in the same words as a missing flag, naming a plain word, so the + skill tells the two apart by the leading `-`. And the store captures the + `--version` call of every session that uses the skill, which outranks real + hits on "version" unless the skill lists it among its echoes. + """ + from gitmemory import __main__ as cli + from gitmemory import __version__ + + with open(os.path.join(ROOT, "pyproject.toml"), "rb") as fh: + release = tomllib.load(fh)["project"]["version"] + assert __version__ == release + skill = " ".join(_read("skills/recall/SKILL.md").split()) + + with pytest.raises(SystemExit) as done: + cli.main(["--version"]) + assert done.value.code == 0 and capsys.readouterr().out == f"gitmemory {release}\n" + assert f"written for gitmemory {release}." in skill, release + + for argv, said in ( + ([], "the following arguments are required: cmd"), + (["no-such-subcommand"], "invalid choice"), + (["recall", "--no-such-flag", "x"], "unrecognized arguments"), + ): + with pytest.raises(SystemExit) as done: + cli.main(argv) + err = capsys.readouterr().err + assert done.value.code == 2 and err.startswith("usage: gitmemory "), err + assert said in err.splitlines()[-1] and f"`{said}`" in skill, err + assert "`usage: gitmemory`" in skill + + with pytest.raises(SystemExit) as done: + cli.main(["recall", "retry", "loop"]) + last = capsys.readouterr().err.splitlines()[-1] + assert done.value.code == 2 and last.endswith("unrecognized arguments: loop"), last + assert "a word starting with `-`" in skill and "went in unquoted" in skill + + echoes = skill.split("## Echoes ", 1)[1].split(" ## ", 1)[0] + assert "`gitmemory --version`" in echoes, echoes From 656136cb696687252510ed0a225afc965ffe26c2 Mon Sep 17 00:00:00 2001 From: doronp <3586743+doronp@users.noreply.github.com> Date: Sun, 18 Oct 2026 13:36:53 +0300 Subject: [PATCH 03/13] fix: latest-user stops taking a subagent's prompt for the user's words A subagent's first turn is user-role text that the parent agent wrote: its dispatch prompt. The default policy counted every user/text block as the user's own words, so a dispatch prompt that was newer than the user's rule, and matched it, led recall ahead of that rule. That is the forgetful-agent failure latest-user exists to prevent, arriving through a subagent. The index already kept the per-turn isSidechain flag in `turns`; neither Hit nor search() could see it. - Hit.is_sidechain, read in search() by a primary-key lookup on turns. No schema bump and no reindex, and search() order is unchanged, so no benchmark number can move. Timed within noise on 120k turns. - latest_user no longer admits a sidechain turn as the user's words. derive._prose still does; changing it would move the secondary-set numbers, and the docstring says so. - recall prints subagent:/ for such a turn, in the same column, so split()[3] is still role/kind and main-session lines are unchanged. The mark is per turn, so old-format sidechain lines inside a main file are labelled too. USAGE.md gives a main_only policy that drops them at read time. - docs/watching.md states the pattern's glob semantics and adds a main-sessions-only recipe, pattern = "*/*.jsonl", with its costs: those turns are never captured, are lost once Claude Code cleans up old transcripts, and drop out of dashboard spend. The brief survives in the parent, and so does the report unless a background agent's or workflow's session ended first; a workflow's journal is left out. The recipe must replace the default table, since a second table on the same root still finds every subagent. A doc test runs the documented block and pins both claims. No exclude key: the existing pattern key already says this. - Three mutation rows, so 606 becomes 609 negative controls. On top of the install and --version commits, the suite is 1,043 tests offline (1,368 with the corpora). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QPBLmJD2q1PVxdkWx8C16U Signed-off-by: doronp <3586743+doronp@users.noreply.github.com> --- CHANGELOG.md | 17 ++++++++ README.md | 8 ++-- docs/REPRODUCE.md | 8 ++-- docs/USAGE.md | 26 ++++++++++-- docs/watching.md | 48 +++++++++++++++++++++- skills/recall/SKILL.md | 9 ++++- src/gitmemory/__main__.py | 7 +++- src/gitmemory/index.py | 34 +++++++++++++--- tests/mutate_index.py | 29 +++++++++++++- tests/test_docs.py | 58 +++++++++++++++++++++++++++ tests/test_index.py | 83 +++++++++++++++++++++++++++++++++++++++ 11 files changed, 303 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 874d1c8..91b343d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,15 @@ tell you. CLI lacks a subcommand or a flag the skill needs, the skill stops and says which instead of retrying without it. See [hook/README.md](hook/README.md#as-a-claude-code-plugin). +- **`recall` marks a subagent's turns.** A turn a subagent or workflow agent + wrote prints its role as `subagent:`, as in `subagent:user/text`, in + the same column, so a line has as many fields as before. `Hit.is_sidechain` + carries the mark, read per turn from the transcript's `isSidechain`. See + [docs/USAGE.md](docs/USAGE.md#find-something). +- **Main sessions only.** [docs/watching.md](docs/watching.md#main-sessions-only-claude-code) + gives a Claude Code watch pattern, `pattern = "*/*.jsonl"`, that leaves + subagent and workflow transcripts out, and what that costs: those turns are + never captured, and the dashboard's spend leaves out what they cost. ### Changed @@ -46,6 +55,14 @@ tell you. every install line to it. See [README.md](README.md#quickstart). +### Fixed + +- **`latest-user` no longer takes a subagent's instructions for the user's + words.** A subagent's first turn is user-role text that the parent agent + wrote, its dispatch prompt, and while it was newer than the user's rule it + led the default ahead of that rule. It now ranks with the agent's turns. + `index.search` order is unchanged. + ## [0.1.0] - 2026-09-28 The first public release. What it contains: diff --git a/README.md b/README.md index e76853f..d0e0d80 100644 --- a/README.md +++ b/README.md @@ -194,13 +194,13 @@ real text, and that is a measurement rather than a suspicion. | What | How it was measured | Result | |---|---|---| | Hook cost in the agent's critical path | Timed against spawning `true` the same way, three runs of 400 | p50 **7.4 – 7.5 ms**, p99 **10.2 – 11.5 ms** | -| The suite | On a fresh checkout, no downloads | **1,039 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | -| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **606** negative controls | +| The suite | On a fresh checkout, no downloads | **1,043 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | +| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **609** negative controls | The two suite counts do not add up, and should not. Switching the corpora on -collects 1364, not 1367. Three conformance cases fill parametrisations that +collects 1368, not 1371. Three conformance cases fill parametrisations that collect as one empty placeholder each while the corpora are absent, so they -replace three of the 1039 rather than joining them. Every figure on this board +replace three of the 1043 rather than joining them. Every figure on this board is pinned by a test, which is how it stays true. **What is not measured**, and is shown as a row on the dashboard rather than diff --git a/docs/REPRODUCE.md b/docs/REPRODUCE.md index 82c607b..c5d148d 100644 --- a/docs/REPRODUCE.md +++ b/docs/REPRODUCE.md @@ -484,11 +484,11 @@ uv run pytest -q --collect-only -p no:cacheprovider | tail -1 uv run pytest -q --collect-only -p no:cacheprovider | grep -c '\.jsonl\]$' ``` -**Expected:** `1039/1040 tests collected (1 deselected)`, then -`1364/1365 tests collected (1 deselected)`, then `328`. The deselected test is +**Expected:** `1043/1044 tests collected (1 deselected)`, then +`1368/1369 tests collected (1 deselected)`, then `328`. The deselected test is the LongMemEval corpus test, which is opt-in: `uv run pytest -q -m corpus` after `bench/fetch_longmemeval.sh`. The README's -paragraph on why 1,039 + 328 is 1,364 and not 1,367 is the arithmetic of these +paragraph on why 1,043 + 328 is 1,368 and not 1,371 is the arithmetic of these three numbers. All three were run while writing this guide. `tests/test_docs.py` holds the README to the first and third numbers, so a @@ -505,7 +505,7 @@ test fail. The count is the number of rows in `tests/mutate_index.py`. uv run python -c "import sys; sys.path.insert(0, 'tests'); from mutate_index import MUTANTS; print(len(MUTANTS))" ``` -**Expected:** `606`. Checked while writing this guide. +**Expected:** `609`. Checked while writing this guide. To run the controls rather than count them: diff --git a/docs/USAGE.md b/docs/USAGE.md index 18643eb..86cc1d3 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -75,7 +75,7 @@ newest first (see [When memories disagree](#when-memories-disagree)): | `2026-09-28T10:03Z` | When the turn was written, in UTC. `undated` if the transcript did not say | | `claude-code//g00` | Agent, session and generation. A new generation starts only when the source file was rewritten | | `@288` | The byte offset of the turn inside that generation's raw bytes | -| `assistant/text` | Role and block kind | +| `assistant/text` | Role and block kind. `subagent:` before them, as in `subagent:user/text` or `subagent:assistant/text`, marks a subagent's or workflow agent's turn, which the transcript marks `isSidechain`: its user text is the instructions your agent gave it, not your words | `-k` sets how many lines come back. @@ -88,7 +88,7 @@ tends to act on. | Policy | Order | Use it when | |---|---|---| -| `latest-user` (default) | The user's own words newest first, then the agent's turns and tool output newest first | The user's later word should win, and nothing else should | +| `latest-user` (default) | The user's own words newest first, then the agent's turns and tool output newest first. A subagent's instructions are the agent's words | The user's later word should win, and nothing else should | | `latest` | Newest first, whoever said it | Any later turn should win, the agent's included | | `relevance` | Best BM25 match first | You want the agent to decide: every line carries its date, so it can see which turn came later | | `module:function` | Whatever your function returns | Anything else: pinned decisions, a cut-off date, a different rule per project | @@ -116,8 +116,9 @@ If two lines disagree, the later date is the current decision; say so. A policy is a function from a list of `Hit`s to a list of `Hit`s. It can reorder and it can drop; it gets nothing that is not already in the store. A -`Hit` has the fields `recall` prints, plus `text` and `ts`, and -`gitmemory.index.when(hit)` reads `ts` as a UTC `datetime`. +`Hit` has the fields `recall` prints, plus `text`, `ts` and `is_sidechain` +(true for a subagent's turn), and `gitmemory.index.when(hit)` reads `ts` as a +UTC `datetime`. A validity window, where everything before a date you name is out of date, is one such rule: @@ -140,6 +141,23 @@ def since_reversal(hits): $ PYTHONPATH=. gitmemory recall "upload retries" --policy myrules:since_reversal ``` +Leaving subagents' turns out when you read, rather than never capturing them +([watching.md](watching.md#main-sessions-only-claude-code)), is another rule +for the same file: + +```python +def main_only(hits): + """Drop every subagent's turn, then order as the default does.""" + return latest_user([h for h in hits if not h.is_sidechain]) +``` + +A policy that drops hits prints fewer than `-k` lines, because it sees only the +top `-k`. Raise `-k` to make up for what it drops: + +```console +$ PYTHONPATH=. gitmemory recall "upload retries" -k 30 --policy myrules:main_only +``` + The module is imported and runs with your permissions, so name only code you would run anyway. A name that does not resolve is an error, not a silent fallback to the default. diff --git a/docs/watching.md b/docs/watching.md index d91ce5c..c9737d2 100644 --- a/docs/watching.md +++ b/docs/watching.md @@ -26,13 +26,59 @@ roots = ["~/.claude/projects"] |---|---|---| | `agent` | yes | Which adapter finds compaction boundaries. See below. | | `roots` | yes | Directories to tail. `~` is expanded; symlinks are resolved. | -| `pattern` | no | Glob under each root. Defaults to `**/*.jsonl`. | +| `pattern` | no | Glob under each root, with `pathlib` semantics: `*` never crosses a `/`, and `**` spans any number of directories. Defaults to `**/*.jsonl`. | Repeat the table for each agent you run. Absent, unreadable, or malformed config means the watcher watches nothing — and says so on every pass, naming the file and the reason, because "nothing configured" and "misconfigured" must not look alike. +### Main sessions only (Claude Code) + +The default pattern takes every transcript under the root, and under +`~/.claude/projects` most of them are not a conversation you had. Claude Code +writes each subagent to `//subagents/agent-.jsonl`, and +each workflow agent, beside the workflow's `journal.jsonl`, under +`//subagents/workflows//`. A pattern one directory deep +takes only the main sessions, `/.jsonl`: + +```toml +[[watch]] +agent = "claude-code" +roots = ["~/.claude/projects"] +pattern = "*/*.jsonl" +``` + +Put the `pattern` line in the `claude-code` table you already have, not in a +second table beside it. The watcher takes the files of every table, so another +table on the same root with the default pattern still finds every subagent. + +A hook doorbell for a subagent's file then captures nothing: the spool only +hurries a file the pattern already found. + +Prefer the default unless the store's size matters more than what this costs. +`recall` marks a subagent's turns `subagent:` (see +[USAGE.md](USAGE.md#find-something)), so they can be told apart without +leaving them out, and a policy can drop them from what it prints +([USAGE.md](USAGE.md#your-own-rule)). What leaving them out costs: + +- **Those turns are never captured.** Once Claude Code cleans up old + transcripts, they are gone. Until then this is reversible: put the default + back, and the next pass captures each subagent file from its first byte. +- **The dashboard's spend undercounts.** It counts only what was captured, and + subagents are where much of the spend goes: one measured session had 78% of + its cache-read tokens in `subagents/**` ([DESIGN.md §2.2](DESIGN.md#22-claude-code-adapter--the-traps-each-a-test-case)). +- **What was sent survives, and usually what came back.** The parent session + keeps the tool call that started a subagent or a workflow, with its + instructions. A subagent's report comes back as that call's result. A + background agent or workflow gets only a launch receipt there; its report + comes later, as a task-notification turn in the parent, or not at all if the + session ended first. What is lost is the subagents' own working turns, and a + workflow's `journal.jsonl`, which holds each of its agents' results. +- Older Claude Code wrote a subagent's turns into the parent's own file, each + line marked. A pattern cannot leave those out; the `subagent:` mark still + labels them. + ### What gets refused, and why A root is skipped, loudly, when it: diff --git a/skills/recall/SKILL.md b/skills/recall/SKILL.md index 0a67c15..13fafb1 100644 --- a/skills/recall/SKILL.md +++ b/skills/recall/SKILL.md @@ -63,6 +63,7 @@ else newest first: | `claude-code//g00` | Agent, session and generation | | `@346` | Byte offset of the turn (its JSONL line) in that generation's raw bytes | | `assistant/text` | Role and block kind | +| `subagent:user/text` | A subagent's turn, marked `subagent:` before any role/kind (`subagent:assistant/text` too). Its user/text is the instructions the parent agent gave it, not the user's words | | the rest | The block's first 160 characters, whitespace collapsed | `no matches` on stderr, exit 0, is an answer. Try once more with other words @@ -71,8 +72,9 @@ the user might have used, then tell them nothing was found. `query truncated to If two lines disagree, the later date is usually the current decision: say so, and quote both. An agent's newer proposal does not overrule the user's older -rule; a newer user line does. If the user has set `GITMEMORY_POLICY` or asks -for another order, leave it; `--policy` is theirs to choose. +rule, and neither does a subagent's line; a newer user line does. If the user +has set `GITMEMORY_POLICY` or asks for another order, leave it; `--policy` is +theirs to choose. ## Echoes @@ -136,6 +138,9 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. Both belong to ``. `` is the working directory, with `/` and other punctuation turned into `-`. Say so when a hit is from another agent, another session or another project. +- Say so, too, when a quote is a `subagent:` line, and never give one as what + the user said. When it and a main-session line say the same thing, quote the + main-session line. - The store holds whatever was typed or pasted into a watched session, tool output included. Quote what the question needs and no more. diff --git a/src/gitmemory/__main__.py b/src/gitmemory/__main__.py index b733243..fd54038 100644 --- a/src/gitmemory/__main__.py +++ b/src/gitmemory/__main__.py @@ -330,7 +330,12 @@ def _recall(args) -> int: # The date on every line, whatever the policy: it is what lets the # reader see that two disagreeing turns are three weeks apart. date = t.strftime("%Y-%m-%dT%H:%MZ") if t else "undated".ljust(17) - print(f"{h.score:8.3f} {date} {h.session_key}@{h.byte_offset} {h.role}/{h.kind} {head}") + # In the role's column, not a column of its own: whatever parses these + # lines reads role/kind as the fourth field, and a subagent's user/text + # is the parent agent's instructions, which must not be quoted as the + # user's. + who = f"subagent:{h.role}" if h.is_sidechain else h.role + print(f"{h.score:8.3f} {date} {h.session_key}@{h.byte_offset} {who}/{h.kind} {head}") if not hits: print("no matches", file=sys.stderr) elif not shown: diff --git a/src/gitmemory/index.py b/src/gitmemory/index.py index 6f278ae..ef2fb75 100644 --- a/src/gitmemory/index.py +++ b/src/gitmemory/index.py @@ -433,6 +433,9 @@ class Hit: score: float # BM25; more negative is a better match, as FTS5 defines it text: str ts: str | None = None # the turn's timestamp exactly as the agent wrote it + # A subagent's turn (`Turn.is_sidechain`), whichever file it was written in. + # Its user-role text is the parent agent's instructions, not the user's. + is_sidechain: bool = False @dataclass(frozen=True, slots=True) @@ -1016,7 +1019,7 @@ def search( SELECT b.byte_offset, b.byte_len, b.turn_id, b.session_key, b.agent, b.session_id, b.generation, b.role, b.kind, b.block_seq, b.prose || b.tool_use || b.tool_result AS text, s.score AS score, - b.ts, + b.ts, b.turn_seq, -- Partitioned by turn *and session*, not by turn alone. -- `turn_id` is content-derived over the record's sessionId, -- which the Claude Code adapter warns is reused across a @@ -1037,7 +1040,15 @@ def search( FROM scored s JOIN blocks b ON b.rowid = s.rid ) SELECT byte_offset, byte_len, turn_id, session_key, agent, - session_id, generation, role, kind, text, score, ts + session_id, generation, role, kind, text, score, ts, + -- Only `turns` knows whose turn it was. A lookup on its primary + -- key, not a join, so the ORDER BY's names stay unambiguous; on + -- 120k turns, against queries that match most of them, it timed + -- within noise of no lookup at all. + COALESCE(( + SELECT t.is_sidechain FROM turns t + WHERE t.session_key = ranked.session_key AND t.seq = ranked.turn_seq + ), 0) AS is_sidechain FROM ranked WHERE rn = 1 -- Ties are broken by position, never by rowid: the answer must not -- depend on the order generations happened to be indexed in. @@ -1060,6 +1071,7 @@ def search( score=r["score"], text=r["text"], ts=r["ts"], + is_sidechain=bool(r["is_sidechain"]), ) for r in rows ) @@ -1131,9 +1143,14 @@ def latest_user(hits: list[Hit]) -> list[Hit]: an earlier session, which is newer than every turn it quotes. A user who reverses themselves still wins here, because their reversal is user text. - "The user's own words" is the admission rule `derive` uses for prose: a - `text` block in the user role that no program injected. A tool result is a - user-role line in Claude Code and is not something the user said. + "The user's own words" is the admission rule `derive` uses for prose, a + `text` block in the user role that no program injected, less a subagent's + turn. A tool result is a user-role line in Claude Code and is not something + the user said, and neither is a subagent's user text: that is the parent + agent's instructions, its dispatch prompt first, which led the default as + the user's word while it was newer than the rule it contradicted. + `derive._prose` still admits a subagent's text; excluding it there too would + move the secondary-set numbers, so that waits for a re-measurement. ponytail: a skill's loaded instructions and a slash command's expansion are user text with no marker, so they pass as the user's words — the same gap @@ -1142,7 +1159,12 @@ def latest_user(hits: list[Hit]) -> list[Hit]: from .derive import _injected # lazy: derive imports this module def typed(h: Hit) -> bool: - return h.role == "user" and h.kind == "text" and not _injected(h.text.strip()) + return ( + h.role == "user" + and h.kind == "text" + and not h.is_sidechain + and not _injected(h.text.strip()) + ) ordered = latest(hits) return [h for h in ordered if typed(h)] + [h for h in ordered if not typed(h)] diff --git a/tests/mutate_index.py b/tests/mutate_index.py index 958e5fb..d157f7b 100644 --- a/tests/mutate_index.py +++ b/tests/mutate_index.py @@ -4597,7 +4597,7 @@ # anchor appear twice and the harness would skip itself. [E7] "the docstring's row count drifts away from the index again", "tests/mutate_index.py", - "a full pass is 6" + "06 mutants", + "a full pass is 6" + "09 mutants", "a full pass is 14" + "6 mutants", "test_the_row_count_in_the_docstring_is_the_row_count", ), @@ -5500,6 +5500,31 @@ " except ZeroDivisionError as e:", "test_a_call_that_fails_every_retry_scores_wrong_and_is_counted", ), + # --- a subagent's dispatch prompt led `latest-user` as the user's word --- + ( + "latest-user takes a subagent's dispatch prompt for the user's word", + "index.py", + " and not h.is_sidechain\n", + "", + "test_a_subagent_prompt_is_not_the_users_word", + ), + ( + "search reads every turn as a main session's", + "index.py", + " COALESCE((\n" + " SELECT t.is_sidechain FROM turns t\n" + " WHERE t.session_key = ranked.session_key AND t.seq = ranked.turn_seq\n" + " ), 0) AS is_sidechain", + " 0 AS is_sidechain", + "test_a_sidechain_line_inside_a_main_file_is_marked", + ), + ( + "recall prints a subagent's turn with the user's role", + "__main__.py", + ' who = f"subagent:{h.role}" if h.is_sidechain else h.role', + " who = h.role", + "test_recall_marks_a_subagent_turn", + ), ] @@ -5652,7 +5677,7 @@ def _select(test: str) -> list[str]: def main() -> int: """Run every mutant, or only those whose name contains an argument. - The filter exists because a full pass is 606 mutants × two suite runs — + The filter exists because a full pass is 609 mutants × two suite runs — long enough that adding one row and checking it used to mean either waiting for the other 567 or trusting the new one untested. diff --git a/tests/test_docs.py b/tests/test_docs.py index aaca56b..2ff50a6 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -677,3 +677,61 @@ def test_the_recall_skill_names_the_release_it_was_written_for(capsys): echoes = skill.split("## Echoes ", 1)[1].split(" ## ", 1)[0] assert "`gitmemory --version`" in echoes, echoes + + +def test_the_main_sessions_only_pattern_does_what_watching_md_says(tmp_path): + """`docs/watching.md` offers a pattern that leaves Claude Code's subagents out, + and says a hook cannot bring one back in. Both are claims about the watcher, + so the documented block is the config here, verbatim, over Claude Code's + layout: a Task subagent, a workflow agent and the workflow's journal beside + the main session. The default pattern finds all four, so the three this one + misses are files the watcher would otherwise capture. The page also says the + recipe must replace the default table, not sit beside it.""" + import dataclasses + + from gitmemory import daemon + + doc = _read("docs/watching.md") + section = re.search(r"^### Main sessions only.*?(?=^#)", doc, re.M | re.S) + assert section, "docs/watching.md has no main-sessions-only recipe" + block = re.search(r"^```toml\n(.*?)^```", section.group(0), re.M | re.S) + assert block, section.group(0) + + turn = '{"type":"user","message":{"role":"user","content":"hello"}}\n' + session = os.path.join(os.path.expanduser("~/.claude/projects"), "-proj", "s1") + parent = session + ".jsonl" + workflow = os.path.join(session, "subagents", "workflows", "wf_1") + child = os.path.join(session, "subagents", "agent-a1.jsonl") + others = [child, *(os.path.join(workflow, f) for f in ("agent-b2.jsonl", "journal.jsonl"))] + for path in (parent, *others): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + fh.write(turn) + home = str(tmp_path / "store") + os.makedirs(os.path.join(home, daemon.SPOOL)) + with open(os.path.join(home, "config.toml"), "w", encoding="utf-8") as fh: + fh.write(block.group(1)) + + watches = daemon.load_watches(home) + found = [p for _, p in daemon.discover(watches)] + assert found == [os.path.realpath(parent)] + every = [dataclasses.replace(w, pattern=daemon.PATTERN) for w in watches] + assert len(daemon.discover(every)) == 4 + first = daemon.tick(home, watches, commit=False) + assert [key.split("/")[1].startswith("s1-") for key in first.captured] == [True] + + # A hook firing as the subagent ends rings for a file the pattern left out. + with open(child, "a", encoding="utf-8") as fh: + fh.write(turn) + with open(os.path.join(home, daemon.SPOOL, "1-2-SessionEnd.json"), "w") as fh: + json.dump({"transcript_path": child}, fh) + second = daemon.tick(home, watches, interval=1e9, commit=False) + assert second.captured == [] and second.spool_consumed == 1 + + # Added as a second table beside a default one on the same root, it leaves + # nothing out, which is why the page says to edit the table you have. + default = re.sub(r"^pattern = .*\n", "", block.group(1), flags=re.M) + assert default != block.group(1) + with open(os.path.join(home, "config.toml"), "w", encoding="utf-8") as fh: + fh.write(default + "\n" + block.group(1)) + assert len(daemon.discover(daemon.load_watches(home))) == 4 diff --git a/tests/test_index.py b/tests/test_index.py index edf1b7f..5d2a3d3 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -1550,3 +1550,86 @@ def roles(*argv: str) -> list[str]: assert roles() == ["user/text", "user/tool_result", "assistant/text"] assert roles("--policy", "latest") == ["user/tool_result", "assistant/text", "user/text"] + + +def sidechain(uid: str, text: str, ts: str) -> dict: + """A subagent's turn. Claude Code marks it `isSidechain` wherever it is + written, and its first one is user-role text the parent agent wrote: the + dispatch prompt.""" + return dated(uid, text, ts) | {"isSidechain": True, "agentId": "a1"} + + +def with_a_subagent(home: str, tmp_path: Path) -> None: + """The user's rule in a main session, then the opposite a day later in a + dispatch prompt to one of its subagents. Claude Code's own layout: + `/.jsonl` and `//subagents/agent-.jsonl`.""" + parent = tmp_path / "proj" / "s1.jsonl" + child = tmp_path / "proj" / "s1" / "subagents" / "agent-a1.jsonl" + child.parent.mkdir(parents=True) + write( + str(parent), + [ + dated("u1", "Don't add a retry loop around the upload.", "2026-09-28T10:00:00Z"), + assistant("a1", [{"type": "text", "text": "Understood: the upload has no retry loop."}]) + | {"timestamp": "2026-09-28T11:00:00Z"}, + ], + ) + prompt = sidechain("u2", "Add a retry loop around the upload.", "2026-09-29T09:00:00Z") + write(str(child), [prompt]) + store.capture(str(parent), "claude-code", "s1", home=home) + store.capture(str(child), "claude-code", "agent-a1", home=home) + index.build(home) + + +def test_a_subagent_prompt_is_not_the_users_word(home, tmp_path): + """The forgetful agent again, arriving through a dispatch prompt rather than a + proposal. The prompt is user text, newer than the rule and matching it, so + the default led with it as the user's own words. The turn is marked, so it + is ranked with the agent's turns, where the parent's instructions belong.""" + with_a_subagent(home, tmp_path) + db = index.open_db(index.db_path(home)) + try: + hits = index.search(db, "retry loop upload") + finally: + db.close() + ordered = index.latest_user(hits) + assert ordered[0].text == "Don't add a retry loop around the upload." + assert [h.is_sidechain for h in ordered] == [False, True, False] + + +def test_recall_marks_a_subagent_turn(home, tmp_path, capsys): + """A prefix on the role, not a new column: `split()[3]` is role/kind to + whatever parses these lines, this file included, and a main session's line + is byte-for-byte what it was.""" + with_a_subagent(home, tmp_path) + capsys.readouterr() + assert main(["--home", home, "recall", "retry loop upload"]) == 0 + lines = capsys.readouterr().out.splitlines() + assert [line.split()[3] for line in lines] == [ + "user/text", + "subagent:user/text", + "assistant/text", + ] + assert lines[1].split()[2].startswith("claude-code/agent-a1/") + + +def test_a_sidechain_line_inside_a_main_file_is_marked(home, src): + """Older Claude Code wrote a subagent's turns into the parent's own file, each + line marked. The mark is read per turn, so those are caught too, which no + rule on the file's path can do.""" + db = built( + home, + src, + [ + dated("u1", "keep the upload single-shot", "2026-09-28T10:00:00Z"), + sidechain("u2", "review the upload code", "2026-09-28T10:05:00Z"), + ], + ) + try: + hits = index.search(db, "upload") + finally: + db.close() + assert {h.text: h.is_sidechain for h in hits} == { + "keep the upload single-shot": False, + "review the upload code": True, + } From beffcf2f4d90270e4f5889f716aad5fe8deb2193 Mon Sep 17 00:00:00 2001 From: doronp <3586743+doronp@users.noreply.github.com> Date: Sun, 18 Oct 2026 13:37:02 +0300 Subject: [PATCH 04/13] feat: gitmemory init writes the standard [[watch]] table Setup meant writing TOML by hand: no command wrote config.toml, and with none the watcher logs "watching nothing". `gitmemory init claude-code` (or `pi`) now adds that agent's documented root to the config. Watch roots still have no default. daemon.ROOTS is read only by init, and only because the user named the agent; load_watches reads the file and nothing else. It appends and never changes a byte already there, because config.toml also holds the [remote.*] push opt-ins and a rewrite would revoke one. Same agent and root already watched: exit 0, nothing written. It refuses with exit 1, writing nothing, when the file is a symlink, has another hard link (the rename would leave the other name holding the old bytes), or is not a regular file; is not valid TOML; has a `watch` a [[watch]] table cannot join; already watches the root under another agent (discover gives a shared root's files to the first watch); or holds a table the watcher would skip. That last check runs the watcher's own rules, split out of load_watches as watches_from, so the refusal is the watcher's sentence and cannot drift from it. Usage errors and a config it could not read stay exit 2: an unreadable file is never taken for a missing one and replaced. Since init now prints the watcher's sentence as its reason, the store check gets one per direction: a root inside the store "is inside the store" rather than "contains the store itself", which was wrong there. The write goes through store._write_atomic, so a running watcher reads the old file or the new one. A killed init can leave config.toml.tmp.., a full copy of the file, at the store root, and the watcher's `git add --all` would commit it, so /config.toml.tmp.* is gitignored. Docs: README quickstart and command table, docs/watching.md, docs/USAGE.md, hook/README.md, ARCHITECTURE.md. Skill setup step 2 tells the user to run init, after the older-CLI bullet and the `--python 3.13` install line already there, and init joins the commands the skill must not run. The plugin description adds the configure step beside the pinned install command. The doc test reads the table under watching.md's "## Configure what to watch", which ends before the main-sessions-only recipe's own toml block. Eight mutation rows (609 -> 617); suite 1,043 -> 1,063 offline. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QPBLmJD2q1PVxdkWx8C16U Signed-off-by: doronp <3586743+doronp@users.noreply.github.com> --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 9 ++ README.md | 26 ++++- docs/ARCHITECTURE.md | 2 +- docs/REPRODUCE.md | 2 +- docs/USAGE.md | 11 +- docs/watching.md | 21 +++- hook/README.md | 2 +- skills/recall/SKILL.md | 14 +-- src/gitmemory/__main__.py | 82 ++++++++++++++ src/gitmemory/daemon.py | 27 ++++- src/gitmemory/gitrepo.py | 4 + tests/mutate_index.py | 69 +++++++++++- tests/test_docs.py | 21 ++++ tests/test_init.py | 220 +++++++++++++++++++++++++++++++++++++ 15 files changed, 486 insertions(+), 26 deletions(-) create mode 100644 tests/test_init.py diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index d16bd30..2b2cab1 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "gitmemory", "version": "0.1.0", - "description": "Registers gitmemory's hook shim on PreCompact and SessionEnd, so a session is captured at the compaction boundary instead of at the watcher's next sweep, and adds a recall skill that has the agent search the captured sessions with gitmemory recall and quote what was said, with the address of each line. The hook is only a doorbell, and the skill writes nothing but the search index: nothing is captured unless the gitmemory watcher is installed (uv tool install --python 3.13 git+https://github.com/doronp/gitmemory) and running, and without the hook gitmemory is still correct.", + "description": "Registers gitmemory's hook shim on PreCompact and SessionEnd, so a session is captured at the compaction boundary instead of at the watcher's next sweep, and adds a recall skill that has the agent search the captured sessions with gitmemory recall and quote what was said, with the address of each line. The hook is only a doorbell, and the skill writes nothing but the search index: nothing is captured unless the gitmemory watcher is installed (uv tool install --python 3.13 git+https://github.com/doronp/gitmemory), configured (gitmemory init claude-code) and running, and without the hook gitmemory is still correct.", "author": { "name": "doronp" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 91b343d..9c309a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ tell you. ### Added +- **`gitmemory init claude-code`** (or `pi`) adds that agent's standard + `[[watch]]` table to `config.toml`, so setup no longer means writing TOML by + hand. Watch roots still have no default: the root is written only because you + named the agent. It appends without changing a byte already there, because + the file also holds the `[remote.*]` push opt-ins. It does nothing if the + table is already there, and it refuses, writing nothing and exiting 1, a root + the watcher would skip or another agent already watches, and a `config.toml` + that is a symlink or a hard link. See + [docs/watching.md](docs/watching.md#configure-what-to-watch). - **`recall --policy`** decides which of two disagreeing hits comes first: `latest-user` (the user's own words newest first, then the rest), `latest` (newest first, whoever said it), `relevance` (best match first), or your own diff --git a/README.md b/README.md index d0e0d80..ff07a22 100644 --- a/README.md +++ b/README.md @@ -71,15 +71,28 @@ never does (`"never"`), put a Python 3.13 on `PATH`.[^pipx] [^pipx]: pipx installs into the Python it runs under, so it needs a 3.13 interpreter: `pipx install --python python3.13 git+https://github.com/doronp/gitmemory`. -Tell it what to watch. Watch roots have no default, and the config lives in -`$GITMEMORY_HOME/config.toml` (by default `~/.gitmemory`): +Tell it what to watch. Watch roots have no default, so name the agent: + +```sh +gitmemory init claude-code # or: gitmemory init pi +``` + +That adds these lines to `$GITMEMORY_HOME/config.toml` (by default +`~/.gitmemory`): ```toml +# added by `gitmemory init claude-code` [[watch]] agent = "claude-code" roots = ["~/.claude/projects"] ``` +It appends without changing a byte already there, does nothing if the table is +already there, and refuses a root the watcher would skip. To write the table +yourself, or for another agent or root, +[docs/watching.md](docs/watching.md#configure-what-to-watch) has the rest of +the keys. + Then: ```sh @@ -104,6 +117,7 @@ agent after compaction, derived key ideas, the dashboard — is in | Command | What it does | |---|---| +| `gitmemory init ` | Adds the standard watch table for `claude-code` or `pi` to `config.toml`; does nothing if it is already there | | `gitmemory watch` | Tails the configured roots, captures what grew, and commits | | `gitmemory capture ` | Copies out one transcript's new bytes, by hand | | `gitmemory verify` | Checks every manifest's contiguity proof | @@ -194,13 +208,13 @@ real text, and that is a measurement rather than a suspicion. | What | How it was measured | Result | |---|---|---| | Hook cost in the agent's critical path | Timed against spawning `true` the same way, three runs of 400 | p50 **7.4 – 7.5 ms**, p99 **10.2 – 11.5 ms** | -| The suite | On a fresh checkout, no downloads | **1,043 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | -| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **609** negative controls | +| The suite | On a fresh checkout, no downloads | **1,063 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | +| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **617** negative controls | The two suite counts do not add up, and should not. Switching the corpora on -collects 1368, not 1371. Three conformance cases fill parametrisations that +collects 1388, not 1391. Three conformance cases fill parametrisations that collect as one empty placeholder each while the corpora are absent, so they -replace three of the 1043 rather than joining them. Every figure on this board +replace three of the 1063 rather than joining them. Every figure on this board is pinned by a test, which is how it stays true. **What is not measured**, and is shown as a row on the dashboard rather than diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 9014a8a..2996385 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -92,7 +92,7 @@ $GITMEMORY_HOME/ # default ~/.gitmemory; a git repository index/ # gitignored: SQLite is not diffable spool/ # gitignored: one file per hook fire .locks/ # gitignored: one per session - config.toml # watch roots; there is no default + config.toml # watch roots; no default (`init` writes one) ``` A capture commits **only the bytes appended since the last recorded offset**, diff --git a/docs/REPRODUCE.md b/docs/REPRODUCE.md index c5d148d..78f292a 100644 --- a/docs/REPRODUCE.md +++ b/docs/REPRODUCE.md @@ -505,7 +505,7 @@ test fail. The count is the number of rows in `tests/mutate_index.py`. uv run python -c "import sys; sys.path.insert(0, 'tests'); from mutate_index import MUTANTS; print(len(MUTANTS))" ``` -**Expected:** `609`. Checked while writing this guide. +**Expected:** `617`. Checked while writing this guide. To run the controls rather than count them: diff --git a/docs/USAGE.md b/docs/USAGE.md index 86cc1d3..89d65c5 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -25,9 +25,16 @@ repository, and lets you search it. So: ## Set it up once -1. Install and configure, as in the [README quickstart](../README.md#quickstart). +1. Install, as in the [README quickstart](../README.md#quickstart), then name + what to watch: + + ```sh + gitmemory init claude-code # or: gitmemory init pi + ``` + Watch roots have no default; `config.toml` names each agent and where its - transcripts live. [agents.md](agents.md) lists what can be captured. + transcripts live, and `init` adds that table for you. [agents.md](agents.md) + lists what can be captured. 2. Run the watcher and leave it running: ```sh diff --git a/docs/watching.md b/docs/watching.md index c9737d2..de3146e 100644 --- a/docs/watching.md +++ b/docs/watching.md @@ -14,7 +14,22 @@ Watch roots have no default, deliberately. A memory tool whose first run scans your home directory is a tool you have to audit before you can trust it, so gitmemory watches exactly what you name and nothing else. -`$GITMEMORY_HOME/config.toml`: +`gitmemory init claude-code` (or `gitmemory init pi`) writes the table for +you, with the root [agents.md](agents.md) gives for that agent. That is still +not a default: the root is written because you named the agent, and the watcher +reads only the file. `init` appends to an existing `config.toml` and never +changes a byte already there, since the file also holds any `[remote.*]` push +opt-in. It writes the result to a new file and renames it into place, so a +running watcher reads the old file or the new one, and the new one is mode +0600. It does nothing if the table is already there. It refuses, writing +nothing and exiting 1, a root another agent already watches, a root the watcher +would skip (listed below; `init` prints the watcher's own message), a file that +is not valid TOML or already has a `watch` of another shape, and a +`config.toml` that is a symlink or has another hard link, because the rename +would leave the other name holding the old file. A `config.toml` it cannot read +is an error, exit 2. A running watcher picks the new table up on its next pass. + +By hand, or for any other agent or root, `$GITMEMORY_HOME/config.toml`: ```toml [[watch]] @@ -151,8 +166,8 @@ anywhere useful. The two failures worth watching for are different shapes: | Symptom | Likely cause | |---|---| -| `no .../config.toml; watching nothing` | The file is not where the watcher is looking. Check `$GITMEMORY_HOME`. | -| Watcher quiet, store empty, no complaint | A root that **does not exist** is kept, not refused — before an agent has run for the first time that is the normal state, and `run` re-reads the config every pass, so it starts working the moment the directory appears. The cost is that a typo'd root and a not-yet-used one look identical. Check the path by hand. | +| `no .../config.toml; watching nothing` | The file is not where the watcher is looking. Check `$GITMEMORY_HOME`, or run `gitmemory init ` to write it there. | +| Watcher quiet, store empty, no complaint | A root that **does not exist** is kept, not refused — before an agent has run for the first time that is the normal state, and `run` re-reads the config every pass, so it starts working the moment the directory appears. The cost is that a typo'd root and a not-yet-used one look identical. Check the path by hand; `gitmemory init` says so when the root it writes does not exist yet. | | `compact_boundaries: []` everywhere | Adapter missing or `--no-parse`. The bytes are intact; only the boundaries are absent. | | `source changed while parsing` | The transcript was rewritten between the parse and the copy — a compaction landing mid-pass. The bytes are captured; that one generation has no boundaries. Nothing to do. | | `was captured from , not ; pass a distinct --session-id` | Two transcripts collided on one session name, and the store refuses to mix them rather than duplicating both for ever. The advice in the message is for the CLI and there is no `--session-id` to pass here: rename or move either file — the name is derived from the path — and the next pass captures both. Until then the second one is **not being captured**, so this is not a warning to leave running. | diff --git a/hook/README.md b/hook/README.md index e857c54..df66aed 100644 --- a/hook/README.md +++ b/hook/README.md @@ -94,7 +94,7 @@ agent run `gitmemory recall` and quote what it returns ([USAGE.md](../docs/USAGE.md#put-it-back-in-front-of-the-agent)). Neither captures anything. **Nothing is captured until the watcher is installed and running** (`uv tool install --python 3.13 git+https://github.com/doronp/gitmemory`, -a watch root in `config.toml`, then `gitmemory watch`, as in the +`gitmemory init claude-code`, then `gitmemory watch`, as in the [Quickstart](../README.md#quickstart)), and gitmemory is correct without the plugin, as it is without the hook. The hook and the skill run with Claude Code's environment, so a `GITMEMORY_HOME` you set for the watcher has to be set diff --git a/skills/recall/SKILL.md b/skills/recall/SKILL.md index 13fafb1..e042a76 100644 --- a/skills/recall/SKILL.md +++ b/skills/recall/SKILL.md @@ -111,10 +111,9 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. or it reports `0 generation(s)`: nothing has been captured there. Tell the user how it gets set up, and stop. Do none of it yourself: 1. `uv tool install --python 3.13 git+https://github.com/doronp/gitmemory` - 2. a `[[watch]]` table in `config.toml` in the store (`$GITMEMORY_HOME`, by - default `~/.gitmemory`) naming the agent and where its transcripts live; - for Claude Code, `agent = "claude-code"` and - `roots = ["~/.claude/projects"]` + 2. `gitmemory init claude-code`, which adds the `[[watch]]` table for + `~/.claude/projects` to `config.toml` in the store (`$GITMEMORY_HOME`, by + default `~/.gitmemory`) 3. `gitmemory watch`, left running. The [Quickstart](https://github.com/doronp/gitmemory#quickstart) has the @@ -149,6 +148,7 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. Run `gitmemory --version`, `gitmemory recall` and `gitmemory index`, and read files under ``. `index` writes only the derived index under `/index/`. Do not run -`gitmemory capture`, `gitmemory watch`, `gitmemory derive`, `gitmemory push` -or `gitmemory dashboard`, and never `gitmemory dashboard --expose`. Do not -edit `config.toml` or anything else in the store. +`gitmemory init`, `gitmemory capture`, `gitmemory watch`, `gitmemory derive`, +`gitmemory push` or `gitmemory dashboard`, and never +`gitmemory dashboard --expose`. Do not edit `config.toml` or anything else in +the store. diff --git a/src/gitmemory/__main__.py b/src/gitmemory/__main__.py index fd54038..423886b 100644 --- a/src/gitmemory/__main__.py +++ b/src/gitmemory/__main__.py @@ -16,7 +16,9 @@ import os import socket import sqlite3 +import stat import sys +import tomllib from . import __version__, daemon, dashboard, derive, gitrepo, index, redact, store from .records import safe_text @@ -51,6 +53,82 @@ def _safe_exc(exc: BaseException) -> str: return redact.safe_path(str(exc)) +def _refuse(why: str) -> int: + print(f"refusing: {why}", file=sys.stderr) + return 1 + + +def _init(args) -> int: + """Add the standard `[[watch]]` table for one agent to `config.toml`. + + Watch roots still have no default: this writes one only because the user + named the agent. It appends and never changes a byte already there, because + the same file holds the `[remote.*]` push opt-ins; it does nothing when the + table is already there; and it refuses, writing nothing, a table the watcher + would skip. + """ + home = store.resolve_home(args.home) + path = os.path.join(home, "config.toml") + root = daemon.ROOTS[args.agent] + real = os.path.realpath(os.path.expanduser(root)) + try: + # `lstat`: `os.replace` over a symlink swaps the link for a file, so the + # user's dotfile would silently stop being the config. And a FIFO hangs. + st = os.lstat(path) + if not stat.S_ISREG(st.st_mode): + return _refuse(f"{path} is not a regular file; add the table by hand") + # A hard link is the same trap without the `l`: the rename gives the + # name a new inode, and the other name keeps the old bytes. + if st.st_nlink > 1: + return _refuse(f"{path} has another hard link; add the table by hand") + with open(path, "rb") as fh: + old = fh.read() + except FileNotFoundError: + old = b"" + try: + before = tomllib.loads(old.decode("utf-8")) + except ValueError as exc: # `TOMLDecodeError`, and `UnicodeDecodeError` on a bad byte + return _refuse(f"{path} is not valid TOML ({exc}); fix it first") + have = before.get("watch", []) + table = ( + f"# added by `gitmemory init {args.agent}`\n" + f'[[watch]]\nagent = "{args.agent}"\nroots = ["{root}"]\n' + ) + sep = b"" if not old else b"\n" if old.endswith(b"\n") else b"\n\n" + new = old + sep + table.encode() + # Re-parsed whole, because appending is not always joining: after a plain + # `[watch]` or an inline `watch = [...]` the result does not parse at all. + try: + after = tomllib.loads(new.decode("utf-8")) + except ValueError: + after = {} + if not isinstance(have, list) or len(after.get("watch") or []) != len(have) + 1: + return _refuse(f"{path} has a `watch` a [[watch]] table cannot join; add it by hand") + for w in daemon.watches_from(before, path, home, lambda _m: None): + if real in w.roots: + if w.agent == args.agent: + print(f"{path} already watches {root} for {args.agent}; nothing to do") + return 0 + # `discover` gives each file under a shared root to the first watch + # in the file, so a table appended after it gets nothing it takes. + return _refuse(f"{path} already watches {root} as {w.agent!r}") + # The watcher's own rules and its own sentence, so the two cannot disagree + # about why: a table it would skip on every pass is not written at all. + notes: list[str] = [] + if not any( + w.agent == args.agent and real in w.roots + for w in daemon.watches_from(after, path, home, notes.append) + ): + mine = f"{path} [[watch]] #{len(have) + 1}:" + return _refuse("; ".join(n for n in notes if n.startswith(mine))) + store._write_atomic(path, new) # 0600, and a running watcher reads old or new + print(f"{'added to' if old else 'wrote'} {path}: watch {root} for {args.agent}") + if not os.path.isdir(real): + print(f"note: {root} does not exist yet; it is watched once it does", file=sys.stderr) + print("next: run `gitmemory watch`; a watcher already running reads this on its next pass") + return 0 + + def _capture(args) -> int: home = store.resolve_home(args.home) cap = daemon.capture_one( @@ -356,6 +434,10 @@ def main(argv: list[str] | None = None) -> int: ) sub = ap.add_subparsers(dest="cmd", required=True) + ini = sub.add_parser("init", help="add an agent's standard [[watch]] table to config.toml") + ini.add_argument("agent", choices=sorted(daemon.ROOTS), help="whose transcript root to watch") + ini.set_defaults(fn=_init) + cap = sub.add_parser("capture", help="copy out bytes appended since the last capture") cap.add_argument("source") cap.add_argument("--agent", default="claude-code") diff --git a/src/gitmemory/daemon.py b/src/gitmemory/daemon.py index a3be2a5..7d3ce00 100644 --- a/src/gitmemory/daemon.py +++ b/src/gitmemory/daemon.py @@ -16,7 +16,8 @@ `config.toml` names. A product whose first run scans `~/.claude/projects` ingests its owner's entire history on install, which is the one thing this project promises not to do. The adapter documents the conventional path; the - config has to name it. + config has to name it. `gitmemory init ` writes it there when asked + by name, and nothing reads `ROOTS` as a default. - **A spool record is a doorbell, not data.** It says something happened to a transcript. Everything else — how many bytes, where the boundary is, whether the file was rewritten — is re-derived from the source file. A record naming a @@ -117,6 +118,14 @@ class Tick: adopted: list[str] = field(default_factory=list) +# Where each shipped adapter's agent writes its transcripts, as docs/agents.md +# lists them. Not a default: `load_watches` never reads this. Only `gitmemory +# init ` does, and it writes the table into `config.toml` because the +# user named the agent, so the config still names every root that is watched. +# Here and not on the adapters, whose contract is two symbols and nothing else. +ROOTS = {"claude-code": "~/.claude/projects", "pi": "~/.pi/agent/sessions"} + + def load_watches(home: str, log=None) -> list[Watch]: """`[[watch]]` tables from `config.toml`. No config means watch nothing. @@ -168,6 +177,14 @@ def load_watches(home: str, log=None) -> list[Watch]: # this and the specific message is the better one. [E4, review: daemon 3] say(f"{path} is not readable as text ({exc.__class__.__name__}); watching nothing") return [] + return watches_from(cfg, path, home, say) + + +def watches_from(cfg: dict, path: str, home: str, say) -> list[Watch]: + """The `[[watch]]` tables of a config already parsed, with every refusal + `load_watches` applies. Split out so `gitmemory init` asks the watcher's own + question of the table it is about to write, rather than keep a second copy + of these rules that drifts from the first.""" out = [] for n, entry in enumerate(cfg.get("watch") or []): where = f"{path} [[watch]] #{n + 1}" @@ -254,7 +271,13 @@ def load_watches(home: str, log=None) -> list[Watch]: # `~/.gitmemory` is safe only because `_hits` skips dotted # components, which is why that skip is preserved rather than # inherited. [E4, review: CLI 3] - say(f"{where}: root {r} contains the store itself; skipped") + # + # Two directions, two sentences: `gitmemory init` prints this as + # its reason, and a root under the store does not contain it. + how = "contains the store itself" + if not _inside(resolved, home): + how = "is inside the store" + say(f"{where}: root {r} {how}; skipped") continue kept.append(resolved) if kept: diff --git a/src/gitmemory/gitrepo.py b/src/gitmemory/gitrepo.py index 62c680c..fba4891 100644 --- a/src/gitmemory/gitrepo.py +++ b/src/gitmemory/gitrepo.py @@ -79,6 +79,10 @@ # about to push to that same remote. `push_allowed` now refuses an inline # credential outright; this line is the belt to that brace, and it also keeps a # store's remote list out of a repository other people can read. [E7] +# +# And its temp. `gitmemory init` publishes the file by rename, so an init killed +# between the write and the `os.replace` leaves a whole copy of it beside it. +/config.toml.tmp.* /config.toml """ diff --git a/tests/mutate_index.py b/tests/mutate_index.py index d157f7b..624e96c 100644 --- a/tests/mutate_index.py +++ b/tests/mutate_index.py @@ -4597,7 +4597,7 @@ # anchor appear twice and the harness would skip itself. [E7] "the docstring's row count drifts away from the index again", "tests/mutate_index.py", - "a full pass is 6" + "09 mutants", + "a full pass is 6" + "17 mutants", "a full pass is 14" + "6 mutants", "test_the_row_count_in_the_docstring_is_the_row_count", ), @@ -5525,6 +5525,71 @@ " who = h.role", "test_recall_marks_a_subagent_turn", ), + # --- `gitmemory init `: the config is appended to, or left alone --- + ( + # `config.toml` also holds `[remote.*] allow_push`, so a rewrite revokes + # a push opt-in nobody asked to revoke. + "init overwrites config.toml instead of appending to it", + "__main__.py", + " new = old + sep + table.encode()", + " new = table.encode()", + "test_cli_init_appends_and_keeps_every_byte_already_there", + ), + ( + "init follows a symlinked config.toml and replaces the link with a file", + "__main__.py", + " st = os.lstat(path)", + " st = os.stat(path)", + "test_cli_init_does_not_replace_a_symlinked_config", + ), + ( + # The rename gives the name a new inode; the dotfile keeps the old bytes. + "init replaces a hard-linked config.toml and splits it from the other name", + "__main__.py", + " if st.st_nlink > 1:", + " if False:", + "test_cli_init_does_not_split_a_hard_linked_config", + ), + ( + # Unreadable taken for absent: the file is replaced by the table alone. + "init takes a config.toml it cannot read for a missing one", + "__main__.py", + ' except FileNotFoundError:\n old = b""', + ' except OSError:\n old = b""', + "test_cli_init_says_error_and_exits_2_when_it_could_not_look", + ), + ( + # init prints this as its reason; the store contains the root, not + # the other way round. + "a root inside the store is said to contain the store", + "daemon.py", + ' how = "is inside the store"', + ' how = "contains the store itself"', + "test_cli_init_refuses_a_table_the_watcher_would_skip_and_writes_nothing", + ), + ( + # The table is written, and then skipped by the watcher on every pass. + "init writes a table the watcher would skip", + "__main__.py", + " if not any(\n w.agent == args.agent and real in w.roots\n", + " if False and not any(\n w.agent == args.agent and real in w.roots\n", + "test_cli_init_refuses_a_table_the_watcher_would_skip_and_writes_nothing", + ), + ( + # `discover` gives a root to the first watch naming it: dead on arrival. + "init adds a second watch on a root another agent already watches", + "__main__.py", + ' return _refuse(f"{path} already watches {root} as {w.agent!r}")', + " continue", + "test_cli_init_refuses_a_root_another_agent_already_watches", + ), + ( + "a killed init's temp copy of config.toml is committed", + "gitrepo.py", + "/config.toml.tmp.*\n", + "", + "test_cli_init_leaves_no_temp_the_watcher_would_commit", + ), ] @@ -5677,7 +5742,7 @@ def _select(test: str) -> list[str]: def main() -> int: """Run every mutant, or only those whose name contains an argument. - The filter exists because a full pass is 609 mutants × two suite runs — + The filter exists because a full pass is 617 mutants × two suite runs — long enough that adding one row and checking it used to mean either waiting for the other 567 or trusting the new one untested. diff --git a/tests/test_docs.py b/tests/test_docs.py index 2ff50a6..bd8f80f 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -735,3 +735,24 @@ def test_the_main_sessions_only_pattern_does_what_watching_md_says(tmp_path): with open(os.path.join(home, "config.toml"), "w", encoding="utf-8") as fh: fh.write(default + "\n" + block.group(1)) assert len(daemon.discover(daemon.load_watches(home))) == 4 + + +def test_the_documented_watch_table_is_the_one_init_writes(tmp_path, capsys): + """The README and watching.md show the table and say `gitmemory init` writes + it. Two copies of one config in prose and one in `daemon.ROOTS` is how a doc + ends up telling people to hand-write a table the command no longer writes, + so the block under each section heading is looked for in what init wrote. + By section, because watching.md may hold other `toml` recipes init does + not write; split on `##` because a TOML comment is a `#` at column 0.""" + from gitmemory.__main__ import main + + home = str(tmp_path / "store") + assert main(["--home", home, "init", "claude-code"]) == 0 + capsys.readouterr() + with open(os.path.join(home, "config.toml"), encoding="utf-8") as fh: + written = fh.read() + for rel, heading in [("README.md", "## Quickstart"), ("docs/watching.md", "## Configure")]: + section = _read(rel).split(f"\n{heading}", 1)[1].split("\n##", 1)[0] + block = re.search(r"^```toml\n(.*?)^```", section, re.M | re.S) + assert block, f"{rel} {heading} no longer shows the table" + assert block.group(1) in written, rel diff --git a/tests/test_init.py b/tests/test_init.py new file mode 100644 index 0000000..09ef86f --- /dev/null +++ b/tests/test_init.py @@ -0,0 +1,220 @@ +# SPDX-FileCopyrightText: 2026 The gitmemory authors +# SPDX-License-Identifier: Apache-2.0 +"""`gitmemory init `: the `[[watch]]` table, written because it was asked for. + +Not `gitrepo.init`, which makes the store's repository and is tested in +`test_gitrepo.py`. This one writes `config.toml`, and the properties under test +are the ones that make writing it safe: the file also holds the `[remote.*]` +push opt-ins, so it is appended to and never rewritten; a second run writes +nothing; and a table the watcher would skip is refused before it is written, +with the watcher's own sentence, rather than written and left to fail quietly +on every pass. +""" + +from __future__ import annotations + +import os +import re +import stat +import subprocess + +import pytest + +from gitmemory import daemon, gitrepo, redact +from gitmemory.__main__ import main + + +def _cfg(home: str) -> str: + return os.path.join(home, "config.toml") + + +def test_cli_init_writes_the_table_the_watcher_reads(tmp_path, capsys, _no_real_home): + """No config, one command, and the watcher watches the documented root.""" + home = str(tmp_path / "store") + assert main(["--home", home, "init", "claude-code"]) == 0 + out, err = capsys.readouterr() + assert out.startswith(f"wrote {_cfg(home)}: watch ~/.claude/projects for claude-code\n") + assert "next: run `gitmemory watch`" in out + assert "~/.claude/projects does not exist yet" in err + with open(_cfg(home), encoding="utf-8") as fh: + assert fh.read().startswith("# added by `gitmemory init claude-code`\n[[watch]]\n") + want = os.path.realpath(_no_real_home / ".claude" / "projects") + assert daemon.load_watches(home) == [daemon.Watch("claude-code", (want,))] + assert stat.S_IMODE(os.stat(_cfg(home)).st_mode) == 0o600 + assert stat.S_IMODE(os.stat(home).st_mode) == 0o700 + + +def test_cli_init_twice_is_one_table(tmp_path, capsys): + home = str(tmp_path / "store") + assert main(["--home", home, "init", "pi"]) == 0 + with open(_cfg(home), "rb") as fh: + first = fh.read() + capsys.readouterr() + assert main(["--home", home, "init", "pi"]) == 0 + with open(_cfg(home), "rb") as fh: + assert fh.read() == first + assert "already watches ~/.pi/agent/sessions for pi; nothing to do" in capsys.readouterr().out + + +def test_cli_init_appends_and_keeps_every_byte_already_there(tmp_path): + """`config.toml` is also where a push is opted into. Rewriting it would + revoke the remote; appending has to leave it, comments and all.""" + home = tmp_path / "store" + home.mkdir() + old = b'# mine\n[remote.origin]\nurl = "git@host:me/p.git"\nallow_push = true' # no final \n + (home / "config.toml").write_bytes(old) + assert main(["--home", str(home), "init", "claude-code"]) == 0 + assert main(["--home", str(home), "init", "pi"]) == 0 + assert (home / "config.toml").read_bytes().startswith(old + b"\n") + assert redact.push_allowed(str(home), "origin")[0] + assert [w.agent for w in daemon.load_watches(str(home))] == ["claude-code", "pi"] + + +@pytest.mark.parametrize("case", ["file-root", "store-under-root", "root-under-store"]) +def test_cli_init_refuses_a_table_the_watcher_would_skip_and_writes_nothing( + tmp_path, capsys, _no_real_home, case +): + """Each one would be skipped by the watcher on every pass. The refusal is + the watcher's own sentence, so the two cannot disagree about why, and the + two overlaps are told apart: a root inside the store does not contain it.""" + projects = _no_real_home / ".claude" / "projects" + if case == "file-root": + projects.parent.mkdir() + projects.write_text("one transcript, not a directory") + home, why = str(tmp_path / "store"), "is a file, not a directory" + elif case == "store-under-root": + projects.mkdir(parents=True) + home, why = str(projects / "store"), "contains the store itself" + else: + home, why = str(_no_real_home / ".claude"), "is inside the store" + assert main(["--home", home, "init", "claude-code"]) == 1 + err = capsys.readouterr().err + assert re.search(rf"^refusing: .*\[\[watch\]\] #1: root ~/.claude/projects {why}", err, re.M) + assert not os.path.lexists(home), "a refusal writes nothing, not even the store" + + +@pytest.mark.parametrize( + ("body", "why"), + [ + (b"[[watch]\nnot toml", "is not valid TOML"), + (b"\xff\xfe", "is not valid TOML"), + (b'watch = [{agent = "pi", roots = ["/nowhere"]}]\n', "cannot join"), + (b'[watch]\nagent = "pi"\n', "cannot join"), + (b"watch = 5\n", "cannot join"), + ], + ids=["syntax", "not-utf8", "inline-array", "plain-table", "not-a-list"], +) +def test_cli_init_refuses_a_config_it_cannot_append_to_and_leaves_it( + tmp_path, capsys, body, why +): + """The last three parse, and appending `[[watch]]` to them does not: the + reason is said rather than left to whichever check happens to trip next.""" + home = tmp_path / "store" + home.mkdir() + (home / "config.toml").write_bytes(body) + assert main(["--home", str(home), "init", "claude-code"]) == 1 + err = capsys.readouterr().err + assert err.startswith("refusing: ") and why in err, err + assert (home / "config.toml").read_bytes() == body + + +@pytest.mark.parametrize("live", [True, False], ids=["live", "dangling"]) +def test_cli_init_does_not_replace_a_symlinked_config(tmp_path, capsys, live): + """`os.replace` over a link swaps the link for a file, so the user's dotfile + would silently stop being the config. Refused instead.""" + home = tmp_path / "store" + home.mkdir() + target = tmp_path / "dotfiles.toml" + if live: + target.write_text("") + (home / "config.toml").symlink_to(target) + assert main(["--home", str(home), "init", "claude-code"]) == 1 + assert "is not a regular file" in capsys.readouterr().err + assert os.path.islink(home / "config.toml") + assert (target.read_text() == "") if live else not target.exists() + + +def test_cli_init_does_not_split_a_hard_linked_config(tmp_path, capsys): + """A regular file, and still a link: `os.replace` gives `config.toml` a new + inode, so the dotfile hard-linked to it would keep the old bytes, push + opt-in and all, and silently stop being the config. Refused, like a symlink.""" + home = tmp_path / "store" + home.mkdir() + dotfile = tmp_path / "dotfiles.toml" + body = '[remote.origin]\nurl = "git@host:me/p.git"\nallow_push = true\n' + dotfile.write_text(body) + os.link(dotfile, home / "config.toml") + assert main(["--home", str(home), "init", "claude-code"]) == 1 + assert "has another hard link" in capsys.readouterr().err + assert os.path.samefile(dotfile, home / "config.toml") + assert dotfile.read_text() == body + + +@pytest.mark.parametrize("case", ["home-is-a-file", "unreadable"]) +def test_cli_init_says_error_and_exits_2_when_it_could_not_look(tmp_path, capsys, case): + """Exit 1 is an answer: init looked, and refused. A config it could not + read is not one, and taking it for absent would replace it with the table + alone, push opt-ins gone. So it is main's `error:` and exit 2, and the + file is left as it was.""" + home = tmp_path / "store" + body = '[remote.origin]\nurl = "git@host:me/p.git"\nallow_push = true\n' + if case == "home-is-a-file": + home.write_text(body) + kept = home + else: + if hasattr(os, "geteuid") and os.geteuid() == 0: + pytest.skip("root reads a mode-000 file") + home.mkdir() + kept = home / "config.toml" + kept.write_text(body) + kept.chmod(0) + try: + assert main(["--home", str(home), "init", "pi"]) == 2 + finally: + kept.chmod(0o600) + assert capsys.readouterr().err.startswith("error: ") + assert kept.read_text() == body + + +def test_cli_init_refuses_a_root_another_agent_already_watches(tmp_path, capsys, _no_real_home): + """Two watches on one root at the same depth: `discover` keeps the first, so + the table init would add is dead on arrival. Spelled absolute and with a + trailing slash, because the comparison is the watcher's resolved path.""" + home = tmp_path / "store" + home.mkdir() + body = f'[[watch]]\nagent = "pi"\nroots = ["{_no_real_home}/.claude/projects/"]\n' + (home / "config.toml").write_text(body) + assert main(["--home", str(home), "init", "claude-code"]) == 1 + assert "already watches ~/.claude/projects as 'pi'" in capsys.readouterr().err + assert (home / "config.toml").read_text() == body + + +def test_cli_init_knows_only_adapters_that_ship_and_roots_the_docs_name(): + """`ROOTS` is a third copy of a path the adapter's docstring and agents.md + already state, so it is checked against both. An agent with no documented + root is not offered: argparse refuses it as a usage error.""" + from gitmemory.adapters import ADAPTERS + + here = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + with open(os.path.join(here, "docs", "agents.md"), encoding="utf-8") as fh: + agents = fh.read() + for agent, root in daemon.ROOTS.items(): + assert agent in ADAPTERS + assert f"`{root}/**/*.jsonl`" in agents, root + assert ADAPTERS[agent].__doc__.split(":")[1].strip().startswith(root), agent + with pytest.raises(SystemExit) as done: + main(["init", "kimi-code"]) + assert done.value.code == 2 + + +def test_cli_init_leaves_no_temp_the_watcher_would_commit(tmp_path): + """`init` publishes `config.toml` by rename, so an init killed between the + write and the `os.replace` leaves a full copy of the file — push credential + included — beside it, and the watcher's `git add --all` would commit it.""" + home = str(tmp_path / "store") + gitrepo.init(home) + stray = "config.toml.tmp.4242.7f" + with open(os.path.join(home, stray), "w", encoding="utf-8") as fh: + fh.write('[remote.origin]\nurl = "https://example.invalid/x.git"\n') + ignored = subprocess.run(["git", "-C", home, "check-ignore", "-q", stray], check=False) + assert ignored.returncode == 0 From b39449aedca40bd2612f76f2c7ed5eb9968d36b1 Mon Sep 17 00:00:00 2001 From: doronp <3586743+doronp@users.noreply.github.com> Date: Sun, 18 Oct 2026 13:47:00 +0300 Subject: [PATCH 05/13] feat: gitmemory service install writes a user unit or LaunchAgent for the watcher The watcher is the only thing that captures, and nothing kept it running: the docs said "no installer" and left the unit to the user. A hand-written unit falls into three traps, each reproduced while designing this. The interpreter's realpath is outside the venv and cannot import gitmemory. `python -m` from a unit's working directory ($HOME) runs a stray ~/gitmemory.py instead of the package. And a user service sees neither the shell's GITMEMORY_HOME nor its PATH, so it watches another store or misses git. `gitmemory service install` (new module service.py) writes the facts only the running process has into one file: sys.executable as given, with `-P -m gitmemory`, an explicit `--home `, and PATH from git's directory (made absolute) plus /usr/bin:/bin. It writes a systemd --user unit on Linux, under $XDG_CONFIG_HOME only when that is absolute and otherwise ~/.config, and a LaunchAgent plist on macOS. It refuses (exit 1, O_EXCL) to overwrite an existing file, in the one-line `refusing: ...` shape `gitmemory init` uses, through the same helper. It exits 2 on another platform, with no git, when it cannot tell which Python is running, and for a store path with a line break, which a unit cannot hold and a plist would turn from \r into \n. It prints the start, check, log, upgrade and remove commands and never runs systemctl, launchctl or loginctl, because enabling it is the user's step, as installing the hook is. The removal is `stop; rm`, not `stop && rm`: `launchctl bootout` of a job written and never started exits 3, and the refusal's way out has to remove the file anyway. Tests run the file's own command line under the manager's conditions (cwd $HOME holding a decoy gitmemory.py, env HOME plus the file's PATH) for both backends, and run the printed removal against stand-in managers that fail. The unit's quoting is read back strictly, as systemd would, so an undoubled %h or $HOME or an unescaped backslash fails the test. Six mutation rows (O_EXCL, -P, realpath, the % and $ doubling, the removal's `;`) are caught by them. Docs: watching.md "Keep it running", USAGE step 2, README table, ARCHITECTURE (whose CLI list now names init and service), the skill's setup steps (install, `gitmemory init`, then `gitmemory service install`) and its do-not-run list, a dated DESIGN.md amendment naming the systemd user unit, CHANGELOG, and the pinned counts, re-measured on top of install, version, subagents and init. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QPBLmJD2q1PVxdkWx8C16U Signed-off-by: doronp <3586743+doronp@users.noreply.github.com> --- CHANGELOG.md | 8 + README.md | 9 +- docs/ARCHITECTURE.md | 3 +- docs/DESIGN.md | 3 + docs/REPRODUCE.md | 8 +- docs/USAGE.md | 9 +- docs/watching.md | 75 +++++++- skills/recall/SKILL.md | 15 +- src/gitmemory/__main__.py | 24 ++- src/gitmemory/service.py | 160 +++++++++++++++++ tests/mutate_index.py | 49 +++++- tests/test_service.py | 355 ++++++++++++++++++++++++++++++++++++++ 12 files changed, 692 insertions(+), 26 deletions(-) create mode 100644 src/gitmemory/service.py create mode 100644 tests/test_service.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c309a4..ed0b04f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,14 @@ tell you. the watcher would skip or another agent already watches, and a `config.toml` that is a symlink or a hard link. See [docs/watching.md](docs/watching.md#configure-what-to-watch). +- **`gitmemory service install`** writes a systemd user unit (Linux) or a + LaunchAgent (macOS) that keeps `gitmemory watch` running, and prints the + commands that start it, check it, follow its log, restart it after an + upgrade and remove it. It starts nothing itself, and it refuses, exiting 1, + to overwrite a file that is already there. The file names the Python, the + store and git's directory outright, because a user service inherits none of + the shell's. See + [docs/watching.md](docs/watching.md#keep-it-running). - **`recall --policy`** decides which of two disagreeing hits comes first: `latest-user` (the user's own words newest first, then the rest), `latest` (newest first, whoever said it), `relevance` (best match first), or your own diff --git a/README.md b/README.md index ff07a22..1f45e93 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,7 @@ agent after compaction, derived key ideas, the dashboard — is in |---|---| | `gitmemory init ` | Adds the standard watch table for `claude-code` or `pi` to `config.toml`; does nothing if it is already there | | `gitmemory watch` | Tails the configured roots, captures what grew, and commits | +| `gitmemory service install` | Writes a systemd user unit (Linux) or a LaunchAgent (macOS) that keeps `watch` running, and prints the line that starts it; it starts nothing itself ([docs/watching.md](docs/watching.md#keep-it-running)) | | `gitmemory capture ` | Copies out one transcript's new bytes, by hand | | `gitmemory verify` | Checks every manifest's contiguity proof | | `gitmemory index` | Rebuilds the SQLite FTS5 index from the store | @@ -208,13 +209,13 @@ real text, and that is a measurement rather than a suspicion. | What | How it was measured | Result | |---|---|---| | Hook cost in the agent's critical path | Timed against spawning `true` the same way, three runs of 400 | p50 **7.4 – 7.5 ms**, p99 **10.2 – 11.5 ms** | -| The suite | On a fresh checkout, no downloads | **1,063 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | -| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **617** negative controls | +| The suite | On a fresh checkout, no downloads | **1,084 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | +| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **623** negative controls | The two suite counts do not add up, and should not. Switching the corpora on -collects 1388, not 1391. Three conformance cases fill parametrisations that +collects 1409, not 1412. Three conformance cases fill parametrisations that collect as one empty placeholder each while the corpora are absent, so they -replace three of the 1063 rather than joining them. Every figure on this board +replace three of the 1084 rather than joining them. Every figure on this board is pinned by a test, which is how it stays true. **What is not measured**, and is shown as a row on the dashboard rather than diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 2996385..52df25a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -67,6 +67,7 @@ right of `git` can be deleted and rebuilt, so `index/` is gitignored and |---|---|---|---| | Trigger | [`hook/gitmemory-hook.sh`](../hook/gitmemory-hook.sh) | Writes the hook's stdin to one spool file and exits 0. No git, no Python, no network. It adds latency, not correctness | §2.3 | | Trigger | [`daemon.py`](../src/gitmemory/daemon.py) | The watcher. It tails the configured roots by `(path, inode, offset, mtime)`, drains the spool, captures and commits. This is what makes capture a guarantee | §2.3 | +| Trigger | [`service.py`](../src/gitmemory/service.py) | Writes the systemd user unit or LaunchAgent that keeps the watcher running, with this Python, the store and git's directory spelled out in it. It never starts one | §2.3 | | Parse | [`adapters/`](../src/gitmemory/adapters/) | Turns agent-native bytes into canonical records. An adapter is two symbols, `AGENT` and `parse(path)`, and it may not touch git, the index or derivation | §2.1–2.2 | | Parse | [`records.py`](../src/gitmemory/records.py) | Canonical `Session` / `Turn` / `Block` / `Event`, with content-derived IDs and sorted-key JSON, so the committed tree stays diffable | §2.1 | | Parse | [`jsonl.py`](../src/gitmemory/jsonl.py) | A streaming JSONL reader with exact byte offsets, vendored from fable (MIT) | §2.5b | @@ -74,7 +75,7 @@ right of `git` can be deleted and rebuilt, so `index/` is gitignored and | Store | [`gitrepo.py`](../src/gitmemory/gitrepo.py) | The only module that runs git, always as `git -C ` | §2.4 | | Derive | [`index.py`](../src/gitmemory/index.py) | A SQLite FTS5 (BM25) index rebuilt from raw bytes. It is never authoritative | §2.7 | | Derive | [`derive.py`](../src/gitmemory/derive.py), [`graph.py`](../src/gitmemory/graph.py) | Key ideas and a timeline. The decision extraction behind the graph is opt-in (`--graph`) | §2.6 | -| Surface | [`__main__.py`](../src/gitmemory/__main__.py) | The CLI: `capture watch verify push index derive dashboard recall` | §2.8 | +| Surface | [`__main__.py`](../src/gitmemory/__main__.py) | The CLI: `init capture watch service verify push index derive dashboard recall` | §2.8 | | Surface | [`dashboard.py`](../src/gitmemory/dashboard.py) | Datasette over the index, `--immutable`, on loopback, behind a single-use sign-in URL | §2.9 | | Egress | [`redact.py`](../src/gitmemory/redact.py) | The only place secrets are looked for. It stands at `push` and scans the files, the segment seams and the object graph a push would transmit | §2.4 | diff --git a/docs/DESIGN.md b/docs/DESIGN.md index df5cfb8..6d9aad9 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -32,6 +32,9 @@ Two properties, and they are the whole product: **No data from this machine is ingested, copied, or committed. Ever.** - Test corpora are synthetic or public-benchmark-derived. - Userspace only: no `crontab`, no `sudo`, no Docker. LaunchAgent is the keep-alive. + **[2026-10-18 — and on Linux a systemd user unit. `gitmemory service install` writes either + one, as the user, and starts neither: enabling it is the user's step, as installing the hook + is.]** - Local-first. No network by default. No remote configured by default. - Apache-2.0. diff --git a/docs/REPRODUCE.md b/docs/REPRODUCE.md index 78f292a..3e51eea 100644 --- a/docs/REPRODUCE.md +++ b/docs/REPRODUCE.md @@ -484,11 +484,11 @@ uv run pytest -q --collect-only -p no:cacheprovider | tail -1 uv run pytest -q --collect-only -p no:cacheprovider | grep -c '\.jsonl\]$' ``` -**Expected:** `1043/1044 tests collected (1 deselected)`, then -`1368/1369 tests collected (1 deselected)`, then `328`. The deselected test is +**Expected:** `1084/1085 tests collected (1 deselected)`, then +`1409/1410 tests collected (1 deselected)`, then `328`. The deselected test is the LongMemEval corpus test, which is opt-in: `uv run pytest -q -m corpus` after `bench/fetch_longmemeval.sh`. The README's -paragraph on why 1,043 + 328 is 1,368 and not 1,371 is the arithmetic of these +paragraph on why 1,084 + 328 is 1,409 and not 1,412 is the arithmetic of these three numbers. All three were run while writing this guide. `tests/test_docs.py` holds the README to the first and third numbers, so a @@ -505,7 +505,7 @@ test fail. The count is the number of rows in `tests/mutate_index.py`. uv run python -c "import sys; sys.path.insert(0, 'tests'); from mutate_index import MUTANTS; print(len(MUTANTS))" ``` -**Expected:** `617`. Checked while writing this guide. +**Expected:** `623`. Checked while writing this guide. To run the controls rather than count them: diff --git a/docs/USAGE.md b/docs/USAGE.md index 89d65c5..5744ec5 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -41,9 +41,12 @@ repository, and lets you search it. So: gitmemory watch ``` - How it stays running (a LaunchAgent, a systemd user unit, a terminal you - leave open) is your choice; [watching.md](watching.md#run-it) covers the - flags and how to tell a misconfigured watcher from an idle one. + How it stays running is your choice. `gitmemory service install` writes a + systemd user unit (Linux) or a LaunchAgent (macOS) that keeps it running + across logins, and prints the line that starts it; it starts nothing itself + ([watching.md](watching.md#keep-it-running)). A terminal you leave open + works too. [watching.md](watching.md#run-it) covers the flags and how to + tell a misconfigured watcher from an idle one. 3. Optional: install the [hook shim](../hook/README.md) so a capture happens at the compaction boundary itself rather than at the next sweep. The watcher alone is enough for correctness. diff --git a/docs/watching.md b/docs/watching.md index de3146e..6274b92 100644 --- a/docs/watching.md +++ b/docs/watching.md @@ -148,10 +148,77 @@ Each pass prints what it did, or nothing when nothing happened. Errors are repeated only when they change or once an interval, so a permanently broken root does not drown the log it belongs in. -Ctrl-C exits 130. There is no daemonisation, no pidfile, and no installer: -`gitmemory watch` is a foreground process, and how you keep it alive — a user -LaunchAgent, a systemd user unit, a terminal you leave open — is yours to -choose. Nothing here needs root. +Ctrl-C exits 130. There is no daemonisation and no pidfile: `gitmemory watch` +is a foreground process, and how you keep it alive — a terminal you leave open, +or a user service, which `gitmemory service install` writes for you +([below](#keep-it-running)) — is yours to choose. Nothing here needs root. + +## Keep it running + +`gitmemory service install` writes one file for your user's service manager, +and starts nothing: + +| Platform | File | Manager | +|---|---|---| +| Linux | `$XDG_CONFIG_HOME/systemd/user/gitmemory.service`, or under `~/.config` when that is unset or not an absolute path | systemd, as your user | +| macOS | `~/Library/LaunchAgents/io.github.doronp.gitmemory.plist` | launchd, in your login session | + +It prints the file it wrote, the command line in it, and the commands below, +and runs none of them: no `systemctl`, no `launchctl`, no `loginctl`. Starting +it is yours to do, as installing the hook is. If the file is already there it +refuses, exit 1, and leaves it alone; the refusal ends with the line that +removes it, so paste that and install again. Any other platform exits 2, and so +does a store path with a line break in it. + +A service starts with none of your shell's state, so the file carries it: + +- **This Python, as it is,** with `-P -m gitmemory`. `-P` because a systemd + user unit starts in `$HOME`, and `python -m` would run a `~/gitmemory.py` in + place of the package. Not the interpreter's real path either, which is + outside the virtual environment and cannot import gitmemory. +- **The store, as `--home `.** A service does not read your shell's rc + files, so an exported `$GITMEMORY_HOME` never reaches it, and the watcher + would read `~/.gitmemory` while the hook writes somewhere else. If you move + the store, remove the file and install again. +- **`PATH`**: the directory `git` was found in, then `/usr/bin:/bin`. No git on + your `PATH` exits 2 and writes nothing. + +On Linux: + +```sh +systemctl --user daemon-reload && systemctl --user enable --now gitmemory.service # start, now and at every login +loginctl enable-linger # keep it running after you log out +systemctl --user status gitmemory.service # check it +journalctl --user -u gitmemory.service -f # its log +systemctl --user restart gitmemory.service # after upgrading gitmemory +systemctl --user disable --now gitmemory.service; rm ~/.config/systemd/user/gitmemory.service +``` + +On macOS: + +```sh +launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/io.github.doronp.gitmemory.plist # start, now and at every login +launchctl print gui/$(id -u)/io.github.doronp.gitmemory # check it +tail -f ~/Library/Logs/gitmemory.log # its log +launchctl kickstart -k gui/$(id -u)/io.github.doronp.gitmemory # after upgrading gitmemory +launchctl bootout gui/$(id -u)/io.github.doronp.gitmemory; rm ~/Library/LaunchAgents/io.github.doronp.gitmemory.plist +``` + +- **Linger** matters on a machine you reach only over SSH: without it, systemd + stops your user services when your last session ends. `loginctl + enable-linger` with no user name is for yourself, which stock systemd's + polkit policy allows without a password; a distribution can change that. +- **It is restarted after a failure, 30 seconds later** (`Restart=on-failure`; + `KeepAlive` on an unsuccessful exit). A watcher that cannot start — a store + inside a git work tree, say — logs one line every 30 seconds until fixed. +- **Editing `config.toml` needs no restart;** the watcher re-reads it every + pass. Upgrading gitmemory does, because the running process keeps the old + code. +- **The macOS log is outside the store,** which commits everything in it. + Nothing rotates it. On Linux the log is the journal. +- **Removing it stops it, then removes the file whether or not that worked.** + A file written and never started has nothing to stop: `launchctl bootout` + says `No such process` and exits 3, and the `rm` after the `;` runs anyway. ## Check it diff --git a/skills/recall/SKILL.md b/skills/recall/SKILL.md index e042a76..93db448 100644 --- a/skills/recall/SKILL.md +++ b/skills/recall/SKILL.md @@ -114,7 +114,9 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. 2. `gitmemory init claude-code`, which adds the `[[watch]]` table for `~/.claude/projects` to `config.toml` in the store (`$GITMEMORY_HOME`, by default `~/.gitmemory`) - 3. `gitmemory watch`, left running. + 3. `gitmemory service install`, which writes a user service that keeps + `gitmemory watch` running and prints the line that starts it; or + `gitmemory watch` in a terminal left open. The [Quickstart](https://github.com/doronp/gitmemory#quickstart) has the rest. @@ -146,9 +148,8 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. ## Read-only Run `gitmemory --version`, `gitmemory recall` and `gitmemory index`, and read -files under ``. -`index` writes only the derived index under `/index/`. Do not run -`gitmemory init`, `gitmemory capture`, `gitmemory watch`, `gitmemory derive`, -`gitmemory push` or `gitmemory dashboard`, and never -`gitmemory dashboard --expose`. Do not edit `config.toml` or anything else in -the store. +files under ``. `index` writes only the derived index under +`/index/`. Do not run `gitmemory init`, `gitmemory capture`, +`gitmemory watch`, `gitmemory service`, `gitmemory derive`, `gitmemory push` +or `gitmemory dashboard`, and never `gitmemory dashboard --expose`. Do not +edit `config.toml` or anything else in the store. diff --git a/src/gitmemory/__main__.py b/src/gitmemory/__main__.py index 423886b..7fba177 100644 --- a/src/gitmemory/__main__.py +++ b/src/gitmemory/__main__.py @@ -14,13 +14,14 @@ import builtins import ipaddress import os +import shlex import socket import sqlite3 import stat import sys import tomllib -from . import __version__, daemon, dashboard, derive, gitrepo, index, redact, store +from . import __version__, daemon, dashboard, derive, gitrepo, index, redact, service, store from .records import safe_text @@ -220,6 +221,20 @@ def _watch(args) -> int: ) +def _service(args) -> int: + home = store.resolve_home(args.home) + target, body, steps = service.render(sys.platform, home) + if not service.write(target, body): + # One line, with the way out last on it so it pastes whole: the + # removal step, which does not wait on stopping a job never started. + way_out = steps[-1][1] + return _refuse(f"{target} exists; to replace it, run this and install again: {way_out}") + print(f"wrote {target}\nit runs: {shlex.join(service.argv(home))}") + for what, command in steps: + print(f"{what}:\n {command}") + return 0 + + def _not_a_store(home: str | None) -> int | None: """Exit 2 and say so, rather than letting an empty glob read as a clean store. @@ -462,6 +477,13 @@ def main(argv: list[str] | None = None) -> int: wat.add_argument("--no-parse", action="store_true", help="skip adapter; record bytes only") wat.set_defaults(fn=_watch) + svc = sub.add_parser("service", help="keep `watch` running under your user service manager") + act = svc.add_subparsers(dest="action", required=True) + act.add_parser( + "install", + help="write a systemd user unit (Linux) or a LaunchAgent (macOS); starts nothing", + ).set_defaults(fn=_service) + ver = sub.add_parser("verify", help="check every manifest's contiguity proof") ver.set_defaults(fn=_verify) diff --git a/src/gitmemory/service.py b/src/gitmemory/service.py new file mode 100644 index 0000000..91416e8 --- /dev/null +++ b/src/gitmemory/service.py @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: 2026 The gitmemory authors +# SPDX-License-Identifier: Apache-2.0 +"""`gitmemory service install`: the file that keeps `watch` running. + +A systemd user unit on Linux, a LaunchAgent on macOS. Written, never enabled: +the command prints the line that starts it and runs no `systemctl`, +`launchctl` or `loginctl` itself. Installing it is the user's action, which is +the hook's rule too (hook/README.md), and a service manager's failures read +better in the user's own terminal than translated here. + +What it is for is the facts only the running process has. A user service gets +none of the shell's state — not its working directory, not an exported +`$GITMEMORY_HOME`, not its `PATH` — so a hand-written unit can start a +different Python, watch a different store, or not find git. Each of those was +reproduced; each is written into the file instead of inherited. +""" + +from __future__ import annotations + +import os +import plistlib +import shlex +import shutil +import sys + +LABEL = "io.github.doronp.gitmemory" +UNIT = "gitmemory.service" +DOCS = "https://github.com/doronp/gitmemory/blob/main/docs/watching.md#keep-it-running" + + +def argv(home: str) -> list[str]: + """The watcher's command line, for a manager that inherits nothing. + + `sys.executable` as given: its `realpath` is the base interpreter outside + the venv, which cannot import gitmemory. `-P`, because a systemd user unit + starts in `$HOME` and `python -m` puts that first on `sys.path`, so a stray + `~/gitmemory.py` ran in place of the package. `--home` always, resolved + now: the manager never sees the shell's `$GITMEMORY_HOME`, and the watcher + would otherwise read a store the hook does not write to. + """ + if not sys.executable: + # Empty or None when Python cannot say where it is; a unit naming "" + # cannot start. + raise RuntimeError( + "cannot tell which Python is running; run this from an installed gitmemory" + ) + return [sys.executable, "-P", "-m", "gitmemory", "--home", home, "watch"] + + +def _word(text: str) -> str: + """One double-quoted word in a unit file (systemd.syntax(7)): `\\` and `"` + escaped, `%` doubled so a store path spelling `%h` is not a specifier.""" + return '"' + text.replace("\\", "\\\\").replace('"', '\\"').replace("%", "%%") + '"' + + +def render(platform: str, home: str) -> tuple[str, bytes, list[tuple[str, str]]]: + """`(path, contents, steps)`: where the file goes, its bytes, and the + `(what, command)` lines the user runs, the removal last.""" + if platform not in ("linux", "darwin"): + raise ValueError( + f"no user service manager known for {platform}; " + "run `gitmemory watch` under your own supervisor" + ) + git = shutil.which("git") + if git is None: + raise RuntimeError("git is not on PATH; the watcher needs it to commit") + # gitrepo finds git through PATH, and a manager's PATH is not the shell's: + # a git from Homebrew or nix is on one and not the other. Absolute, since a + # relative PATH entry would resolve against the service's working + # directory; not realpath, so a symlinked git keeps the shell's spelling. + path = f"{os.path.abspath(os.path.dirname(git))}:/usr/bin:/bin" + words = argv(home) + # Checked once for both: a unit file would end the line there, and plistlib + # turns `\r` into `\n`, so a plist would name a path that does not exist. + for word in (*words, path): + if "\n" in word or "\r" in word: + raise ValueError(f"a service file cannot hold a line break: {word!r}") + if platform == "linux": + # The XDG rule: a relative value is invalid and is ignored, and a unit + # written relative to wherever this ran is one no user manager reads. + base = os.environ.get("XDG_CONFIG_HOME", "") + if not os.path.isabs(base): + base = os.path.expanduser("~/.config") + target = os.path.join(base, "systemd", "user", UNIT) + # `$` is expanded on a command line and has no meaning in Environment=, + # so it is doubled in one and not the other. No WorkingDirectory: a + # user unit's default is `$HOME`, which is what the plist below sets. + body = ( + "[Unit]\n" + "Description=gitmemory watcher\n" + f"Documentation={DOCS}\n" + "\n" + "[Service]\n" + f"ExecStart={' '.join(_word(w.replace('$', '$$')) for w in words)}\n" + f"Environment={_word('PATH=' + path)}\n" + "Restart=on-failure\n" + "RestartSec=30\n" + "\n" + "[Install]\n" + "WantedBy=default.target\n" + ).encode() + ctl = "systemctl --user" + steps = [ + ("Nothing is running yet. Start it now and at every login", + f"{ctl} daemon-reload && {ctl} enable --now {UNIT}"), + ("Keep it running after you log out, on a machine you reach over SSH", + "loginctl enable-linger"), + ("Check it", f"{ctl} status {UNIT}"), + ("Its log", f"journalctl --user -u {UNIT} -f"), + ("After upgrading gitmemory", f"{ctl} restart {UNIT}"), + # `;`: the file goes even when the manager cannot be reached. + ("Remove it", f"{ctl} disable --now {UNIT}; rm {shlex.quote(target)}"), + ] + return target, body, steps + target = os.path.expanduser(f"~/Library/LaunchAgents/{LABEL}.plist") + # Not in the store: the watcher commits it with `git add --all`. + log = os.path.expanduser("~/Library/Logs/gitmemory.log") + body = plistlib.dumps( + { + "Label": LABEL, + "ProgramArguments": words, + "EnvironmentVariables": {"PATH": path}, + # launchd starts a job in `/`; a relative watch root should resolve + # where it does under systemd. + "WorkingDirectory": os.path.expanduser("~"), + "RunAtLoad": True, + "KeepAlive": {"SuccessfulExit": False}, + "ThrottleInterval": 30, + "StandardOutPath": log, + "StandardErrorPath": log, + } + ) + job = f"gui/$(id -u)/{LABEL}" + # `;`, not `&&`, in the removal: `bootout` of a job that was written and + # never bootstrapped exits 3, and the file has to go all the same. + steps = [ + ("Nothing is running yet. Start it now and at every login", + f"launchctl bootstrap gui/$(id -u) {shlex.quote(target)}"), + ("Check it", f"launchctl print {job}"), + ("Its log", f"tail -f {shlex.quote(log)}"), + ("After upgrading gitmemory", f"launchctl kickstart -k {job}"), + ("Remove it", f"launchctl bootout {job}; rm {shlex.quote(target)}"), + ] + return target, body, steps + + +def write(path: str, body: bytes) -> bool: + """Create `path` with `body`, or return False and touch nothing if it exists. + + `O_EXCL`, not a check then a write: a file already there may be one the + user edited, and a second install is not a reason to lose it. + """ + os.makedirs(os.path.dirname(path), exist_ok=True) + try: + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) + except FileExistsError: + return False + with os.fdopen(fd, "wb") as fh: + fh.write(body) + return True diff --git a/tests/mutate_index.py b/tests/mutate_index.py index 624e96c..6c15dde 100644 --- a/tests/mutate_index.py +++ b/tests/mutate_index.py @@ -4597,7 +4597,7 @@ # anchor appear twice and the harness would skip itself. [E7] "the docstring's row count drifts away from the index again", "tests/mutate_index.py", - "a full pass is 6" + "17 mutants", + "a full pass is 6" + "23 mutants", "a full pass is 14" + "6 mutants", "test_the_row_count_in_the_docstring_is_the_row_count", ), @@ -5590,6 +5590,51 @@ "", "test_cli_init_leaves_no_temp_the_watcher_would_commit", ), + # --- `gitmemory service install`: the file a user service manager runs --- + ( + "service install overwrites a unit file that is already there", + "service.py", + "os.O_WRONLY | os.O_CREAT | os.O_EXCL", + "os.O_WRONLY | os.O_CREAT | os.O_TRUNC", + "test_service_install_refuses_to_overwrite_an_existing_file", + ), + ( + "the service runs python -m without -P, so a gitmemory.py in $HOME shadows it", + "service.py", + 'sys.executable, "-P", "-m", "gitmemory"', + 'sys.executable, "-m", "gitmemory"', + "test_the_installed_service_runs_the_watcher_from_its_own_working_directory", + ), + ( + "the service resolves sys.executable to an interpreter outside the venv", + "service.py", + "return [sys.executable, ", + "return [os.path.realpath(sys.executable), ", + "test_the_installed_service_runs_the_watcher_from_its_own_working_directory", + ), + ( + # The quoting test's decoder used to turn `%%` into `%`, which let an + # undoubled `%h` through as itself: this mutant survived. [review] + "the unit leaves % undoubled, so a store path spelling %h names the home directory", + "service.py", + '.replace("%", "%%")', + '.replace("%", "%")', + "test_a_unit_survives_a_home_with_quotes_percent_and_dollar_in_it", + ), + ( + "ExecStart leaves $ undoubled, so a store path spelling $HOME is expanded", + "service.py", + "_word(w.replace('$', '$$'))", + "_word(w)", + "test_a_unit_survives_a_home_with_quotes_percent_and_dollar_in_it", + ), + ( + "the printed removal waits on bootout, so a plist never bootstrapped is never removed", + "service.py", + 'f"launchctl bootout {job}; rm {shlex.quote(target)}"', + 'f"launchctl bootout {job} && rm {shlex.quote(target)}"', + "test_the_printed_removal_works_when_the_job_was_never_started", + ), ] @@ -5742,7 +5787,7 @@ def _select(test: str) -> list[str]: def main() -> int: """Run every mutant, or only those whose name contains an argument. - The filter exists because a full pass is 617 mutants × two suite runs — + The filter exists because a full pass is 623 mutants × two suite runs — long enough that adding one row and checking it used to mean either waiting for the other 567 or trusting the new one untested. diff --git a/tests/test_service.py b/tests/test_service.py new file mode 100644 index 0000000..6de417d --- /dev/null +++ b/tests/test_service.py @@ -0,0 +1,355 @@ +# SPDX-FileCopyrightText: 2026 The gitmemory authors +# SPDX-License-Identifier: Apache-2.0 +"""`gitmemory service install`: one inert file that keeps `watch` running. + +The property under test: what a user service manager starts is the watcher the +user's shell would have run. systemd and launchd hand a job none of the shell's +state — not its working directory, not `$GITMEMORY_HOME`, not its `PATH` — so +everything the watcher needs has to be written into the file, and the only +proof that it was is to run the file's own command line under the manager's +conditions. That is the end-to-end test below; the others pin what each +manager reads and that the command writes, refuses and never starts. + +Nothing here runs `systemctl`, `launchctl` or `loginctl`. HOME comes from +conftest and XDG_CONFIG_HOME is set or unset per test, so every file lands in +tmp_path, and both renderers run on either OS by setting `sys.platform`. The +one test that runs a printed command puts stand-ins for the managers first on +its PATH. +""" + +from __future__ import annotations + +import os +import plistlib +import re +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from gitmemory.__main__ import main + +LABEL = "io.github.doronp.gitmemory" + + +@pytest.fixture +def xdg(tmp_path, monkeypatch): + path = tmp_path / "xdg" + monkeypatch.setenv("XDG_CONFIG_HOME", str(path)) + return path + + +@pytest.fixture +def no_spawn(monkeypatch): + """Fail on any child process: the command writes a file and starts nothing. + + `Popen` is what `run`, `call` and `check_output` all construct, so one patch + covers every way the module could reach `systemctl` or `launchctl`. + """ + + def spawned(*args, **kwargs): + raise AssertionError(f"service install spawned {args!r}") + + monkeypatch.setattr(subprocess, "Popen", spawned) + + +def _install(monkeypatch, platform: str, *argv: str) -> int: + with monkeypatch.context() as m: + m.setattr(sys, "platform", platform) + return main([*argv, "service", "install"]) + + +def _unit(path) -> dict[str, str]: + """`key -> value` for a unit file's assignments; sections are not needed.""" + lines = Path(path).read_text(encoding="utf-8").splitlines() + return dict(line.split("=", 1) for line in lines if "=" in line) + + +_QUOTED = re.compile(r'"((?:[^"\\]|\\.)*)"(?: |$)', re.S) + + +def _words(value: str, *, dollars: bool) -> list[str]: + """Read a unit's words back as systemd.syntax(7) says systemd does, and + fail on anything it would read as something else. + + Strict, because a lenient reader is what made the quoting test decorative: + turning `%%` into `%` also lets an undoubled `%h` through as itself, so the + round trip held whether or not the unit escaped anything. To systemd a `%h` + is a specifier, a lone `$` on a command line is a variable, and `\\z` is an + escape it rejects, so each is a failure here. Order as systemd's: unquote, + then specifiers, then (`ExecStart=` only) variables. The test's reading of + the manual, not systemd's parser; there is no systemd to ask on a Mac. + """ + + def strict(pattern: str, keep: tuple[str, ...], word: str) -> str: + def one(m: re.Match) -> str: + assert m.group(1) in keep, f"systemd reads {m.group(0)!r} in {value!r}" + return m.group(1) + + return re.sub(pattern, one, word, flags=re.S) + + words, pos = [], 0 + while pos < len(value): + m = _QUOTED.match(value, pos) + assert m, f"not a double-quoted word at {pos}: {value!r}" + word = strict(r"%(.?)", ("%",), strict(r"\\(.?)", ("\\", '"'), m.group(1))) + words.append(strict(r"\$(.?)", ("$",), word) if dollars else word) + pos = m.end() + return words + + +def _expected(home) -> list[str]: + return [sys.executable, "-P", "-m", "gitmemory", "--home", os.path.realpath(home), "watch"] + + +def test_service_install_writes_a_systemd_user_unit_and_starts_nothing( + tmp_path, monkeypatch, capsys, xdg, no_spawn +): + home = tmp_path / "store" + assert _install(monkeypatch, "linux", "--home", str(home)) == 0 + path = xdg / "systemd" / "user" / "gitmemory.service" + unit = _unit(path) + assert _words(unit["ExecStart"], dollars=True) == _expected(home) + env = _words(unit["Environment"], dollars=False) + assert env == [f"PATH={os.path.dirname(shutil.which('git'))}:/usr/bin:/bin"], env + assert unit["Restart"] == "on-failure" and unit["RestartSec"] == "30" + assert unit["WantedBy"] == "default.target" + out, err = capsys.readouterr() + assert out.startswith(f"wrote {path}\n") and not err + ctl = "systemctl --user" + assert f"{ctl} daemon-reload && {ctl} enable --now gitmemory.service" in out + assert "loginctl enable-linger" in out + assert f"{ctl} disable --now gitmemory.service; rm {shlex.quote(str(path))}" in out + + +def test_service_install_writes_a_launch_agent_and_starts_nothing( + tmp_path, monkeypatch, capsys, no_spawn +): + """No `--home`: the store comes from `$GITMEMORY_HOME`, which a LaunchAgent + started at login never sees, so the file has to name it outright.""" + home = tmp_path / "elsewhere" + monkeypatch.setenv("GITMEMORY_HOME", str(home)) + assert _install(monkeypatch, "darwin") == 0 + user = os.environ["HOME"] + path = os.path.join(user, "Library", "LaunchAgents", f"{LABEL}.plist") + with open(path, "rb") as fh: + job = plistlib.load(fh) + assert job["Label"] == LABEL + assert job["ProgramArguments"] == _expected(home) + git = os.path.dirname(shutil.which("git")) + assert job["EnvironmentVariables"] == {"PATH": f"{git}:/usr/bin:/bin"} + assert job["WorkingDirectory"] == user + assert job["RunAtLoad"] is True and job["KeepAlive"] == {"SuccessfulExit": False} + assert job["ThrottleInterval"] == 30 + # Outside the store, which commits everything in it with `git add --all`. + log = os.path.join(user, "Library", "Logs", "gitmemory.log") + assert job["StandardOutPath"] == job["StandardErrorPath"] == log + out, _ = capsys.readouterr() + assert f"launchctl bootstrap gui/$(id -u) {shlex.quote(path)}" in out + assert f"launchctl bootout gui/$(id -u)/{LABEL}; rm {shlex.quote(path)}" in out + + +@pytest.mark.parametrize("platform", ["linux", "darwin"]) +def test_the_installed_service_runs_the_watcher_from_its_own_working_directory( + tmp_path, monkeypatch, capsys, xdg, platform +): + """Run the file's command line the way its manager would, plus `--once`. + + The manager's conditions are: the working directory is `$HOME` (systemd's + default for a user unit, and what the plist sets), the environment is HOME + and the file's PATH and nothing else, and `$HOME` holds a `gitmemory.py`. + Each is a trap a hand-written unit fell into: `realpath(sys.executable)` + leaves the venv and cannot import gitmemory; `python -m` without `-P` puts + the working directory first on `sys.path` and runs the stray file instead; + and without `--home` the watcher reads `~/.gitmemory`, not the store the + user configured, and watches nothing. + """ + home, roots = tmp_path / "store", tmp_path / "roots" + os.makedirs(home) + os.makedirs(roots / "proj") + (roots / "proj" / "s1.jsonl").write_text( + '{"type":"user","message":{"role":"user","content":"hello"}}\n' + ) + (home / "config.toml").write_text(f'[[watch]]\nagent = "claude-code"\nroots = ["{roots}"]\n') + user = os.environ["HOME"] + with open(os.path.join(user, "gitmemory.py"), "w") as fh: + fh.write("raise SystemExit('a gitmemory.py in the working directory ran instead')\n") + assert _install(monkeypatch, platform, "--home", str(home)) == 0 + if platform == "linux": + unit = _unit(xdg / "systemd" / "user" / "gitmemory.service") + argv = _words(unit["ExecStart"], dollars=True) + path = _words(unit["Environment"], dollars=False)[0].removeprefix("PATH=") + cwd = user + else: + with open(os.path.join(user, "Library", "LaunchAgents", f"{LABEL}.plist"), "rb") as fh: + job = plistlib.load(fh) + argv, path = job["ProgramArguments"], job["EnvironmentVariables"]["PATH"] + cwd = job["WorkingDirectory"] + capsys.readouterr() + done = subprocess.run( + [*argv, "--once"], + cwd=cwd, + env={"HOME": user, "PATH": path}, + capture_output=True, + text=True, + timeout=120, + ) + assert done.returncode == 0, done.stderr + assert "captured 1 " in done.stderr, done.stderr + assert main(["--home", str(home), "verify"]) == 0 + + +@pytest.mark.parametrize("platform", ["linux", "darwin"]) +def test_service_install_refuses_to_overwrite_an_existing_file( + tmp_path, monkeypatch, capsys, xdg, no_spawn, platform +): + """Exit 1, the file untouched, and one line that says how to replace it. + + A file already there may be one the user edited; a second install is not a + reason to lose that.""" + if platform == "linux": + path = xdg / "systemd" / "user" / "gitmemory.service" + else: + path = Path(os.environ["HOME"], "Library", "LaunchAgents", f"{LABEL}.plist") + os.makedirs(path.parent) + path.write_bytes(b"the user's own\n") + assert _install(monkeypatch, platform, "--home", str(tmp_path / "store")) == 1 + assert path.read_bytes() == b"the user's own\n" + out, err = capsys.readouterr() + assert not out and err.count("\n") == 1, err + assert err.startswith(f"refusing: {path} exists") and f"rm {shlex.quote(str(path))}" in err + + +def test_a_unit_survives_a_home_with_quotes_percent_and_dollar_in_it( + tmp_path, monkeypatch, capsys, xdg, no_spawn +): + """systemd expands `%h` and `$HOME` inside a quoted word and rejects `\\z`, + so a store whose path spells any of them would be read as some other path, + or the unit not read at all. `_words` fails on each unless it is escaped.""" + home = tmp_path / 'a b %h $HOME "q" \\z' + assert _install(monkeypatch, "linux", "--home", str(home)) == 0 + path = xdg / "systemd" / "user" / "gitmemory.service" + assert _words(_unit(path)["ExecStart"], dollars=True) == _expected(home) + + +@pytest.mark.parametrize("platform", ["linux", "darwin"]) +@pytest.mark.parametrize("brk", ["\n", "\r"], ids=["lf", "cr"]) +def test_a_line_break_in_the_store_path_is_refused_on_either_platform( + tmp_path, monkeypatch, capsys, xdg, no_spawn, platform, brk +): + """Exit 2 and no file. A unit file would end the line there; a plist holds + `\\n`, but plistlib turns `\\r` into `\\n`, so that service would watch a + path that does not exist rather than refuse.""" + assert _install(monkeypatch, platform, "--home", str(tmp_path / f"a{brk}b")) == 2 + assert capsys.readouterr().err.startswith("error: ") + assert not os.path.exists(xdg) and not os.listdir(os.environ["HOME"]) + + +@pytest.mark.parametrize("platform", ["linux", "darwin"]) +def test_the_printed_removal_works_when_the_job_was_never_started( + tmp_path, monkeypatch, capsys, xdg, platform +): + """The refusal's way out, pasted, removes the file even when the stop fails. + + Written and never started is the state a second install meets: the user + reads `it runs:`, sees the wrong store, and installs again. `launchctl + bootout` of a job that was never bootstrapped exits 3 ("No such process"), + and a user manager that cannot be reached fails `systemctl --user` too, so + a removal that waits on the stop leaves the file, and the next install is + refused with the same advice. The stand-ins below fail that way; the real + managers are never on the PATH they run with. + """ + if platform == "linux": + path = xdg / "systemd" / "user" / "gitmemory.service" + else: + path = Path(os.environ["HOME"], "Library", "LaunchAgents", f"{LABEL}.plist") + store = str(tmp_path / "store") + assert _install(monkeypatch, platform, "--home", store) == 0 + out = capsys.readouterr().out + printed = out.split("Remove it:\n ", 1)[1].rstrip("\n") + assert _install(monkeypatch, platform, "--home", store) == 1 + refusal = capsys.readouterr().err + assert refusal.rstrip("\n").endswith(f": {printed}"), refusal + stubs = tmp_path / "stubs" + stubs.mkdir() + fails = "#!/bin/sh\necho 'Boot-out failed: 3: No such process' >&2\nexit 3\n" + for manager in ("systemctl", "launchctl"): + (stubs / manager).write_text(fails) + (stubs / manager).chmod(0o755) + done = subprocess.run( + ["/bin/sh", "-c", printed], + env={"HOME": os.environ["HOME"], "PATH": f"{stubs}:/usr/bin:/bin"}, + capture_output=True, + text=True, + timeout=30, + ) + assert not path.exists(), (printed, done.stderr) + assert _install(monkeypatch, platform, "--home", store) == 0 + + +@pytest.mark.parametrize("value", [None, "", "rel"], ids=["unset", "empty", "relative"]) +def test_a_unit_lands_under_dot_config_without_an_absolute_xdg_config_home( + tmp_path, monkeypatch, capsys, no_spawn, value +): + """The default most Linux users get, and the XDG rule that a relative + `$XDG_CONFIG_HOME` is invalid and ignored: a unit written relative to + wherever install ran is one no user manager looks for.""" + monkeypatch.chdir(tmp_path) + if value is None: + monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) + else: + monkeypatch.setenv("XDG_CONFIG_HOME", value) + assert _install(monkeypatch, "linux", "--home", str(tmp_path / "store")) == 0 + path = os.path.join(os.environ["HOME"], ".config", "systemd", "user", "gitmemory.service") + assert _words(_unit(path)["ExecStart"], dollars=True) == _expected(tmp_path / "store") + assert not (tmp_path / "rel").exists() + out = capsys.readouterr().out + assert out.startswith(f"wrote {path}\n") and f"; rm {shlex.quote(path)}\n" in out + + +@pytest.mark.parametrize("platform", ["linux", "darwin"]) +def test_git_found_through_a_relative_path_entry_is_written_absolute( + tmp_path, monkeypatch, capsys, xdg, no_spawn, platform +): + """`PATH=relbin:...` makes `which` answer `relbin/git`. Written as it is, + the service would resolve it against its own working directory, `$HOME`, + not the directory install ran in.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(shutil, "which", lambda name, *a, **k: os.path.join("relbin", name)) + assert _install(monkeypatch, platform, "--home", str(tmp_path / "store")) == 0 + if platform == "linux": + unit = _unit(xdg / "systemd" / "user" / "gitmemory.service") + path = _words(unit["Environment"], dollars=False)[0].removeprefix("PATH=") + else: + plist = Path(os.environ["HOME"], "Library", "LaunchAgents", f"{LABEL}.plist") + path = plistlib.loads(plist.read_bytes())["EnvironmentVariables"]["PATH"] + assert path == f"{os.path.join(os.getcwd(), 'relbin')}:/usr/bin:/bin" + + +@pytest.mark.parametrize("exe", ["", None], ids=["empty", "none"]) +def test_service_install_refuses_when_it_cannot_tell_which_python_runs( + tmp_path, monkeypatch, capsys, xdg, no_spawn, exe +): + """`sys.executable` is empty or None when Python cannot say where it is. + A unit naming `""` cannot start, and None was a traceback; exit 2 instead, + with nothing written.""" + monkeypatch.setattr(sys, "executable", exe) + assert _install(monkeypatch, "linux", "--home", str(tmp_path / "store")) == 2 + assert capsys.readouterr().err.startswith("error: ") + assert not os.path.exists(xdg) + + +def test_service_install_without_git_or_on_an_unknown_platform_writes_nothing( + tmp_path, monkeypatch, capsys, xdg, no_spawn +): + """Exit 2 either way: the watcher commits with git, and it would find none.""" + home = str(tmp_path / "store") + assert _install(monkeypatch, "freebsd14", "--home", home) == 2 + assert "freebsd14" in capsys.readouterr().err + monkeypatch.setattr(shutil, "which", lambda name, *a, **k: None) + assert _install(monkeypatch, "linux", "--home", home) == 2 + assert "git" in capsys.readouterr().err + assert not os.path.exists(xdg) and not os.listdir(os.environ["HOME"]) From 5c6a874fe3c42fedc4065f9b086436f147394ea3 Mon Sep 17 00:00:00 2001 From: doronp <3586743+doronp@users.noreply.github.com> Date: Sun, 18 Oct 2026 13:33:36 +0300 Subject: [PATCH 06/13] feat(recall): say when hook records have waited an hour for a watcher recall checked nothing about capture, so a stopped watcher was invisible from it: every search answered from the store as it was when the watcher stopped, with an empty stderr. The time since the last capture cannot tell that from an idle machine, because a running watcher on an idle machine captures nothing either. The spool can: every watcher pass drains every complete record, covered or not, so a record an hour old means no pass has run on that store since it arrived. daemon.STALE_RECORD and a read-only daemon.uncollected() count the spool's complete records and the age of the oldest, by drain_spool's own rule (a dot-name is a hook still writing). A directory is not counted either: no pass can unlink one, so it would warn for ever with a watcher running. recall prints one stderr line before the results when that age passes the threshold, naming the store, so a hook and a watcher on different stores show up too. Exit code unchanged, and the subagent label on result lines is untouched. Skipped under --db, whose index names no store, and silent on any store the hook does not ring: capture-only stores, tests and bench. The skill quotes the line and tells the user to start `gitmemory watch` and keep it running with `gitmemory service install`; it runs neither. It does not rule out a reindex, which still finds what the watcher captured before it stopped. docs/watching.md names the other causes: a service that is installed but not started or failing, one whose --home is another store, and a watcher that runs passes less often than hourly. Three tests, seven negative controls (623 -> 630), and the README and REPRODUCE.md counts, measured. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QPBLmJD2q1PVxdkWx8C16U Signed-off-by: doronp <3586743+doronp@users.noreply.github.com> --- CHANGELOG.md | 7 ++++ README.md | 8 ++-- docs/REPRODUCE.md | 8 ++-- docs/USAGE.md | 11 +++++ docs/watching.md | 1 + hook/README.md | 5 +++ skills/recall/SKILL.md | 10 +++++ src/gitmemory/__main__.py | 21 ++++++++++ src/gitmemory/daemon.py | 28 +++++++++++++ tests/mutate_index.py | 54 +++++++++++++++++++++++- tests/test_docs.py | 11 ++++- tests/test_index.py | 87 +++++++++++++++++++++++++++++++++++++++ 12 files changed, 240 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed0b04f..ee58e44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,13 @@ tell you. gives a Claude Code watch pattern, `pattern = "*/*.jsonl"`, that leaves subagent and workflow transcripts out, and what that costs: those turns are never captured, and the dashboard's spend leaves out what they cost. +- **`recall` says when no watcher is collecting.** When a hook record has + waited in the spool for more than an hour, `recall` prints one line on + stderr before the results, naming the store and asking whether + `gitmemory watch` is running for it. The time since the last capture cannot + tell a stopped watcher from an idle one; a record no pass has drained can. + Not under `--db`, and never on a store the hook does not ring. See + [docs/watching.md](docs/watching.md#check-it). ### Changed diff --git a/README.md b/README.md index 1f45e93..b519932 100644 --- a/README.md +++ b/README.md @@ -209,13 +209,13 @@ real text, and that is a measurement rather than a suspicion. | What | How it was measured | Result | |---|---|---| | Hook cost in the agent's critical path | Timed against spawning `true` the same way, three runs of 400 | p50 **7.4 – 7.5 ms**, p99 **10.2 – 11.5 ms** | -| The suite | On a fresh checkout, no downloads | **1,084 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | -| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **623** negative controls | +| The suite | On a fresh checkout, no downloads | **1,087 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | +| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **630** negative controls | The two suite counts do not add up, and should not. Switching the corpora on -collects 1409, not 1412. Three conformance cases fill parametrisations that +collects 1412, not 1415. Three conformance cases fill parametrisations that collect as one empty placeholder each while the corpora are absent, so they -replace three of the 1084 rather than joining them. Every figure on this board +replace three of the 1087 rather than joining them. Every figure on this board is pinned by a test, which is how it stays true. **What is not measured**, and is shown as a row on the dashboard rather than diff --git a/docs/REPRODUCE.md b/docs/REPRODUCE.md index 3e51eea..b660691 100644 --- a/docs/REPRODUCE.md +++ b/docs/REPRODUCE.md @@ -484,11 +484,11 @@ uv run pytest -q --collect-only -p no:cacheprovider | tail -1 uv run pytest -q --collect-only -p no:cacheprovider | grep -c '\.jsonl\]$' ``` -**Expected:** `1084/1085 tests collected (1 deselected)`, then -`1409/1410 tests collected (1 deselected)`, then `328`. The deselected test is +**Expected:** `1087/1088 tests collected (1 deselected)`, then +`1412/1413 tests collected (1 deselected)`, then `328`. The deselected test is the LongMemEval corpus test, which is opt-in: `uv run pytest -q -m corpus` after `bench/fetch_longmemeval.sh`. The README's -paragraph on why 1,084 + 328 is 1,409 and not 1,412 is the arithmetic of these +paragraph on why 1,087 + 328 is 1,412 and not 1,415 is the arithmetic of these three numbers. All three were run while writing this guide. `tests/test_docs.py` holds the README to the first and third numbers, so a @@ -505,7 +505,7 @@ test fail. The count is the number of rows in `tests/mutate_index.py`. uv run python -c "import sys; sys.path.insert(0, 'tests'); from mutate_index import MUTANTS; print(len(MUTANTS))" ``` -**Expected:** `623`. Checked while writing this guide. +**Expected:** `630`. Checked while writing this guide. To run the controls rather than count them: diff --git a/docs/USAGE.md b/docs/USAGE.md index 5744ec5..64d6a85 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -89,6 +89,17 @@ newest first (see [When memories disagree](#when-memories-disagree)): `-k` sets how many lines come back. +If the [hook](../hook/README.md) has rung and no watcher has collected the +record for an hour, `recall` says so on stderr, before the results it still +prints: +``1 hook record(s) uncollected for 3 hours: is `gitmemory watch` running for ? These results may be missing everything said since.`` +A running watcher drains the spool on every pass, so none has run on that store +since: start `gitmemory watch` with the hook's `$GITMEMORY_HOME`, and +`gitmemory service install` to keep it running +([watching.md](watching.md#keep-it-running)). [Check it](watching.md#check-it) +covers the other causes, such as a watcher that runs passes less often than +hourly. With `--db`, recall does not look. + ## When memories disagree A decision made three weeks ago and reversed last week are both in the store, diff --git a/docs/watching.md b/docs/watching.md index 6274b92..5a72d43 100644 --- a/docs/watching.md +++ b/docs/watching.md @@ -235,6 +235,7 @@ anywhere useful. The two failures worth watching for are different shapes: |---|---| | `no .../config.toml; watching nothing` | The file is not where the watcher is looking. Check `$GITMEMORY_HOME`, or run `gitmemory init ` to write it there. | | Watcher quiet, store empty, no complaint | A root that **does not exist** is kept, not refused — before an agent has run for the first time that is the normal state, and `run` re-reads the config every pass, so it starts working the moment the directory appears. The cost is that a typo'd root and a not-yet-used one look identical. Check the path by hand; `gitmemory init` says so when the root it writes does not exist yet. | +| `recall` says `N hook record(s) uncollected for ` | No watcher has run a pass on that store for that long: it is stopped, it watches a store other than the hook's `$GITMEMORY_HOME` (a service's store is the `--home` written into it), or it runs a pass less often than once an hour (`--poll` of 3600 or more, or a supervisor re-running `--once` that seldom). Start `gitmemory watch` with the hook's `$GITMEMORY_HOME` and keep it running with `gitmemory service install` ([above](#keep-it-running)), or run passes more often. A service that is installed but not started, or failing, shows here too; its check and log lines are above. A pass drains the spool even when no watch covers the record, so silence here means a pass ran, not that this agent is captured. | | `compact_boundaries: []` everywhere | Adapter missing or `--no-parse`. The bytes are intact; only the boundaries are absent. | | `source changed while parsing` | The transcript was rewritten between the parse and the copy — a compaction landing mid-pass. The bytes are captured; that one generation has no boundaries. Nothing to do. | | `was captured from , not ; pass a distinct --session-id` | Two transcripts collided on one session name, and the store refuses to mix them rather than duplicating both for ever. The advice in the message is for the CLI and there is no `--session-id` to pass here: rename or move either file — the name is derived from the path — and the next pass captures both. Until then the second one is **not being captured**, so this is not a warning to leave running. | diff --git a/hook/README.md b/hook/README.md index df66aed..18f0bbc 100644 --- a/hook/README.md +++ b/hook/README.md @@ -206,6 +206,11 @@ it and then re-derives everything — how many bytes are new, where the boundary is, whether the file was rewritten — from the transcript itself. A record naming a path outside the configured watch roots is dropped. +Every pass removes every complete record, dropped or not, so a record older +than an hour means no watcher has run a pass on this store since it arrived, +and `gitmemory recall` says so on stderr. That is also how a hook and a watcher +with different `$GITMEMORY_HOME`s show up: the message names the store. + ## Latency The shim is in the agent's critical path, so it is measured rather than asserted. diff --git a/skills/recall/SKILL.md b/skills/recall/SKILL.md index 93db448..22799d4 100644 --- a/skills/recall/SKILL.md +++ b/skills/recall/SKILL.md @@ -107,6 +107,16 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. - The watcher captures; it does not index. If a search misses something the user says was said recently, the index may be older than the capture: run `gitmemory index` and search once more. +- `` hook record(s) uncollected for : is `gitmemory watch` running for ? These results may be missing everything said since.`` + on stderr, before the results, exit 0: the hook has rung and no watcher has + run a pass on that store since, so nothing said after that is being captured + there. Answer from the results, say they may be missing everything since, and + tell the user to start `gitmemory watch` with the `GITMEMORY_HOME` this + session has, and to keep it running with `gitmemory service install` (step 3 + below). If they have installed a service already, it is not started, it is + failing (its log says why), or it watches another store. Do not run any of + it yourself. Reindexing cannot bring back what was said since; if the hits + miss something older, the bullet above still applies. - The shell cannot find `gitmemory`, or `index` prints `not a store: `, or it reports `0 generation(s)`: nothing has been captured there. Tell the user how it gets set up, and stop. Do none of it yourself: diff --git a/src/gitmemory/__main__.py b/src/gitmemory/__main__.py index 7fba177..efda51d 100644 --- a/src/gitmemory/__main__.py +++ b/src/gitmemory/__main__.py @@ -393,6 +393,15 @@ def _derive(args) -> int: return 0 +def _ago(seconds: float) -> str: + """Minutes, hours or days. Said only past `STALE_RECORD`, so never one.""" + if seconds < 7200: + return f"{int(seconds // 60)} minutes" + if seconds < 172800: + return f"{int(seconds // 3600)} hours" + return f"{int(seconds // 86400)} days" + + def _recall(args) -> int: # First, so a mistyped policy is the error you see, with or without an index. resolve = index.policy(args.policy) @@ -400,6 +409,18 @@ def _recall(args) -> int: if not os.path.exists(path): print(f"no index at {path}; run `gitmemory index` first", file=sys.stderr) return 2 + # Nothing here said the watcher had stopped, and the time since the last + # capture cannot: a running one on an idle machine captures nothing either. + # A record no pass has drained can. Not under `--db`: an index names no store. + if args.db is None: + home = store.resolve_home(args.home) + waiting, age = daemon.uncollected(home) + if waiting and age >= daemon.STALE_RECORD: + print( + f"{waiting} hook record(s) uncollected for {_ago(age)}: is `gitmemory watch` " + f"running for {home}? These results may be missing everything said since.", + file=sys.stderr, + ) db = index.open_db(path) try: hits = index.search(db, args.query, k=args.k) diff --git a/src/gitmemory/daemon.py b/src/gitmemory/daemon.py index 7d3ce00..28b83cf 100644 --- a/src/gitmemory/daemon.py +++ b/src/gitmemory/daemon.py @@ -58,6 +58,11 @@ ERROR_REPEAT = 3600.0 POLL = 5.0 # seconds between ticks STALE_TMP = 3600.0 # a `.tmp-` this old is a killed hook, not a live one +# A complete spool record this old means no watcher pass has run on this store +# since it arrived: every pass drains every record, whatever it names. 720 +# default polls, and no further behind than `INTERVAL` already leaves a growing +# session. Read by `recall`. Its own constant, for `ERROR_REPEAT`'s reason. +STALE_RECORD = 3600.0 # A module constant, not `Watch.pattern`: on a `slots=True` dataclass the class # attribute is the slot's `member_descriptor`, so reading the default off the @@ -489,6 +494,29 @@ def drain_spool( return wanted, tick +def uncollected(home: str, now: float | None = None) -> tuple[int, float]: + """`(complete records in the spool, age of the oldest in seconds)`. + + `drain_spool`'s rule for what a record is: a dot-name is a hook still + writing. A directory is not one either: no pass can unlink it, so counting + it warned for ever with a watcher running. Arrival is the record's own + mtime (hook/README.md). Read-only and total, because `recall` asks: + anything it cannot list is `(0, 0.0)`. + """ + now = time.time() if now is None else now + ages = [] + try: + with os.scandir(os.path.join(home, SPOOL)) as entries: + for entry in entries: + if not entry.name.startswith("."): + with contextlib.suppress(OSError): + if not entry.is_dir(follow_symlinks=False): + ages.append(now - entry.stat(follow_symlinks=False).st_mtime) + except OSError: + return 0, 0.0 + return len(ages), max(ages, default=0.0) + + def _file_key(path: str) -> tuple[int, int] | None: """`(st_dev, st_ino)` — the identity two spellings of one file agree on.""" try: diff --git a/tests/mutate_index.py b/tests/mutate_index.py index 6c15dde..d552ff9 100644 --- a/tests/mutate_index.py +++ b/tests/mutate_index.py @@ -3852,6 +3852,56 @@ " if False:", "test_recall_says_a_query_was_truncated_before_it_says_no_matches", ), + # --- install feedback: the hook rang and no watcher answered ----------- # + ( + "recall says nothing when the hook rang and no watcher collected it", + "__main__.py", + " if waiting and age >= daemon.STALE_RECORD:", + " if False:", + "test_recall_says_the_hook_rang_and_no_watcher_answered", + ), + ( + "recall reports a spool record a watcher could still be draining", + "__main__.py", + " if waiting and age >= daemon.STALE_RECORD:", + " if waiting:", + "test_recall_says_nothing_about_a_spool_a_watcher_could_still_be_draining", + ), + ( + "a hook's temp file counts as an uncollected record", + "daemon.py", + ' if not entry.name.startswith("."):', + " if True:", + "test_recall_says_nothing_about_a_spool_a_watcher_could_still_be_draining", + ), + ( + "a directory no pass can unlink counts as an uncollected record", + "daemon.py", + " if not entry.is_dir(follow_symlinks=False):", + " if True:", + "test_one_watcher_pass_is_what_clears_it", + ), + ( + "recall reads a store's spool for an index named by --db", + "__main__.py", + " if args.db is None:", + " if True:", + "test_recall_says_nothing_about_a_spool_a_watcher_could_still_be_draining", + ), + ( + "the uncollected age is the newest record's, not the oldest's", + "daemon.py", + "max(ages, default=0.0)", + "min(ages, default=0.0)", + "test_recall_says_the_hook_rang_and_no_watcher_answered", + ), + ( + "a watcher pass leaves the records it consumed in the spool", + "daemon.py", + " _unlink(path)\n tick.spool_consumed += 1", + " tick.spool_consumed += 1", + "test_one_watcher_pass_is_what_clears_it", + ), # --- E7 index-F8: the caps bound the shape of the input, not its cost -- # ( "sumy's own quadratic idf is used again", @@ -4597,7 +4647,7 @@ # anchor appear twice and the harness would skip itself. [E7] "the docstring's row count drifts away from the index again", "tests/mutate_index.py", - "a full pass is 6" + "23 mutants", + "a full pass is 6" + "30 mutants", "a full pass is 14" + "6 mutants", "test_the_row_count_in_the_docstring_is_the_row_count", ), @@ -5787,7 +5837,7 @@ def _select(test: str) -> list[str]: def main() -> int: """Run every mutant, or only those whose name contains an argument. - The filter exists because a full pass is 623 mutants × two suite runs — + The filter exists because a full pass is 630 mutants × two suite runs — long enough that adding one row and checking it used to mean either waiting for the other 567 or trusting the new one untested. diff --git a/tests/test_docs.py b/tests/test_docs.py index bd8f80f..10cabb4 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -558,7 +558,8 @@ def test_the_plugin_recall_skill_runs_only_commands_the_cli_has(tmp_path, capsys The commands it forbids are checked the same way, because a prohibition on a command that no longer exists guards nothing. The messages it tells the agent to branch on are the ones the CLI prints, got by running it on a path - that is no store and on an empty one. + that is no store, on an empty one, and on one whose spool no watcher has + drained. `skills/` holds this one skill and neither manifest adds a skills path; its name is the one the three documents that mention it tell you to type. """ @@ -622,6 +623,14 @@ def run(home: str, *argv: str) -> tuple[int, str, str]: assert code == 0 and out.startswith("0 generation(s)") and "`0 generation(s)`" in flat code, _, err = run(empty, "recall", "x") assert code == 0 and err == "no matches\n" and "`no matches` on stderr, exit 0" in flat + os.makedirs(os.path.join(empty, "spool")) + with open(os.path.join(empty, "spool", "1-SessionEnd.json"), "w") as fh: + fh.write("{}") + os.utime(fh.name, (0, 0)) + code, _, err = run(empty, "recall", "x") + said = err.splitlines()[0].replace("", "") + said = re.sub(r"for \d+ \w+:", "for :", re.sub(r"^\d+", "", said)) + assert code == 0 and f"``{said}``" in flat, said def test_the_recall_skill_names_the_release_it_was_written_for(capsys): diff --git a/tests/test_index.py b/tests/test_index.py index 5d2a3d3..d901f1b 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -1633,3 +1633,90 @@ def test_a_sidechain_line_inside_a_main_file_is_marked(home, src): "keep the upload single-shot": False, "review the upload code": True, } + + +# --------------------------------------------------------------------------- # +# the hook rang and no watcher answered +# --------------------------------------------------------------------------- # + + +def _ring(home: str, age: float, name: str = "4242-SessionEnd.json") -> None: + """A spool record as the shim leaves it, arrived `age` seconds ago.""" + spool = os.path.join(home, "spool") + os.makedirs(spool, exist_ok=True) + path = os.path.join(spool, name) + with open(path, "w") as fh: + fh.write('{"transcript_path": "/nowhere/s.jsonl"}') + then = time.time() - age + os.utime(path, (then, then)) + + +def _indexed(home: str, src: str) -> None: + write(src, [user("u1", "the peculiar marmoset")]) + assert main(["--home", home, "capture", src]) == 0 + assert main(["--home", home, "index"]) == 0 + + +def test_recall_says_the_hook_rang_and_no_watcher_answered(home, src, capsys): + """A stopped watcher was invisible from `recall`: it answered from the store + as it was when the watcher stopped, and said nothing. Time since the last + capture cannot say so, because a running watcher on an idle machine captures + nothing either. A record the watcher would have drained within one poll can. + Before the answer, like the truncation notice. [install feedback]""" + _indexed(home, src) + _ring(home, 3 * 3600 + 5) + _ring(home, 60, "4243-PreCompact.json") + capsys.readouterr() + assert main(["--home", home, "recall", "marmoset"]) == 0 + out, err = capsys.readouterr() + assert "marmoset" in out and "uncollected" not in out, out + assert err == ( + "2 hook record(s) uncollected for 3 hours: is `gitmemory watch` running for " + f"{home}? These results may be missing everything said since.\n" + ), err + assert main(["--home", home, "recall", "aardvark"]) == 0 + err = capsys.readouterr().err + assert err.index("uncollected") < err.index("no matches"), err + for age, said in ((90 * 60, "90 minutes"), (3 * 86400, "3 days")): + _ring(home, age) + assert main(["--home", home, "recall", "marmoset"]) == 0 + assert f"uncollected for {said}:" in capsys.readouterr().err + + +def test_recall_says_nothing_about_a_spool_a_watcher_could_still_be_draining( + home, src, capsys +): + """Silent below the threshold, on a hook still writing, and under `--db`, + whose index records no store to look in.""" + _indexed(home, src) + _ring(home, 30 * 60) + _ring(home, 3 * 3600, ".tmp-4242-0") + capsys.readouterr() + assert main(["--home", home, "recall", "marmoset"]) == 0 + assert capsys.readouterr().err == "", "a fresh record or a hook's temp was reported" + _ring(home, 3 * 3600) + assert main(["--home", home, "recall", "marmoset", "--db", index.db_path(home)]) == 0 + assert capsys.readouterr().err == "", "--db names an index, not a store" + + +def test_one_watcher_pass_is_what_clears_it(home, src, capsys): + """The notice is true exactly as long as `drain_spool` has not run: every + pass consumes every complete record, covered or not. A directory in the + spool is no record and no pass can unlink it, so counting it warned for + ever with a watcher running. [review]""" + root = json.dumps(os.path.dirname(src)) + with open(os.path.join(home, "config.toml"), "w") as fh: + fh.write(f'[[watch]]\nagent = "claude-code"\nroots = [{root}]\n') + _indexed(home, src) + _ring(home, 3 * 3600) + stuck = os.path.join(home, "spool", "4244-Stop.json") + os.mkdir(stuck) + os.utime(stuck, (time.time() - 3 * 3600,) * 2) + capsys.readouterr() + assert main(["--home", home, "recall", "marmoset"]) == 0 + assert capsys.readouterr().err.startswith("1 hook record(s) uncollected for 3 hours:") + assert main(["--home", home, "watch", "--once"]) == 0 + assert os.path.isdir(stuck), "the pass removed the directory; this test no longer bites" + capsys.readouterr() + assert main(["--home", home, "recall", "marmoset"]) == 0 + assert "uncollected" not in capsys.readouterr().err From dba5db7163bc183d185854a2a77127b02e889730 Mon Sep 17 00:00:00 2001 From: doronp <3586743+doronp@users.noreply.github.com> Date: Sun, 18 Oct 2026 13:35:20 +0300 Subject: [PATCH 07/13] feat: add gitmemory show, which prints the turn at a recall address recall stops each turn at 160 characters and hands back an address, but nothing resolved one: the rest of the turn was segment arithmetic done by hand under raw/, which the recall skill spelled out for the agent. `gitmemory show //gNN@ [-C N | --context N]` reads the store, not the index, so it needs no index and a stale or tampered one cannot choose what it prints. It parses the generation the way `index` does, so a turn cut by a segment boundary comes back whole, and an offset inside a turn resolves to that turn with its own start in the header. A subagent's turn is headed subagent:, because its user text is the parent agent's brief, not the user's words. recall and show now share one _date() helper, and recall's line keeps its subagent: label in the role column. The address is input. Its names go through store.identity() before anything is read and are only compared with what sessions() found, so it never becomes a path; `[0-9]` keeps int() from reading Arabic-Indic digits. Errors name the address as it was typed, masked, since a session id can be a token: identity() lowercases, and a lowercased AWS key id is one no detector matches. Exit 2 for a malformed address or not a store, 1 for an address the store does not hold or two manifests claiming it. Only header lines start in column 0: tool names are whitespace-collapsed and text is split on "\n" only, which closes the two header forgeries the prototype reproduced. -C rejects a negative count, which plain type=int turned into empty output and exit 0. Eight mutation rows cover the guards (638 rows). README, USAGE (with a full example address), the recall skill (show first; for a CLI older than show, tell the user as the older-CLI rule says and read raw/ by hand; show in the read-only list and the --home sentence; and a captured show result listed as an echo, since it holds the whole turn dated later than the turn), DESIGN 2.8, ARCHITECTURE and CHANGELOG are updated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QPBLmJD2q1PVxdkWx8C16U Signed-off-by: doronp <3586743+doronp@users.noreply.github.com> --- CHANGELOG.md | 9 ++ README.md | 9 +- docs/ARCHITECTURE.md | 2 +- docs/DESIGN.md | 4 +- docs/REPRODUCE.md | 8 +- docs/USAGE.md | 39 +++++- skills/recall/SKILL.md | 40 ++++--- src/gitmemory/__main__.py | 110 ++++++++++++++++- tests/mutate_index.py | 61 +++++++++- tests/test_show.py | 243 ++++++++++++++++++++++++++++++++++++++ 10 files changed, 494 insertions(+), 31 deletions(-) create mode 100644 tests/test_show.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ee58e44..5b6c73f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,15 @@ tell you. tell a stopped watcher from an idle one; a record no pass has drained can. Not under `--db`, and never on a store the hook does not ring. See [docs/watching.md](docs/watching.md#check-it). +- **`gitmemory show
`** prints the whole turn at an address `recall` + printed, where `recall` stops at 160 characters. It reads the store, not the + index, so it needs no index and a stale one cannot change the answer. + `--context N` (`-C N`) adds N turns either side; a turn cut by a segment + boundary comes back whole; an offset inside a turn shows that turn; a + subagent's turn is headed `subagent:`. An address that is not one exits + 2, and one the store does not hold exits 1. The recall skill uses it in place + of reading `raw/` by hand. See + [USAGE.md](docs/USAGE.md#read-the-whole-turn). ### Changed diff --git a/README.md b/README.md index b519932..ea4d3f5 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ agent after compaction, derived key ideas, the dashboard — is in | `gitmemory verify` | Checks every manifest's contiguity proof | | `gitmemory index` | Rebuilds the SQLite FTS5 index from the store | | `gitmemory recall ""` | Searches the index and prints one line per turn, dated, the user's own words newest first ([`--policy`](docs/USAGE.md#when-memories-disagree)) | +| `gitmemory show
` | Prints the whole turn at an address `recall` printed, read from the store; `--context N` adds N turns either side ([more](docs/USAGE.md#read-the-whole-turn)) | | `gitmemory derive [--graph]` | Rebuilds key ideas and a timeline; `--graph` adds the decision graph (opt-in) | | `gitmemory dashboard` | Serves the index with Datasette, read-only, on loopback, behind a sign-in | | `gitmemory push` | Runs the redaction gate over what a push would send (it does not send yet) | @@ -209,13 +210,13 @@ real text, and that is a measurement rather than a suspicion. | What | How it was measured | Result | |---|---|---| | Hook cost in the agent's critical path | Timed against spawning `true` the same way, three runs of 400 | p50 **7.4 – 7.5 ms**, p99 **10.2 – 11.5 ms** | -| The suite | On a fresh checkout, no downloads | **1,087 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | -| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **630** negative controls | +| The suite | On a fresh checkout, no downloads | **1,107 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | +| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **638** negative controls | The two suite counts do not add up, and should not. Switching the corpora on -collects 1412, not 1415. Three conformance cases fill parametrisations that +collects 1432, not 1435. Three conformance cases fill parametrisations that collect as one empty placeholder each while the corpora are absent, so they -replace three of the 1087 rather than joining them. Every figure on this board +replace three of the 1107 rather than joining them. Every figure on this board is pinned by a test, which is how it stays true. **What is not measured**, and is shown as a row on the dashboard rather than diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 52df25a..f5bc23f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -75,7 +75,7 @@ right of `git` can be deleted and rebuilt, so `index/` is gitignored and | Store | [`gitrepo.py`](../src/gitmemory/gitrepo.py) | The only module that runs git, always as `git -C ` | §2.4 | | Derive | [`index.py`](../src/gitmemory/index.py) | A SQLite FTS5 (BM25) index rebuilt from raw bytes. It is never authoritative | §2.7 | | Derive | [`derive.py`](../src/gitmemory/derive.py), [`graph.py`](../src/gitmemory/graph.py) | Key ideas and a timeline. The decision extraction behind the graph is opt-in (`--graph`) | §2.6 | -| Surface | [`__main__.py`](../src/gitmemory/__main__.py) | The CLI: `init capture watch service verify push index derive dashboard recall` | §2.8 | +| Surface | [`__main__.py`](../src/gitmemory/__main__.py) | The CLI: `init capture watch service verify push index derive dashboard recall show` | §2.8 | | Surface | [`dashboard.py`](../src/gitmemory/dashboard.py) | Datasette over the index, `--immutable`, on loopback, behind a single-use sign-in URL | §2.9 | | Egress | [`redact.py`](../src/gitmemory/redact.py) | The only place secrets are looked for. It stands at `push` and scans the files, the segment seams and the object graph a push would transmit | §2.4 | diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 6d9aad9..ab4a8df 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -629,7 +629,9 @@ answers the token-burn objection.)* hooks' `additionalContext` outputs concatenate or last-wins. **As built:** the CLI only. There is no MCP server and no `UserPromptSubmit` hook in `src/`, so -nothing is injected; injection cost is carried as `NOT BUILT` on the dashboard (§2.9). +nothing is injected; injection cost is carried as `NOT BUILT` on the dashboard (§2.9). `recall` +prints each hit's address, and `show` resolves one to the whole turn from the store, not the +index, so a stale or tampered index cannot change what the address is shown to say. ### 2.9 Dashboard diff --git a/docs/REPRODUCE.md b/docs/REPRODUCE.md index b660691..29b0b91 100644 --- a/docs/REPRODUCE.md +++ b/docs/REPRODUCE.md @@ -484,11 +484,11 @@ uv run pytest -q --collect-only -p no:cacheprovider | tail -1 uv run pytest -q --collect-only -p no:cacheprovider | grep -c '\.jsonl\]$' ``` -**Expected:** `1087/1088 tests collected (1 deselected)`, then -`1412/1413 tests collected (1 deselected)`, then `328`. The deselected test is +**Expected:** `1107/1108 tests collected (1 deselected)`, then +`1432/1433 tests collected (1 deselected)`, then `328`. The deselected test is the LongMemEval corpus test, which is opt-in: `uv run pytest -q -m corpus` after `bench/fetch_longmemeval.sh`. The README's -paragraph on why 1,087 + 328 is 1,412 and not 1,415 is the arithmetic of these +paragraph on why 1,107 + 328 is 1,432 and not 1,435 is the arithmetic of these three numbers. All three were run while writing this guide. `tests/test_docs.py` holds the README to the first and third numbers, so a @@ -505,7 +505,7 @@ test fail. The count is the number of rows in `tests/mutate_index.py`. uv run python -c "import sys; sys.path.insert(0, 'tests'); from mutate_index import MUTANTS; print(len(MUTANTS))" ``` -**Expected:** `630`. Checked while writing this guide. +**Expected:** `638`. Checked while writing this guide. To run the controls rather than count them: diff --git a/docs/USAGE.md b/docs/USAGE.md index 64d6a85..04a89a9 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -100,6 +100,38 @@ since: start `gitmemory watch` with the hook's `$GITMEMORY_HOME`, and covers the other causes, such as a watcher that runs passes less often than hourly. With `--db`, recall does not look. +## Read the whole turn + +A line stops at 160 characters. `show` prints the whole turn at an address, +read from the store rather than the index, so it needs no index and a stale +one cannot change what it prints. Copy the address whole: the ones above are +cut short with `…` to fit this page, and a shortened address is refused with +`not an address` and exit 2. + +```console +$ gitmemory show claude-code/0f3a9c2e-1b2d-4c5e-8f90-3a53cc4a5233-fc59494095426389/g00@288 --context 1 +- claude-code/0f3a9c2e-1b2d-4c5e-8f90-3a53cc4a5233-fc59494095426389/g00@0 2026-09-28T10:02Z user + text + Don't add a retry loop around the upload; the API is idempotent only per request id. +> claude-code/0f3a9c2e-1b2d-4c5e-8f90-3a53cc4a5233-fc59494095426389/g00@288 2026-09-28T10:03Z assistant + text + Understood. I dropped the retry loop and pass a request id instead, so a replay cannot double-charge. +- claude-code/0f3a9c2e-1b2d-4c5e-8f90-3a53cc4a5233-fc59494095426389/g00@592 2026-09-28T10:04Z user + text + Good. Ship it. +``` + +Each turn starts with a header: `>` for the turn at the address, `-` for the +neighbours `--context N` (or `-C N`) adds either side, then the turn's own +address, date and role. A subagent's turns say `subagent:user` and +`subagent:assistant`. Under the header comes each block's kind, with the tool's +name for a tool call, and then its text, indented. Only headers start in +column 0, so nothing in a transcript can print a line that reads as one. + +An offset inside a turn shows that turn, and its header gives where it starts. +An address the store does not hold (`not in the store`, `no turn at byte N`) +exits 1. + ## When memories disagree A decision made three weeks ago and reversed last week are both in the store, @@ -184,9 +216,10 @@ would run anyway. A name that does not resolve is an error, not a silent fallback to the default. The offset is the point: it takes you back to the exact bytes, not to a -paraphrase of them. The raw segments live under -`raw///g/`, named by the byte range they hold, so a turn at -offset 288 is in the segment whose range contains 288. +paraphrase of them. [`show`](#read-the-whole-turn) finds them for you. To check +it by hand, the raw segments live under `raw///g/`, named +by the byte range they hold, so a turn at offset 288 is in the segment whose +range contains 288. ## Put it back in front of the agent diff --git a/skills/recall/SKILL.md b/skills/recall/SKILL.md index 22799d4..f276fdc 100644 --- a/skills/recall/SKILL.md +++ b/skills/recall/SKILL.md @@ -17,8 +17,8 @@ redaction gate: that gate stands at `gitmemory push`. The store is `$GITMEMORY_HOME`, or `~/.gitmemory` when that is unset; the CLI reads it, so leave it as this session has it. If the user keeps the store somewhere else, pass their path before the subcommand, as in -`gitmemory --home recall ""`, and the same for `index`. `` -below means that directory: the path the user gave, or what +`gitmemory --home recall ""`, and the same for `index` and +`show`. `` below means that directory: the path the user gave, or what `echo "${GITMEMORY_HOME:-$HOME/.gitmemory}"` prints. Do not type `~/.gitmemory` in its place from memory. @@ -86,6 +86,10 @@ recalls match the same words. Skip these hits; they are not sources: - a `gitmemory recall ...` tool call and its result, lines that start with a score, `N generation(s)` or `no matches`; - a `gitmemory --version` tool call and the one line it printed; +- a `gitmemory show ...` tool call and its result, text that starts `> ` or + `- ` and an `//gNN@` address. It holds the whole + turn, dated later than the turn was: the source is the turn at the address + in its `> ` header, not the echo's own address; - an earlier answer to the same question, and the lines it quoted. If they crowd out the rest, search again with `-k 30`. Quote the original turn. @@ -136,12 +140,19 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. - Quote the text field verbatim, with its address copied whole, never shortened. A paraphrase is not a quote: if you summarise, say that you are, and keep the address beside it. -- The text stops at 160 characters. If the question needs the rest, the turn - is in `/raw///g/`. Segment file names are byte - ranges, start included, end excluded. In the one whose range holds the - offset, the turn's JSON line starts `offset - start` bytes in, and the block - is inside it. If the file ends before the line does, the line goes on at the - start of the next segment, whose range starts where this one ends. +- The text stops at 160 characters. If the question needs the rest, run + `gitmemory show
` with the address copied whole, or + `gitmemory show --context 2
` for the two turns either side too. A + header starts `>` for the turn at the address and `-` for a neighbour, and + only headers start in column 0; quote the text under one without its indent. + `subagent:user` in a header is a brief from the agent that started the + subagent, not the user's words. If the CLI is older than `show` (exit 2, + `invalid choice: 'show'`), tell the user as [When it fails](#when-it-fails) + says, and read the turn by hand: it is the JSON line that starts + `offset - start` bytes into the segment under + `/raw///g/` whose name, the byte range + `start-end` with the end excluded, holds the offset, and it may run on into + the next segment. - This session is `${CLAUDE_SESSION_ID}`. A hit's manifest, `/sessions///g.json`, has its transcript's `source_path`: `/.jsonl`, or for a subagent (a session @@ -157,9 +168,10 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. ## Read-only -Run `gitmemory --version`, `gitmemory recall` and `gitmemory index`, and read -files under ``. `index` writes only the derived index under -`/index/`. Do not run `gitmemory init`, `gitmemory capture`, -`gitmemory watch`, `gitmemory service`, `gitmemory derive`, `gitmemory push` -or `gitmemory dashboard`, and never `gitmemory dashboard --expose`. Do not -edit `config.toml` or anything else in the store. +Run `gitmemory --version`, `gitmemory recall`, `gitmemory show` and +`gitmemory index`, and read files under ``. `index` writes only the +derived index under `/index/`. Do not run `gitmemory init`, +`gitmemory capture`, `gitmemory watch`, `gitmemory service`, +`gitmemory derive`, `gitmemory push` or `gitmemory dashboard`, and never +`gitmemory dashboard --expose`. Do not edit `config.toml` or anything else in +the store. diff --git a/src/gitmemory/__main__.py b/src/gitmemory/__main__.py index efda51d..18f8f85 100644 --- a/src/gitmemory/__main__.py +++ b/src/gitmemory/__main__.py @@ -14,6 +14,7 @@ import builtins import ipaddress import os +import re import shlex import socket import sqlite3 @@ -205,6 +206,21 @@ def _positive_int(text: str) -> int: return value +def _count(text: str) -> int: + """A non-negative integer, for `show --context`, whose default is 0. + + Plain `type=int` took `-C -1`, and the slice that asked for was empty, so + `show` printed nothing and exited 0. + """ + try: + value = int(text) + except ValueError as err: + raise argparse.ArgumentTypeError(f"invalid int value: {text!r}") from err + if value < 0: + raise argparse.ArgumentTypeError(f"cannot be negative: {text}") + return value + + def _watch(args) -> int: home = store.resolve_home(args.home) # The emptiness check used to live here, once, before the loop — so it @@ -440,10 +456,9 @@ def _recall(args) -> int: raise ValueError(f"policy {args.policy!r} failed: {exc!r}") from exc for h in shown: head = " ".join(h.text.split())[:160] - t = index.when(h) # The date on every line, whatever the policy: it is what lets the # reader see that two disagreeing turns are three weeks apart. - date = t.strftime("%Y-%m-%dT%H:%MZ") if t else "undated".ljust(17) + date = _date(h) # In the role's column, not a column of its own: whatever parses these # lines reads role/kind as the fourth field, and a subagent's user/text # is the parent agent's instructions, which must not be quoted as the @@ -459,6 +474,90 @@ def _recall(args) -> int: return 0 +def _date(item) -> str: + """When a hit or a turn was written, as `recall` and `show` both print it: + to the minute in UTC, or `undated` padded to the same width.""" + t = index.when(item) + return t.strftime("%Y-%m-%dT%H:%MZ") if t else "undated".ljust(17) + + +# `//gNN@`, as `recall` prints it. `[0-9]`, not `\d`: +# `\d` is Unicode, and `int()` reads an Arabic-Indic digit as an offset. +_ADDRESS = re.compile(r"([^/]+)/([^/]+)/g([0-9]+)@([0-9]+)\Z") + + +def _show(args) -> int: + """The turn at an address `recall` printed, read from the store. + + The store, not the index: the watcher does not index, and a stale or + hostile index would choose what this prints, when this is how a reader + checks what `recall` said. The parse is `index`'s own, so a turn cut by a + segment boundary comes back whole. + + The address never becomes a path. Its names go through `store.identity`, + the rule `capture` filed them under, so `..`, a dotfile or a newline is + refused before anything is read, and the result is only compared with what + `sessions()` found, which already keeps manifests and segments inside the + store. + """ + m = _ADDRESS.match(args.address) + try: + agent, session_id = store.identity(m[1], m[2]) if m else ("", "") + except ValueError: + m = None + if not m: + want = "expected //gNN@, as recall prints it" + print(f"not an address: {redact.safe_path(args.address)!r}; {want}", file=sys.stderr) + return 2 + if (code := _not_a_store(args.home)) is not None: + return code + gen, offset = int(m[3]), int(m[4]) + key = f"{agent}/{session_id}/g{gen:02d}" + # Each error below names the address as it was typed, masked. A session id + # is whatever `capture --session-id` was handed, so it can be a token, and + # `key` is lowercased, which hides an AWS key id from its detector. + said = redact.safe_path(f"{m[1]}/{m[2]}/g{gen:02d}") + found = [ + s + for s in store.sessions(store.resolve_home(args.home)) + if (s.agent, s.session_id, s.generation) == (agent, session_id, gen) + ] + if len(found) != 1: + # Two is a manifest declaring an identity its path does not have, which + # `verify` reports. Picking one would be choosing the answer. + why = "not in the store" + if found: + why = f"claimed by {len(found)} manifests; run `gitmemory verify`" + print(f"{said}: {why}", file=sys.stderr) + return 1 + turns = index.parse_generation(found[0]).turns + # An offset inside a turn is that turn, as a bench gold offset is; the + # header then gives its start. + at = next( + (i for i, t in enumerate(turns) if t.byte_offset <= offset < t.byte_offset + t.byte_len), + None, + ) + if at is None: + print(f"{said}: no turn at byte {offset}", file=sys.stderr) + return 1 + # Only a header starts in column 0, and `safe_text` leaves "\n" the only + # line break that reaches the terminal, so no transcript can print a line + # that reads as one. Two ways past that were reproduced and are closed + # here: a tool name is any string the transcript chose, so it is collapsed, + # and `splitlines()` broke on "\r", "\x85" and U+2028 before `safe_text` + # could escape them. + lo = max(0, at - args.context) + for i, t in enumerate(turns[lo : at + args.context + 1], start=lo): + # A subagent's user turn is the brief its parent wrote, not the user. + who = f"subagent:{t.role}" if t.is_sidechain else t.role + print(f"{'>' if i == at else '-'} {key}@{t.byte_offset} {_date(t)} {who}") + for b in t.blocks: + print(" " + " ".join([b.kind, *(b.tool_name or "").split()])) + for line in b.text.split("\n") if b.text else (): + print(f" {line}") + return 0 + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(prog="gitmemory") # The plugin and the CLI are installed apart and can be releases apart, so @@ -551,6 +650,13 @@ def main(argv: list[str] | None = None) -> int: ) rec.set_defaults(fn=_recall) + sho = sub.add_parser("show", help="print the turn at an address recall printed, from the store") + sho.add_argument("address", help="//gNN@, as recall prints it") + sho.add_argument( + "-C", "--context", type=_count, default=0, metavar="N", help="also N turns either side" + ) + sho.set_defaults(fn=_show) + args = ap.parse_args(argv) try: return args.fn(args) diff --git a/tests/mutate_index.py b/tests/mutate_index.py index d552ff9..39ce9cb 100644 --- a/tests/mutate_index.py +++ b/tests/mutate_index.py @@ -4647,7 +4647,7 @@ # anchor appear twice and the harness would skip itself. [E7] "the docstring's row count drifts away from the index again", "tests/mutate_index.py", - "a full pass is 6" + "30 mutants", + "a full pass is 6" + "38 mutants", "a full pass is 14" + "6 mutants", "test_the_row_count_in_the_docstring_is_the_row_count", ), @@ -5476,6 +5476,63 @@ " if False:\n", "test_a_non_positive_count_is_refused_at_the_command_line", ), + # `gitmemory show`: the address is input, and the output is a terminal's. + ( + "show takes an address's names without the store's path rule", + "__main__.py", + ' agent, session_id = store.identity(m[1], m[2]) if m else ("", "")', + ' agent, session_id = (m[1], m[2]) if m else ("", "")', + "test_a_malformed_or_escaping_address_is_not_an_address", + ), + ( + "show reads an Arabic-Indic digit as an offset", + "__main__.py", + '_ADDRESS = re.compile(r"([^/]+)/([^/]+)/g([0-9]+)@([0-9]+)\\Z")', + '_ADDRESS = re.compile(r"([^/]+)/([^/]+)/g(\\d+)@(\\d+)\\Z")', + "test_a_malformed_or_escaping_address_is_not_an_address", + ), + ( + "show picks one of two manifests that claim an address", + "__main__.py", + " if len(found) != 1:", + " if not found:", + "test_two_manifests_claiming_one_address_are_refused", + ), + ( + "show prints a credential-shaped session id in its error", + "__main__.py", + ' said = redact.safe_path(f"{m[1]}/{m[2]}/g{gen:02d}")', + " said = key", + "test_a_credential_in_an_address_is_masked_in_the_error", + ), + ( + "show masks the lowercased key, which hides an AWS key id", + "__main__.py", + ' said = redact.safe_path(f"{m[1]}/{m[2]}/g{gen:02d}")', + " said = redact.safe_path(key)", + "test_a_credential_in_an_address_is_masked_in_the_error", + ), + ( + "show prints a tool name's newline, so a transcript forges a header", + "__main__.py", + ' print(" " + " ".join([b.kind, *(b.tool_name or "").split()]))', + ' print(" " + " ".join([b.kind, *filter(None, [b.tool_name])]))', + "test_a_transcript_cannot_forge_a_show_header_or_reach_the_terminal", + ), + ( + "show splits block text before a carriage return is escaped", + "__main__.py", + ' for line in b.text.split("\\n") if b.text else ():', + " for line in b.text.splitlines() if b.text else ():", + "test_a_transcript_cannot_forge_a_show_header_or_reach_the_terminal", + ), + ( + "show takes a negative --context and prints nothing", + "__main__.py", + '"-C", "--context", type=_count,', + '"-C", "--context", type=int,', + "test_show_on_no_store_and_a_negative_context_exit_2", + ), ( "every ssh login name but git is refused as a credential", "redact.py", @@ -5837,7 +5894,7 @@ def _select(test: str) -> list[str]: def main() -> int: """Run every mutant, or only those whose name contains an argument. - The filter exists because a full pass is 630 mutants × two suite runs — + The filter exists because a full pass is 638 mutants × two suite runs — long enough that adding one row and checking it used to mean either waiting for the other 567 or trusting the new one untested. diff --git a/tests/test_show.py b/tests/test_show.py new file mode 100644 index 0000000..1bf5db9 --- /dev/null +++ b/tests/test_show.py @@ -0,0 +1,243 @@ +# SPDX-FileCopyrightText: 2026 The gitmemory authors +# SPDX-License-Identifier: Apache-2.0 +"""`gitmemory show
`: the turn behind a line `recall` printed. + +`recall` cuts a turn to 160 characters and hands back an address. Until `show`, +the rest of the turn was segment arithmetic done by hand under `raw/`. These +tests hold the three things that make the address worth printing: it resolves +from the store and not from the index, it never becomes a path, and nothing in +a transcript can make the output say a different address than the one it is. +""" + +from __future__ import annotations + +import json +import os + +import pytest +from test_index import assistant, user, write + +from gitmemory import store +from gitmemory.__main__ import main + + +@pytest.fixture +def home(tmp_path): + h = tmp_path / "store" + h.mkdir() + return str(h) + + +@pytest.fixture +def src(tmp_path): + p = tmp_path / "src" / "sess.jsonl" + p.parent.mkdir() + return str(p) + + +def size(obj: dict) -> int: + """One JSONL line's length, newline included: the next turn's offset.""" + return len(json.dumps(obj).encode()) + 1 + + +def show(home: str, capsys, *argv: str) -> tuple[int, str, str]: + code = main(["--home", home, "show", *argv]) + out, err = capsys.readouterr() + return code, out, err + + +def test_show_prints_the_whole_turn_recall_cut_at_160(home, src, capsys): + long = "the marmoset rule: " + "never retry the upload " * 20 + "END-OF-TURN" + write(src, [user("u1", long) | {"timestamp": "2026-09-28T10:00:00Z"}]) + store.capture(src, "claude-code", "sess", home=home) + assert main(["--home", home, "index"]) == 0 + capsys.readouterr() + assert main(["--home", home, "recall", "marmoset"]) == 0 + out = capsys.readouterr().out + assert "END-OF-TURN" not in out + code, out, err = show(home, capsys, out.split()[2]) + assert (code, err) == (0, "") + assert out == f"> claude-code/sess/g00@0 2026-09-28T10:00Z user\n text\n {long}\n" + + +def test_show_needs_no_index(home, src, capsys): + write(src, [user("u1", "aardvark")]) + store.capture(src, "claude-code", "sess", home=home) + code, out, _ = show(home, capsys, "claude-code/sess/g00@0") + assert code == 0 and " aardvark\n" in out + assert not os.path.exists(os.path.join(home, "index")) + + +def test_show_resolves_a_turn_that_straddles_a_segment_cut(home, src, capsys): + first, second = user("u1", "first"), user("u2", "the peculiar marmoset") + write(src, [first, second]) + with open(src, "rb") as fh: + whole = fh.read() + with open(src, "wb") as fh: + fh.write(whole[: size(first) + 10]) # the cut lands inside the second line + store.capture(src, "claude-code", "sess", home=home) + with open(src, "wb") as fh: + fh.write(whole) + store.capture(src, "claude-code", "sess", home=home) + assert len(store.sessions(home)[0].segments) == 2 + code, out, _ = show(home, capsys, f"claude-code/sess/g00@{size(first)}") + assert code == 0 + assert out.splitlines()[0].startswith(f"> claude-code/sess/g00@{size(first)} ") + assert " the peculiar marmoset" in out.splitlines() + + +def test_show_context_prints_neighbours_in_order_and_marks_the_target(home, src, capsys): + turns = [user(f"u{i}", f"turn-{i}") for i in range(5)] + write(src, turns) + store.capture(src, "claude-code", "sess", home=home) + at = size(turns[0]) + size(turns[1]) + code, out, _ = show(home, capsys, f"claude-code/sess/g00@{at}", "--context", "1") + assert code == 0 + heads = [ln for ln in out.splitlines() if not ln.startswith(" ")] + assert [h[0] for h in heads] == ["-", ">", "-"] + assert [ln.strip() for ln in out.splitlines() if ln.startswith(" ")] == [ + "turn-1", + "turn-2", + "turn-3", + ] + + +def test_an_offset_inside_a_turn_shows_that_turn_at_its_start(home, src, capsys): + write(src, [user("u1", "aardvark"), user("u2", "marmoset")]) + store.capture(src, "claude-code", "sess", home=home) + start = size(user("u1", "aardvark")) + code, out, _ = show(home, capsys, f"claude-code/sess/g00@{start + 5}") + assert code == 0 and out.startswith(f"> claude-code/sess/g00@{start} ") + assert "marmoset" in out and "aardvark" not in out + + +def test_a_subagent_turn_is_headed_subagent_not_user(home, src, capsys): + """A sidechain's user-role text is the parent agent's brief, not the user's + word, so the header says whose it is. Same column, so a reader that splits + the header still finds the role where it was.""" + brief = user("u1", "review the upload code") | {"isSidechain": True} + write(src, [brief, assistant("a2", [{"type": "text", "text": "done"}])]) + store.capture(src, "claude-code", "sess", home=home) + code, out, _ = show(home, capsys, "claude-code/sess/g00@0", "--context", "1") + assert code == 0 + heads = [ln for ln in out.splitlines() if not ln.startswith(" ")] + assert heads == [ + "> claude-code/sess/g00@0 undated subagent:user", + f"- claude-code/sess/g00@{size(brief)} undated assistant", + ] + + +@pytest.mark.parametrize( + "address", + [ + "../../etc/passwd@0", + "claude-code/../g00@0", + "claude-code/.git/g00@0", + "claude-code/sess\n/g00@0", + "claude-code/sess/g00@\u0663", # ARABIC-INDIC DIGIT THREE: int() reads it as 3 + "claude-code/sess/g00", + "claude-code/sess@0", + "/etc/sess/g00@0", + ], +) +def test_a_malformed_or_escaping_address_is_not_an_address(home, src, capsys, address): + write(src, [user("u1", "aardvark")]) + store.capture(src, "claude-code", "sess", home=home) + code, out, err = show(home, capsys, address) + assert (code, out) == (2, "") and err.startswith("not an address: ") + assert err.count("\n") == 1 + + +def test_an_address_the_store_does_not_hold_is_an_answer_exit_1(home, src, capsys): + write(src, [user("u1", "aardvark")]) + store.capture(src, "claude-code", "sess", home=home) + code, out, err = show(home, capsys, "claude-code/sess/g01@0") + assert (code, out, err) == (1, "", "claude-code/sess/g01: not in the store\n") + code, out, err = show(home, capsys, "claude-code/sess/g00@99999") + assert (code, out, err) == (1, "", "claude-code/sess/g00: no turn at byte 99999\n") + + +def test_a_credential_in_an_address_is_masked_in_the_error(home, src, capsys): + """A session id is whatever `capture --session-id` was given, and these + pass the path-component rule. The error masks the address as it was typed: + `identity()` lowercases, and an AWS key id lowercased is one no detector + knows, so masking the store's spelling printed it whole.""" + write(src, [user("u1", "aardvark")]) + store.capture(src, "claude-code", "sess", home=home) + token = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8" + code, _, err = show(home, capsys, f"claude-code/{token}/g00@0") + assert code == 1 and "ghp_" in err and token.lower() not in err.lower() + key = "AKIAZZZZQQQQWWWW1234" # synthetic, right shape + code, _, err = show(home, capsys, f"claude-code/{key}/g00@0") + assert code == 1 and "AKIA" in err and key.lower() not in err.lower(), err + + +def test_a_transcript_cannot_forge_a_show_header_or_reach_the_terminal(home, src, capsys): + """Only a header starts in column 0, so a line that looks like one has to + have come from `show`. Two forgeries got past the first version: a tool + name with a newline in it printed a header of its own, and `splitlines()` + broke on a carriage return before `safe_text` could escape it.""" + forged = "> claude-code/forged/g00@0 2026-01-01T00:00Z user" + text = f"needle \x1b[2K\r{forged}\n{forged}\u2028{forged}\x85{forged}" + tool = {"type": "tool_use", "id": "t", "name": f"Bash\n{forged}", "input": {"command": "ls"}} + write(src, [user("u1", text), assistant("a2", [tool])]) + store.capture(src, "claude-code", "sess", home=home) + code, out, _ = show(home, capsys, "claude-code/sess/g00@0", "--context", "1") + assert code == 0 + assert not any(c in out for c in "\x1b\r\u2028\x85") + assert out.split("\n")[2] == " needle \\x1b[2K\\x0d" + forged + heads = [ln for ln in out.splitlines() if not ln.startswith(" ")] + assert len(heads) == 2 and not any("forged" in h for h in heads), heads + + +def test_two_manifests_claiming_one_address_are_refused(home, src, capsys): + """`verify` flags a manifest whose declared identity is not its path. Until + it has been run, which of the two is the turn is not something `show` can + know, so it does not pick one.""" + write(src, [user("u1", "aardvark")]) + store.capture(src, "claude-code", "sess", home=home) + store.capture(src, "claude-code", "other", home=home) + other = os.path.join(home, "sessions", "claude-code", "other", "g00.json") + with open(other) as fh: + manifest = json.load(fh) + manifest["session_id"] = "sess" + with open(other, "w") as fh: + json.dump(manifest, fh) + code, out, err = show(home, capsys, "claude-code/sess/g00@0") + assert (code, out) == (1, "") and "claimed by 2 manifests" in err + + +def test_a_captured_show_result_is_an_echo_the_skill_lists(home, tmp_path, capsys): + """A session that ran `show` was captured too, with the whole turn in a + tool result dated after it, so `recall` finds the turn again under the + echo's own address. The skill tells an agent to run `show`, so it has to + tell it how to know the echo: by the header at the front of its text.""" + first, later = str(tmp_path / "a.jsonl"), str(tmp_path / "b.jsonl") + rule = user("u1", "the marmoset rule: never retry the upload") + write(first, [rule | {"timestamp": "2026-09-01T10:00:00Z"}]) + store.capture(first, "claude-code", "a", home=home) + code, said, _ = show(home, capsys, "claude-code/a/g00@0") + call = {"type": "tool_use", "id": "t1", "name": "Bash", "input": {"command": "gitmemory show"}} + result = {"type": "tool_result", "tool_use_id": "t1", "content": said} + echo = user("u3", "") | {"timestamp": "2026-10-01T10:00:00Z"} + echo["message"]["content"] = [result] + write(later, [assistant("a2", [call]), echo]) + store.capture(later, "claude-code", "b", home=home) + assert code == 0 and main(["--home", home, "index"]) == 0 + capsys.readouterr() + assert main(["--home", home, "recall", "marmoset"]) == 0 + heads = [ln.split(None, 4) for ln in capsys.readouterr().out.splitlines()] + assert [h[2].split("@")[0] for h in heads] == ["claude-code/a/g00", "claude-code/b/g00"] + assert heads[1][4].startswith("> claude-code/a/g00@0 "), heads + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + with open(os.path.join(root, "skills", "recall", "SKILL.md")) as fh: + echoes = fh.read().split("\n## Echoes\n")[1].split("\n## ")[0] + assert "`gitmemory show ...`" in echoes and "`> `" in echoes, echoes + + +def test_show_on_no_store_and_a_negative_context_exit_2(tmp_path, capsys): + code, _, err = show(str(tmp_path / "absent"), capsys, "claude-code/sess/g00@0") + assert code == 2 and err.startswith("not a store: ") + with pytest.raises(SystemExit) as done: + main(["--home", str(tmp_path), "show", "claude-code/sess/g00@0", "-C", "-1"]) + assert done.value.code == 2 From b9a5f57c7f133048c3e7fc7429fab423e0ec277b Mon Sep 17 00:00:00 2001 From: doronp <3586743+doronp@users.noreply.github.com> Date: Sun, 18 Oct 2026 13:31:48 +0300 Subject: [PATCH 08/13] feat: recall --redact and show --redact mask the push gate's credential shapes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Through the plugin's skill, recall and show output goes to the model provider, and both printed transcript text as written: an AKIA key, a private key, a session id that is a token. The redaction gate stands at push only, on purpose, because a store on the owner's own disk is not an egress. So this is opt-in masking, not a gate: `--redact` on recall or show, or GITMEMORY_REDACT set to anything but empty or 0, masks both tiers' matches in the printed text and in the address. It refuses nothing, and the default is unchanged. redact.mask() is safe_path's span masking with one widening: private_key_block matches only the BEGIN line, so masking the match printed the base64 key under a masked header. Its span now runs to the END line, or to the end of the text. Only a header no earlier span covers searches for its END, and the END pattern's runs are bounded: a page of headers with no END, or one END line that never closes, made the widening quadratic (a 2 MB tool_result took recall --redact 41 s; it now takes 0.28 s). Spans are joined once rather than with `bytes +=`. safe_path delegates to mask(). The index stores records.safe_text's spelling of a control or invisible character (``, `​`, `\x7f`), and each spelling ends in a word character, so a key right after one lost the `\b` its detector needs while the push gate, reading raw bytes, still found it. mask() also scans a copy with each spelling blanked to as many spaces and masks the union. recall masks after the whitespace collapse, which can join a PEM header a newline split, and before the 160-character cut, because a key straddling the cut leaves a prefix no detector matches (graph._label measured both). The masked address drops the marker's space, so the line keeps its columns, and the subagent mark stays in the role's field. show masks each block whole before it splits it into lines, because a private key's body is the lines under its header, and masks the header's address and a tool's name too. It keeps the line breaks recall flattens, so it asks mask() for a third scan, `collapse=True`: the blanked copy with every whitespace run one space, its matches mapped back to the text's own offsets. recall does not ask for it, since its text is flattened already. The flag lives in a _redact_flag helper that both commands take. conftest clears GITMEMORY_REDACT, as the docs tell the owner to make it standing in settings.json. The skill passes --redact to recall and show whenever GITMEMORY_REDACT is set and not 0, so an older CLI fails with an argparse error instead of printing unmasked, and while masking it never reads a turn from raw/, and says a masked address is not one show can look up. SECURITY.md and USAGE.md list what it does not cover. DESIGN.md's "exactly two boundaries" gets a dated amendment, and its stale detector count (nine and three) is corrected to twelve and four. Twelve tests and sixteen mutation rows (recall: cut, collapse, PEM widening, address and its one column, env parsing, the spelled-control scan and its union, the linear widening, the bounded END pattern; show: whole blocks, address, its one field, tool name, the collapsed scan and its offset mapping), each caught by its intended test. README: 1,119 tests, 654 negative controls. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QPBLmJD2q1PVxdkWx8C16U Signed-off-by: doronp <3586743+doronp@users.noreply.github.com> --- CHANGELOG.md | 14 ++- README.md | 12 +-- SECURITY.md | 38 ++++++-- docs/ARCHITECTURE.md | 2 +- docs/DESIGN.md | 13 ++- docs/REPRODUCE.md | 8 +- docs/USAGE.md | 70 +++++++++++++- hook/README.md | 16 ++++ skills/recall/SKILL.md | 42 ++++++--- src/gitmemory/__main__.py | 53 +++++++++-- src/gitmemory/redact.py | 110 +++++++++++++++++++--- tests/conftest.py | 4 + tests/mutate_index.py | 134 +++++++++++++++++++++++++-- tests/test_index.py | 189 +++++++++++++++++++++++++++++++++++++- tests/test_show.py | 52 ++++++++++- tests/test_store.py | 3 +- 16 files changed, 694 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b6c73f..4bef05e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,8 @@ tell you. compaction or from an earlier session. It runs `gitmemory index` when the index is missing or stale, and writes nothing else. See [docs/USAGE.md](docs/USAGE.md#put-it-back-in-front-of-the-agent). What it - reads goes to the model provider, past no redaction gate. The plugin's + reads goes to the model provider, past no redaction gate; `recall --redact` + (below) masks credential shapes in it, and blocks nothing. The plugin's version is still 0.1.0, so a copy installed from 0.1.0 does not get the skill from `claude plugin update`; reinstall it, or wait for the next release ([hook/README.md](hook/README.md#as-a-claude-code-plugin)). @@ -73,6 +74,17 @@ tell you. 2, and one the store does not hold exits 1. The recall skill uses it in place of reading `raw/` by hand. See [USAGE.md](docs/USAGE.md#read-the-whole-turn). +- **`recall --redact`** and **`show --redact`** mask the credential shapes + the push gate detects in what they print, the text and the address, as + `AKIA…[20 bytes]`; a private key is masked down to its END line, or to the + end of the text without one. `show` masks each block whole before it splits + it into lines, so a key's body goes with its header. Off by default. + `GITMEMORY_REDACT` makes it standing for both (unset, empty or `0` is off), + and the plugin's skill passes the flag to both whenever it is set and not + `0`, and then never reads a turn from `raw/`. It masks only those shapes, + and only in what the two commands print; see + [docs/USAGE.md](docs/USAGE.md#masking-credentials-in-what-it-prints) for + what it does not cover. ### Changed diff --git a/README.md b/README.md index ea4d3f5..165a8b8 100644 --- a/README.md +++ b/README.md @@ -123,8 +123,8 @@ agent after compaction, derived key ideas, the dashboard — is in | `gitmemory capture ` | Copies out one transcript's new bytes, by hand | | `gitmemory verify` | Checks every manifest's contiguity proof | | `gitmemory index` | Rebuilds the SQLite FTS5 index from the store | -| `gitmemory recall ""` | Searches the index and prints one line per turn, dated, the user's own words newest first ([`--policy`](docs/USAGE.md#when-memories-disagree)) | -| `gitmemory show
` | Prints the whole turn at an address `recall` printed, read from the store; `--context N` adds N turns either side ([more](docs/USAGE.md#read-the-whole-turn)) | +| `gitmemory recall ""` | Searches the index and prints one line per turn, dated, the user's own words newest first ([`--policy`](docs/USAGE.md#when-memories-disagree); [`--redact`](docs/USAGE.md#masking-credentials-in-what-it-prints) masks credentials) | +| `gitmemory show
` | Prints the whole turn at an address `recall` printed, read from the store; `--context N` adds N turns either side, `--redact` masks credentials ([more](docs/USAGE.md#read-the-whole-turn)) | | `gitmemory derive [--graph]` | Rebuilds key ideas and a timeline; `--graph` adds the decision graph (opt-in) | | `gitmemory dashboard` | Serves the index with Datasette, read-only, on loopback, behind a sign-in | | `gitmemory push` | Runs the redaction gate over what a push would send (it does not send yet) | @@ -210,13 +210,13 @@ real text, and that is a measurement rather than a suspicion. | What | How it was measured | Result | |---|---|---| | Hook cost in the agent's critical path | Timed against spawning `true` the same way, three runs of 400 | p50 **7.4 – 7.5 ms**, p99 **10.2 – 11.5 ms** | -| The suite | On a fresh checkout, no downloads | **1,107 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | -| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **638** negative controls | +| The suite | On a fresh checkout, no downloads | **1,119 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | +| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **654** negative controls | The two suite counts do not add up, and should not. Switching the corpora on -collects 1432, not 1435. Three conformance cases fill parametrisations that +collects 1444, not 1447. Three conformance cases fill parametrisations that collect as one empty placeholder each while the corpora are absent, so they -replace three of the 1107 rather than joining them. Every figure on this board +replace three of the 1119 rather than joining them. Every figure on this board is pinned by a test, which is how it stays true. **What is not measured**, and is shown as a row on the dashboard rather than diff --git a/SECURITY.md b/SECURITY.md index 8cd4d1d..63201e4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -38,7 +38,8 @@ maintained back-branches. In scope: anything in this repository that runs on a user's machine. That means capture, the store and its contiguity proof, `verify`, the redaction gate -behind `push`, the index, `derive`, the dashboard, the watcher, the hook shim and the plugin's +behind `push`, the masking behind `recall --redact` and `show --redact`, the +index, `derive`, the dashboard, the watcher, the hook shim and the plugin's recall skill. Worth reporting, for example: @@ -47,7 +48,9 @@ Worth reporting, for example: `derive` crash, hang, write outside `$GITMEMORY_HOME`, or cost superlinear time - a way to make `verify` accept a store whose segments do not tile, or whose bytes do not match the recorded digest -- a secret that the redaction gate lets through on `push` +- a secret that the redaction gate lets through on `push`, or that + `recall --redact` or `show --redact` prints unmasked, of a shape its + detectors are meant to catch - the dashboard being reachable without the sign-in URL, or writable Out of scope: vulnerabilities in the third-party dependencies themselves @@ -90,7 +93,30 @@ that gets waved through on a deadline. The gate stands at `push` and nowhere else. The Claude Code plugin's recall skill reads the store into the agent's context, so each line it quotes goes to -the model provider with the rest of the conversation, unscanned. That includes -text another watched agent never sent to that provider, and a credential `push` -would refuse to ship. The skill quotes what the question needs and no more, -which is a rule it follows, not a gate. +the model provider with the rest of the conversation, unscanned unless you turn +on the masking below. That includes text another watched agent never sent to +that provider, and a credential `push` would refuse to ship. The skill quotes +what the question needs and no more, which is a rule it follows, not a gate. + +`recall --redact` and `show --redact`, or `GITMEMORY_REDACT` set to anything +but empty or `0`, mask the gate's shapes in what those two commands print, and +the skill passes the flag to both whenever the variable is set and not `0`, +and then does not read `raw/`. It masks and never refuses, and it is off +unless you turn it on. What it does not guarantee: + +- It masks only the sixteen shapes in [`redact.py`](src/gitmemory/redact.py): + twelve issuer-prefixed keys and four SUSPECT patterns. Anything else prints. + Measured examples: a bare AWS secret key, `PGPASSWORD=…`, a custom + `GH_TOKEN=…`, `glpat-` and `npm_` tokens, an Azure `AccountKey=…`, a password + in prose, `curl -u user:pass`, a `.netrc` line. +- Nothing is decoded. Base64, URL-encoding and a token wrapped across lines get + through. JSON `\u` escapes are masked, because both commands read the parsed + text. +- It is credential masking, not confidentiality. Hostnames, customer data, + personal data and code are untouched. +- It covers only what `recall` and `show` print. `raw/`, the index database, + the dashboard, `derived/` and any file the agent opens are not masked by it. + The environment setting is a floor, not a boundary: an agent with a shell can + unset it, or read `raw/`. +- A mask reveals the first four bytes and the length. +- The SUSPECT patterns also mask some prose about configuration. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f5bc23f..d8088e4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -77,7 +77,7 @@ right of `git` can be deleted and rebuilt, so `index/` is gitignored and | Derive | [`derive.py`](../src/gitmemory/derive.py), [`graph.py`](../src/gitmemory/graph.py) | Key ideas and a timeline. The decision extraction behind the graph is opt-in (`--graph`) | §2.6 | | Surface | [`__main__.py`](../src/gitmemory/__main__.py) | The CLI: `init capture watch service verify push index derive dashboard recall show` | §2.8 | | Surface | [`dashboard.py`](../src/gitmemory/dashboard.py) | Datasette over the index, `--immutable`, on loopback, behind a single-use sign-in URL | §2.9 | -| Egress | [`redact.py`](../src/gitmemory/redact.py) | The only place secrets are looked for. It stands at `push` and scans the files, the segment seams and the object graph a push would transmit | §2.4 | +| Egress | [`redact.py`](../src/gitmemory/redact.py) | The only place secrets are looked for. It stands at `push` and scans the files, the segment seams and the object graph a push would transmit. `recall --redact` and `show --redact` use its detectors to mask what they print | §2.4 | ## On disk diff --git a/docs/DESIGN.md b/docs/DESIGN.md index ab4a8df..6d08684 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -215,6 +215,15 @@ coalescing only changes where the cut points are, never whether they tile. **Raw is committed locally, always** — that is the thesis. Redaction applies at exactly two boundaries: anything written to `derived/`, and anything **`gitmemory push`** sends. **`push` is opt-in per remote and refuses unless the redaction gate passed.** Default config has no remote. +**[2026-10-18 — `recall --redact` and `show --redact` are a third point, and they mask; they are +not a gate. Off by default, because a store on the owner's own disk is still not an egress. The +owner turns it on, or makes it standing with `GITMEMORY_REDACT`, when their output goes on to a +model provider through the plugin's skill. Both tiers are masked, unlike `derived/` below, because a +mask costs the span and not the sentence; a PEM block from its BEGIN line to its END line, or to +the end of the text when there is none; and the session id in the address. `recall` masks after +the whitespace collapse and before the 160-character cut. `show` masks each block whole before it +splits it into lines, and, because it keeps the line breaks, also scans a copy with each whitespace +run collapsed. Neither refuses anything. `raw/` and the index stay verbatim.]** Not "anything that leaves the machine" — that claim was wider than the code and is withdrawn **[E7]**. The store is an ordinary git repository in a directory the owner can read. A `git push` @@ -228,8 +237,8 @@ deleted is absent from the first two and present in the third **[E7]**. A startup assertion refuses to run if `$GITMEMORY_HOME` is inside another work tree, and every git call is explicit `git -C $GITMEMORY_HOME` — never cwd-inherited **[R1]**. -The `derived/` boundary acts on the **HIGH tier only** — the nine issuer-anchored shapes, not the -three SUSPECT ones. A sentence carrying a high-confidence key is dropped before ranking and +The `derived/` boundary acts on the **HIGH tier only** — the twelve issuer-anchored shapes, not +the four SUSPECT ones. A sentence carrying a high-confidence key is dropped before ranking and counted in `sentences_redacted`; `derive._write` then refuses outright to publish a payload that still contains one, because an artifact reaching that door with a key in it is a bug upstream, not a sentence to quietly lose. SUSPECT is excluded deliberately: it is `password: "..."`-shaped and diff --git a/docs/REPRODUCE.md b/docs/REPRODUCE.md index 29b0b91..6653f98 100644 --- a/docs/REPRODUCE.md +++ b/docs/REPRODUCE.md @@ -484,11 +484,11 @@ uv run pytest -q --collect-only -p no:cacheprovider | tail -1 uv run pytest -q --collect-only -p no:cacheprovider | grep -c '\.jsonl\]$' ``` -**Expected:** `1107/1108 tests collected (1 deselected)`, then -`1432/1433 tests collected (1 deselected)`, then `328`. The deselected test is +**Expected:** `1119/1120 tests collected (1 deselected)`, then +`1444/1445 tests collected (1 deselected)`, then `328`. The deselected test is the LongMemEval corpus test, which is opt-in: `uv run pytest -q -m corpus` after `bench/fetch_longmemeval.sh`. The README's -paragraph on why 1,107 + 328 is 1,432 and not 1,435 is the arithmetic of these +paragraph on why 1,119 + 328 is 1,444 and not 1,447 is the arithmetic of these three numbers. All three were run while writing this guide. `tests/test_docs.py` holds the README to the first and third numbers, so a @@ -505,7 +505,7 @@ test fail. The count is the number of rows in `tests/mutate_index.py`. uv run python -c "import sys; sys.path.insert(0, 'tests'); from mutate_index import MUTANTS; print(len(MUTANTS))" ``` -**Expected:** `638`. Checked while writing this guide. +**Expected:** `654`. Checked while writing this guide. To run the controls rather than count them: diff --git a/docs/USAGE.md b/docs/USAGE.md index 04a89a9..72ee9f5 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -130,7 +130,54 @@ column 0, so nothing in a transcript can print a line that reads as one. An offset inside a turn shows that turn, and its header gives where it starts. An address the store does not hold (`not in the store`, `no turn at byte N`) -exits 1. +exits 1. `--redact` masks credentials in what it prints +([below](#masking-credentials-in-what-it-prints)). + +## Masking credentials in what it prints + +`recall` and `show` print what the transcript says, credentials included, +because the store is yours and on your disk. When their output is going +further, to an agent and with it to the agent's model provider, `--redact` +masks every shape the push gate knows, in the text and in the address. That is +twelve issuer-prefixed keys and four SUSPECT patterns, listed in +[`redact.py`](../src/gitmemory/redact.py): + +```console +$ gitmemory recall --redact "staging key" + -0.000 2026-09-28T10:02Z claude-code/0f3a9c2e-…-fc59494095426389/g00@0 user/text Use AKIA…[20 bytes] for the staging key, never the prod one. +``` + +A masked span keeps its first four bytes and its length; in an address the +marker drops its space, so each line keeps its columns and each header its +fields. A private key is masked from its BEGIN line to its END line, or to the +end of the text when there is none. `show --redact` masks each block of the +turn whole before it splits it into lines, so a key's body goes with its +header, and it masks a tool's name too. It keeps the turn's line breaks, where +`recall` flattens a line before it masks, so it also finds a header that a +line break split, as the flattened line would have shown it. A masked address +is not one `show` can look up. + +`GITMEMORY_REDACT=1` makes it standing for both: unset, empty or `0` is off, +and any other value is on. There is no flag that turns a standing setting off; +for one call, run `GITMEMORY_REDACT=0 gitmemory recall ...`. + +What it does not guarantee: + +- It masks only those sixteen shapes. Anything else prints as written: a bare + AWS secret key, `PGPASSWORD=…`, a custom `GH_TOKEN=…`, `glpat-` and `npm_` + tokens, an Azure `AccountKey=…`, a password in prose, `curl -u user:pass`, a + `.netrc` line. +- Nothing is decoded. Base64, URL-encoding and a token wrapped across lines get + through. JSON `\u` escapes are masked, because both commands read the parsed + text, not the raw bytes. +- It masks credentials; it does not make the output confidential. Hostnames, + customer data, personal data and code are untouched. +- It covers only what `recall` and `show` print. `raw/`, the index database, + the dashboard, `derived/` and any file an agent opens are not masked by it. + The environment setting is a floor, not a boundary: an agent with a shell can + unset it, or read `raw/`. +- A mask shows the first four bytes and the length. +- The SUSPECT patterns also mask some prose about configuration. ## When memories disagree @@ -246,7 +293,21 @@ asking about. It quotes each line with its address and says when a hit comes from another session. It writes nothing but the index. It runs `gitmemory --version` first and says when the CLI is older than the plugin; the [hook's README](../hook/README.md#as-a-claude-code-plugin) has how to hold -the two at one release. +the two at one release. To have it mask credentials +([above](#masking-credentials-in-what-it-prints)), set the variable in Claude +Code's own environment, in `~/.claude/settings.json`: + +```json +{ + "env": { + "GITMEMORY_REDACT": "1" + } +} +``` + +The skill then passes `--redact` to every `recall` and `show` it runs, so a +CLI too old to know the flag stops with an error instead of printing unmasked, +and it does not read a turn from `raw/` by hand. ## Derive key ideas and decisions @@ -296,6 +357,11 @@ finding, and there is no override flag. It does not send yet. If a credential has already landed in the store, [SECURITY.md](../SECURITY.md#if-a-credential-lands-in-the-store) says what to do. +`recall` and `show` output that an agent reads goes to that agent's model +provider, and no gate stands in that path. `--redact` masks the gate's shapes +in it ([above](#masking-credentials-in-what-it-prints)); it masks, and it +never refuses. + ## What it does not do - **It does not summarise for you.** There is no LLM in the product. `recall` diff --git a/hook/README.md b/hook/README.md index 18f0bbc..3d476d8 100644 --- a/hook/README.md +++ b/hook/README.md @@ -101,6 +101,22 @@ Code's environment, so a `GITMEMORY_HOME` you set for the watcher has to be set there too, and be absolute, as above. What the skill reads from the store goes to the model provider with the rest of the conversation, and no redaction gate stands in that path ([SECURITY.md](../SECURITY.md#if-a-credential-lands-in-the-store)). +To have `recall` and `show` mask the credential shapes the gate knows in what +they print, set `GITMEMORY_REDACT` in that environment, in +`~/.claude/settings.json`: + +```json +{ + "env": { + "GITMEMORY_REDACT": "1" + } +} +``` + +The skill then passes `--redact` to every `recall` and `show` it runs, and +does not read `raw/`. It masks; it is not a gate, and +[USAGE.md](../docs/USAGE.md#masking-credentials-in-what-it-prints) lists what +it does not cover. An install made before the skill was added does not have it. Claude Code keeps a plugin's cached copy until its version changes, and the skill was added diff --git a/skills/recall/SKILL.md b/skills/recall/SKILL.md index f276fdc..1cb8782 100644 --- a/skills/recall/SKILL.md +++ b/skills/recall/SKILL.md @@ -11,8 +11,14 @@ byte, and `gitmemory index` builds a search index from that record. This skill searches it with the `gitmemory` CLI and quotes the words as they were written. It reads the store and rebuilds the index when it has to; it captures and configures nothing. It sends nothing anywhere itself, but what it reads enters -this conversation and goes to the model provider with the rest of it, past no -redaction gate: that gate stands at `gitmemory push`. +this conversation and goes to the model provider with the rest of it. The +redaction gate stands at `gitmemory push`, not here. `--redact` masks the +credential shapes that gate knows in what `recall` and `show` print, as +`sk-a…[103 bytes]`. Pass it to both whenever `echo "${GITMEMORY_REDACT:-}"` +prints anything but nothing or `0`, and whenever the user asks you to keep +credentials out of the conversation. Masking covers only what those two print, +so while you are masking, do not read a turn from `raw/` by hand, and if +either command rejects the flag, do not run it without it. The store is `$GITMEMORY_HOME`, or `~/.gitmemory` when that is unset; the CLI reads it, so leave it as this session has it. If the user keeps the store @@ -46,7 +52,8 @@ user is referring to, in the words they would have used. `recall` ranks by BM25: it matches words, not meaning. ```sh -gitmemory recall "" # 10 lines by default; -k N for more or fewer +gitmemory recall "" # 10 lines by default; -k N for more or fewer +gitmemory recall --redact "" # the same, masked ``` One line per turn: what the user said first, newest first, then everything @@ -142,17 +149,21 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. and keep the address beside it. - The text stops at 160 characters. If the question needs the rest, run `gitmemory show
` with the address copied whole, or - `gitmemory show --context 2
` for the two turns either side too. A + `gitmemory show --context 2
` for the two turns either side too, + and `gitmemory show --redact
` if you passed `--redact` to + `recall`. A masked address (one with `…[` in it) is not one `show` can + look up: say the line stops there and give the address as printed. A header starts `>` for the turn at the address and `-` for a neighbour, and only headers start in column 0; quote the text under one without its indent. `subagent:user` in a header is a brief from the agent that started the subagent, not the user's words. If the CLI is older than `show` (exit 2, `invalid choice: 'show'`), tell the user as [When it fails](#when-it-fails) - says, and read the turn by hand: it is the JSON line that starts - `offset - start` bytes into the segment under - `/raw///g/` whose name, the byte range - `start-end` with the end excluded, holds the offset, and it may run on into - the next segment. + says. If you are masking, stop there: `raw/` is unmasked, so say the line + stops at 160 characters and give the address. Otherwise read the turn by + hand: it is the JSON line that starts `offset - start` bytes into the + segment under `/raw///g/` whose name, the byte + range `start-end` with the end excluded, holds the offset, and it may run + on into the next segment. - This session is `${CLAUDE_SESSION_ID}`. A hit's manifest, `/sessions///g.json`, has its transcript's `source_path`: `/.jsonl`, or for a subagent (a session @@ -169,9 +180,10 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. ## Read-only Run `gitmemory --version`, `gitmemory recall`, `gitmemory show` and -`gitmemory index`, and read files under ``. `index` writes only the -derived index under `/index/`. Do not run `gitmemory init`, -`gitmemory capture`, `gitmemory watch`, `gitmemory service`, -`gitmemory derive`, `gitmemory push` or `gitmemory dashboard`, and never -`gitmemory dashboard --expose`. Do not edit `config.toml` or anything else in -the store. +`gitmemory index`, and read files under ``, but not `raw/` while you +are masking. `index` writes only the derived index under `/index/`. Do +not run `gitmemory init`, `gitmemory capture`, `gitmemory watch`, +`gitmemory service`, `gitmemory derive`, `gitmemory push` or +`gitmemory dashboard`, and never `gitmemory dashboard --expose`. Do not edit +`config.toml` or anything else in the store. Leave `GITMEMORY_REDACT` as this +session has it. diff --git a/src/gitmemory/__main__.py b/src/gitmemory/__main__.py index 18f8f85..b338c75 100644 --- a/src/gitmemory/__main__.py +++ b/src/gitmemory/__main__.py @@ -12,6 +12,7 @@ import argparse import builtins +import functools import ipaddress import os import re @@ -48,9 +49,11 @@ def _safe_exc(exc: BaseException) -> str: Not in `print` itself, deliberately. `recall` prints transcript content because that is what it is for, and a store on the owner's own disk is not - an egress; masking there would make the tool withhold the user's own bytes - from them. Egress is `push` and `serve`, and an exception is neither — it is - the third case, and it is the one that leaks by accident. + an egress; masking there by default would make the tool withhold the user's + own bytes from them. `--redact` on `recall` and `show` is the owner choosing + to withhold them, because the output is going on: through the plugin's + skill, to the model provider. Egress is `push` and `serve`, and an exception + is neither — it is the third case, and it is the one that leaks by accident. """ return redact.safe_path(str(exc)) @@ -454,17 +457,25 @@ def _recall(args) -> int: shown = list(resolve(hits)) except Exception as exc: # a custom policy is the user's code: report it, no traceback raise ValueError(f"policy {args.policy!r} failed: {exc!r}") from exc + # After the policy, so a custom one cannot route around it. After the + # collapse, which can join a PEM header a newline split, and before the cut: + # a key that straddles character 160 leaves a prefix no detector matches. + # `graph._label` measured both. The address too: a session id can be a key. + shown_as = redact.mask if args.redact else str for h in shown: - head = " ".join(h.text.split())[:160] + head = shown_as(" ".join(h.text.split()))[:160] # The date on every line, whatever the policy: it is what lets the # reader see that two disagreeing turns are three weeks apart. date = _date(h) + # One column still: a session key has no space in it (`store._SAFE_RE`), + # and the one a mask's marker adds would split the address in two. + where = shown_as(f"{h.session_key}@{h.byte_offset}").replace(" ", "") # In the role's column, not a column of its own: whatever parses these # lines reads role/kind as the fourth field, and a subagent's user/text # is the parent agent's instructions, which must not be quoted as the # user's. who = f"subagent:{h.role}" if h.is_sidechain else h.role - print(f"{h.score:8.3f} {date} {h.session_key}@{h.byte_offset} {who}/{h.kind} {head}") + print(f"{h.score:8.3f} {date} {where} {who}/{h.kind} {head}") if not hits: print("no matches", file=sys.stderr) elif not shown: @@ -546,18 +557,42 @@ def _show(args) -> int: # here: a tool name is any string the transcript chose, so it is collapsed, # and `splitlines()` broke on "\r", "\x85" and U+2028 before `safe_text` # could escape them. + # + # `--redact` masks as recall's does, on each block whole, before the split: + # a private key's body is the lines under its header. These lines keep + # their breaks, where recall's are flattened first, so the mask is asked to + # join `-----BEGIN RSA` and `PRIVATE KEY-----` across one itself. + shown_as = functools.partial(redact.mask, collapse=True) if args.redact else str lo = max(0, at - args.context) for i, t in enumerate(turns[lo : at + args.context + 1], start=lo): # A subagent's user turn is the brief its parent wrote, not the user. who = f"subagent:{t.role}" if t.is_sidechain else t.role - print(f"{'>' if i == at else '-'} {key}@{t.byte_offset} {_date(t)} {who}") + # One field, as in recall: the marker's space would split the address. + where = shown_as(f"{key}@{t.byte_offset}").replace(" ", "") + print(f"{'>' if i == at else '-'} {where} {_date(t)} {who}") for b in t.blocks: - print(" " + " ".join([b.kind, *(b.tool_name or "").split()])) - for line in b.text.split("\n") if b.text else (): + print(" " + shown_as(" ".join([b.kind, *(b.tool_name or "").split()]))) + for line in shown_as(b.text).split("\n") if b.text else (): print(f" {line}") return 0 +def _redact_flag(parser: argparse.ArgumentParser) -> None: + """`--redact`, defined once for each command that prints transcript text. + + The flag can only turn masking on. A standing `GITMEMORY_REDACT` is undone + for one call by `GITMEMORY_REDACT=0`, and any value but empty or `0` means + on, so a mistyped one fails toward masking. + """ + parser.add_argument( + "--redact", + action="store_true", + default=os.environ.get("GITMEMORY_REDACT", "") not in ("", "0"), + help="mask the credential shapes the push gate detects in what is printed " + "(default on when $GITMEMORY_REDACT is set and not 0)", + ) + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(prog="gitmemory") # The plugin and the CLI are installed apart and can be releases apart, so @@ -648,6 +683,7 @@ def main(argv: list[str] | None = None) -> int: help="which hit goes first when they disagree: latest-user, latest, relevance, or " "module:function (default $GITMEMORY_POLICY or %(default)s)", ) + _redact_flag(rec) rec.set_defaults(fn=_recall) sho = sub.add_parser("show", help="print the turn at an address recall printed, from the store") @@ -655,6 +691,7 @@ def main(argv: list[str] | None = None) -> int: sho.add_argument( "-C", "--context", type=_count, default=0, metavar="N", help="also N turns either side" ) + _redact_flag(sho) sho.set_defaults(fn=_show) args = ap.parse_args(argv) diff --git a/src/gitmemory/redact.py b/src/gitmemory/redact.py index 8836f0b..dbcd886 100644 --- a/src/gitmemory/redact.py +++ b/src/gitmemory/redact.py @@ -6,7 +6,8 @@ would make the bytes we commit not the bytes the agent wrote (DESIGN.md §2.5a records agentcairn losing exactly that way). So redaction lives at the two boundaries where bytes leave: anything written to `derived/`, and anything -pushed to a remote. +pushed to a remote. `mask` also serves `recall --redact` and `show --redact`, +which mask what they print when the owner asks them to, and block nothing. Two tiers, because a one-tier gate is either useless or unpassable: @@ -30,18 +31,20 @@ from __future__ import annotations +import bisect import os import re import stat import tomllib import urllib.parse from collections.abc import Iterable -from dataclasses import dataclass +from dataclasses import dataclass, replace __all__ = [ "Finding", "contents", "gate", + "mask", "push_allowed", "safe_path", "scan_bytes", @@ -172,18 +175,103 @@ def safe_path(path: str) -> str: append-only. Masking the span rather than the whole string keeps the finding useful: it still names the directory and the file. [E7] """ - data = path.encode("utf-8", "surrogateescape") - found = scan_bytes(data) - if not found: - return path - out, cursor = b"", 0 + return mask(path) + + +# Where a PEM block ends. Not the detector with BEGIN swapped for END: the gate +# refuses on the header alone, and this only has to say where the key stops. +# Bounded, because `[A-Z0-9 ]*KEY[A-Z ]*` retried every `KEY` in a `-----END +# KEY KEY …` line to the end of it: 125 KB took 4.1 s. The longest real END +# line has eighteen characters before `KEY` and six after; one longer than the +# bounds is not found, and the mask runs to the end of the text instead. +_PEM_END = re.compile(rb"-----END [A-Z0-9 ]{0,40}KEY[A-Z ]{0,10}-----") + +# How `records.safe_text` spells a control or an invisible character, which is +# how the index stores one: `\x7f`, `\u200b`, and `\ue0041` for a tag character. +_SPELLED = re.compile(rb"\\x[0-9a-f]{2}|\\u(?:e00[0-7][0-9a-f]|[0-9a-f]{4})") + +# A whitespace run that `" ".join(text.split())` narrows: two or more, or one +# that is not a space. Greedy and unanchored on either side, so it stays linear. +_RUN = re.compile(rb"\s{2,}|[\t\n\r\f\v]") + + +def mask(text: str, *, collapse: bool = False) -> str: + """`text` with every match of both tiers masked, the rest left readable. + + One span is widened. `private_key_block` matches the BEGIN line, and the + secret is the base64 under it: masking the match alone printed the header + as `----…[31 bytes]` and then every byte of the key. So that span runs to + the next END line, or to the end of `text` when there is none. + + Text holding a spelled-out control is scanned twice. Every spelling ends in + a word character, so `\\ufeffAKIA…` has no `\\b` before the key, where the + push gate, reading the raw bytes, sees a BOM and a boundary. The second scan + reads a copy with each spelling blanked to as many spaces, so its offsets + are the text's. The first stays, because a spelling can sit inside a match + (`password=abc\\x01defghijklmn`) that the blanked copy cuts short. + + `collapse` is for a caller that prints the line breaks. `recall` flattens + each line before it masks, and `-----BEGIN RSA\\nPRIVATE KEY-----` is a + header only once flattened, so a caller that keeps the break asks for a + third scan, of the blanked copy with every whitespace run one space. Its END + line is still looked for as written, so one a break split is not found, and + the mask runs to the end of the text. + """ + data = text.encode("utf-8", "surrogateescape") + plain = _SPELLED.sub(lambda m: b" " * len(m[0]), data) + found = scan_bytes(data) + (scan_bytes(plain) if plain != data else []) + if collapse: + found += _found_collapsed(plain) + widened, reach = [], -1 + for f in sorted(found, key=lambda f: f.offset): + # Only a header no earlier span already covers. Each one searching for + # its own END line made a page of headers with none cost headers x + # length: 16,000 of them, 437 KB, took 2.4 s. + if f.detector == "private_key_block" and f.offset >= reach: + end = _PEM_END.search(plain, f.offset + f.length) + reach = end.end() if end else len(data) + f = replace(f, length=reach - f.offset) + widened.append(f) + if not widened: + return text + out, cursor = [], 0 # Merged spans, because two detectors matching the same key (`sk-ant-…` # trips both `anthropic_api_key` and `openai_api_key`) would otherwise mask - # it twice and mangle the rest of the path with it. - for start, end in _merged(found): - out += data[cursor:start] + _mask(data[start:end]).encode("utf-8") + # it twice and mangle the rest of the text with it. Joined once at the end: + # `bytes +=` copies everything so far for every span. + for start, end in _merged(widened): + out += (data[cursor:start], _mask(data[start:end]).encode("utf-8")) cursor = end - return (out + data[cursor:]).decode("utf-8", "replace") + out.append(data[cursor:]) + return b"".join(out).decode("utf-8", "replace") + + +def _found_collapsed(data: bytes) -> list[Finding]: + """What a scan finds in `data` with each whitespace run one space, at the + offsets of `data`: a match that starts or ends on a run's space covers the + whole run.""" + pieces, spaces, ends, cursor, shift = [], [], [], 0, 0 + for m in _RUN.finditer(data): + pieces += (data[cursor : m.start()], b" ") + # Where the run's one space lands in the copy, and where the run ends. + spaces.append(m.start() - shift) + ends.append(m.end()) + shift += len(m[0]) - 1 + cursor = m.end() + if not spaces: + return [] + pieces.append(data[cursor:]) + + def back(x: int) -> int: + # The last run whose space is before `x`; past it, bytes map one to one. + i = bisect.bisect_left(spaces, x) - 1 + return x if i < 0 else ends[i] + x - spaces[i] - 1 + + found = [] + for f in scan_bytes(b"".join(pieces)): + start = back(f.offset) + found.append(replace(f, offset=start, length=back(f.offset + f.length) - start)) + return found def _merged(findings: list[Finding]) -> list[tuple[int, int]]: diff --git a/tests/conftest.py b/tests/conftest.py index 9a04c23..d95705f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,6 +35,10 @@ def _no_real_home(tmp_path, monkeypatch): home.mkdir(exist_ok=True) monkeypatch.setenv("HOME", str(home)) monkeypatch.delenv("GITMEMORY_HOME", raising=False) + # The docs tell the owner to make masking standing in `settings.json`, and a + # Claude Code session running this suite inherits that: every test that + # reads recall's default output would read the masked one instead. + monkeypatch.delenv("GITMEMORY_REDACT", raising=False) return home diff --git a/tests/mutate_index.py b/tests/mutate_index.py index 39ce9cb..d6a9138 100644 --- a/tests/mutate_index.py +++ b/tests/mutate_index.py @@ -3902,6 +3902,128 @@ " tick.spool_consumed += 1", "test_one_watcher_pass_is_what_clears_it", ), + # --- recall --redact: masked where the detectors can still see the key -- # + ( + "recall --redact masks the head after the 160-character cut", + "__main__.py", + ' head = shown_as(" ".join(h.text.split()))[:160]', + ' head = shown_as(" ".join(h.text.split())[:160])', + "test_recall_redact_masks_a_key_the_160_character_cut_would_have_halved", + ), + ( + "recall --redact masks the head before the collapse assembles a header", + "__main__.py", + ' head = shown_as(" ".join(h.text.split()))[:160]', + ' head = " ".join(shown_as(h.text).split())[:160]', + "test_recall_redact_masks_a_header_that_only_the_collapse_assembles", + ), + ( + "recall --redact masks a private key's header and prints its body", + "redact.py", + " f = replace(f, length=reach - f.offset)\n", + "", + "test_recall_redact_masks_a_private_key_body_and_not_only_its_header", + ), + ( + "recall --redact prints a session id that is itself a credential", + "__main__.py", + ' where = shown_as(f"{h.session_key}@{h.byte_offset}").replace(" ", "")', + ' where = f"{h.session_key}@{h.byte_offset}".replace(" ", "")', + "test_recall_redact_masks_a_session_id_that_is_itself_a_credential", + ), + ( + # The marker is `ghp_…[40 bytes]`, and its space moved role/kind to [4]. + "recall --redact splits a masked address into two columns", + "__main__.py", + ' where = shown_as(f"{h.session_key}@{h.byte_offset}").replace(" ", "")', + ' where = shown_as(f"{h.session_key}@{h.byte_offset}")', + "test_recall_redact_masks_a_session_id_that_is_itself_a_credential", + ), + ( + # The obvious spelling, and it reads `GITMEMORY_REDACT=0` as on. + "recall --redact reads GITMEMORY_REDACT=0 as on", + "__main__.py", + ' default=os.environ.get("GITMEMORY_REDACT", "") not in ("", "0"),', + ' default=bool(os.environ.get("GITMEMORY_REDACT")),', + "test_gitmemory_redact_makes_masking_standing_and_0_turns_it_off", + ), + ( + # The index spells a BOM `\ufeff`, and its `f` takes away the key's `\b`. + "recall --redact prints a key behind a control the index spells out", + "redact.py", + ' plain = _SPELLED.sub(lambda m: b" " * len(m[0]), data)', + " plain = data", + "test_recall_redact_masks_a_key_behind_a_character_the_index_spells_out", + ), + ( + "recall --redact scans only the copy with the spellings blanked", + "redact.py", + " found = scan_bytes(data) + (scan_bytes(plain) if plain != data else [])", + " found = scan_bytes(plain)", + "test_recall_redact_masks_a_key_behind_a_character_the_index_spells_out", + ), + ( + "every private key header searches for its own END line again", + "redact.py", + ' if f.detector == "private_key_block" and f.offset >= reach:', + ' if f.detector == "private_key_block":', + "test_mask_stays_linear_on_a_page_of_key_headers_and_an_end_line_that_never_closes", + ), + ( + "the END line pattern's runs are unbounded again", + "redact.py", + "{0,40}KEY[A-Z ]{0,10}-----", + "*KEY[A-Z ]*-----", + "test_mask_stays_linear_on_a_page_of_key_headers_and_an_end_line_that_never_closes", + ), + # --- show --redact: the whole turn, masked as recall's line is ---------- # + ( + # A private key's body is the lines under its header, and no detector + # matches a line of base64 on its own. + "show --redact masks each line apart and prints a private key's body", + "__main__.py", + ' for line in shown_as(b.text).split("\\n") if b.text else ():', + ' for line in map(shown_as, b.text.split("\\n")) if b.text else ():', + "test_show_redact_masks_a_whole_turn_and_the_default_prints_it", + ), + ( + "show --redact prints a session id that is itself a credential", + "__main__.py", + ' where = shown_as(f"{key}@{t.byte_offset}").replace(" ", "")', + ' where = f"{key}@{t.byte_offset}"', + "test_show_redact_masks_a_whole_turn_and_the_default_prints_it", + ), + ( + "show --redact splits a masked address into two fields", + "__main__.py", + ' where = shown_as(f"{key}@{t.byte_offset}").replace(" ", "")', + ' where = shown_as(f"{key}@{t.byte_offset}")', + "test_show_redact_masks_a_whole_turn_and_the_default_prints_it", + ), + ( + "show --redact prints a tool's name unmasked", + "__main__.py", + ' print(" " + shown_as(" ".join([b.kind, *(b.tool_name or "").split()])))', + ' print(" " + " ".join([b.kind, *(b.tool_name or "").split()]))', + "test_show_redact_masks_a_whole_turn_and_the_default_prints_it", + ), + ( + # recall flattens a turn before it masks; show prints the line breaks. + "show --redact masks without joining a key header a line break split", + "__main__.py", + "functools.partial(redact.mask, collapse=True)", + "functools.partial(redact.mask, collapse=False)", + "test_show_redact_joins_a_key_header_a_line_break_split", + ), + ( + # The copy is shorter by every run it narrowed, so its offsets are not + # the text's: unmapped, the mask eats the bytes before the key. + "the collapsed scan masks at the copy's offsets, not the text's", + "redact.py", + " return x if i < 0 else ends[i] + x - spaces[i] - 1", + " return x", + "test_show_redact_joins_a_key_header_a_line_break_split", + ), # --- E7 index-F8: the caps bound the shape of the input, not its cost -- # ( "sumy's own quadratic idf is used again", @@ -4647,7 +4769,7 @@ # anchor appear twice and the harness would skip itself. [E7] "the docstring's row count drifts away from the index again", "tests/mutate_index.py", - "a full pass is 6" + "38 mutants", + "a full pass is 6" + "54 mutants", "a full pass is 14" + "6 mutants", "test_the_row_count_in_the_docstring_is_the_row_count", ), @@ -5515,15 +5637,15 @@ ( "show prints a tool name's newline, so a transcript forges a header", "__main__.py", - ' print(" " + " ".join([b.kind, *(b.tool_name or "").split()]))', - ' print(" " + " ".join([b.kind, *filter(None, [b.tool_name])]))', + ' print(" " + shown_as(" ".join([b.kind, *(b.tool_name or "").split()])))', + ' print(" " + shown_as(" ".join([b.kind, *filter(None, [b.tool_name])])))', "test_a_transcript_cannot_forge_a_show_header_or_reach_the_terminal", ), ( "show splits block text before a carriage return is escaped", "__main__.py", - ' for line in b.text.split("\\n") if b.text else ():', - " for line in b.text.splitlines() if b.text else ():", + ' for line in shown_as(b.text).split("\\n") if b.text else ():', + " for line in shown_as(b.text).splitlines() if b.text else ():", "test_a_transcript_cannot_forge_a_show_header_or_reach_the_terminal", ), ( @@ -5894,7 +6016,7 @@ def _select(test: str) -> list[str]: def main() -> int: """Run every mutant, or only those whose name contains an argument. - The filter exists because a full pass is 638 mutants × two suite runs — + The filter exists because a full pass is 654 mutants × two suite runs — long enough that adding one row and checking it used to mean either waiting for the other 567 or trusting the new one untested. diff --git a/tests/test_index.py b/tests/test_index.py index d901f1b..eb95941 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -22,7 +22,7 @@ import pytest -from gitmemory import gitrepo, index, records, store +from gitmemory import gitrepo, index, records, redact, store from gitmemory.__main__ import main @@ -1720,3 +1720,190 @@ def test_one_watcher_pass_is_what_clears_it(home, src, capsys): capsys.readouterr() assert main(["--home", home, "recall", "marmoset"]) == 0 assert "uncollected" not in capsys.readouterr().err + + +# --------------------------------------------------------------------------- # +# recall --redact: the owner asks for the gate's shapes to be masked +# --------------------------------------------------------------------------- # + +# Synthetic, right shape, and assembled here so that no credential-shaped +# literal sits in the tree for a push scanner to find. +AKIA = "AKIA" + "ZQ" * 8 +PEM_BODY = "b" * 64 +PEM = ( + "-----BEGIN RSA " + "PRIVATE KEY-----\n" + PEM_BODY + "\n" + PEM_BODY + "\n" + "-----END RSA " + "PRIVATE KEY-----" +) + + +def _recall_out(home: str, capsys, *argv: str) -> str: + assert main(["--home", home, "recall", *argv]) == 0 + return capsys.readouterr().out + + +def test_recall_redact_masks_a_key_the_160_character_cut_would_have_halved(home, src, capsys): + """Masked before the cut. The key starts at character 150, so the cut keeps + ten of its twenty characters, and ten match no detector: masking the + printed head instead would scan a fragment clean and print it.""" + text = "zebra " * 25 + AKIA + " tail" + assert text.index(AKIA) == 150 + built(home, src, [user("u1", text)]).close() + capsys.readouterr() + assert AKIA[:10] in _recall_out(home, capsys, "zebra"), "the default stays verbatim" + out = _recall_out(home, capsys, "zebra", "--redact") + assert "AKIA…[" in out and AKIA[4:10] not in out, out + + +def test_recall_redact_masks_a_private_key_body_and_not_only_its_header(home, src, capsys): + """`private_key_block` matches the BEGIN line, and the key is the base64 + under it. Masking the match alone printed `----…[31 bytes]` followed by + every byte of the key.""" + built(home, src, [user("u1", "pemcase " + PEM + " after the key")]).close() + capsys.readouterr() + assert PEM_BODY[:8] in _recall_out(home, capsys, "pemcase"), "premise: the body is there" + out = _recall_out(home, capsys, "pemcase", "--redact") + assert PEM_BODY[:8] not in out and "PRIVATE KEY" not in out, out + assert "pemcase ----…[" in out and "after the key" in out, out + + +def test_recall_redact_masks_a_header_that_only_the_collapse_assembles(home, src, capsys): + """`-----BEGIN RSA\\nPRIVATE KEY-----` scans clean as written and is a + header once the line is flattened, and the flattened line is what recall + prints. So the mask runs after the collapse, as `graph._label` found it + had to. [E7b L3-F1]""" + split = PEM.replace("RSA PRIVATE", "RSA\nPRIVATE", 1) + assert redact.mask(split) == split, "premise: nothing matches before the collapse" + built(home, src, [user("u1", "pemcase " + split)]).close() + capsys.readouterr() + out = _recall_out(home, capsys, "pemcase", "--redact") + assert PEM_BODY[:8] not in out and "PRIVATE KEY" not in out, out + + +def test_recall_redact_masks_a_session_id_that_is_itself_a_credential(home, src, capsys): + """The address is printed too, and `_SAFE_RE` accepts a token as a session + id. Lowercase, because the store case-folds ids. + + The masked address stays one column. The mask's marker has a space in it, + and with it `split()[3]` read `bytes]/g00@0` where role/kind belongs.""" + ghp = "ghp_" + "a" * 36 + write(src, [user("u1", "walrus note")]) + store.capture(src, "claude-code", ghp, home=home) + index.build(home) + capsys.readouterr() + assert ghp in _recall_out(home, capsys, "walrus") + out = _recall_out(home, capsys, "walrus", "--redact") + assert ghp not in out and "/ghp_…[40bytes]/g00@0 user/text" in out, out + assert out.split()[3] == "user/text", out + + +def test_recall_redact_masks_a_key_the_raw_bytes_only_spell_escaped(home, src, capsys): + """The push gate reads raw bytes and cannot see `\\u0073k-ant-…`. The index + holds the parsed text, where the escape is already the key, so recall's + mask sees what the push gate does not.""" + key = "sk-ant-api03-" + "A1b2_-" * 15 + with open(src, "ab") as fh: + fh.write( + b'{"type":"user","uuid":"u1","sessionId":"s1","message":{"role":"user",' + b'"content":"escaped \\u0073' + key[1:].encode() + b' unicorn"}}\n' + ) + assert redact.scan_bytes(Path(src).read_bytes()) == [], "premise: the raw bytes scan clean" + store.capture(src, "claude-code", "sess", home=home) + index.build(home) + capsys.readouterr() + out = _recall_out(home, capsys, "unicorn", "--redact") + assert key[4:20] not in out and "sk-a…[103 bytes]" in out, out + + +def test_gitmemory_redact_makes_masking_standing_and_0_turns_it_off( + home, src, capsys, monkeypatch +): + """Unset, empty and `0` are off, and anything else is on, so a mistyped + value fails toward masking. Read when the parser is built, as + `GITMEMORY_POLICY` is.""" + built(home, src, [user("u1", "aws " + AKIA + " done")]).close() + capsys.readouterr() + for value, masked in (("1", True), ("yes", True), ("0", False), ("", False)): + monkeypatch.setenv("GITMEMORY_REDACT", value) + assert (AKIA not in _recall_out(home, capsys, "aws")) is masked, value + monkeypatch.delenv("GITMEMORY_REDACT") + assert AKIA in _recall_out(home, capsys, "aws") + assert AKIA not in _recall_out(home, capsys, "aws", "--redact") + + +def test_mask_returns_clean_text_unchanged_and_keeps_the_words_around_a_key(): + clean = "the lock is held across the await, see a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0" + assert redact.mask(clean) is clean + assert redact.mask(f"use {AKIA} for staging") == "use AKIA…[20 bytes] for staging" + # Each header runs to its own END line, not the last one in the text. + two = redact.mask(f"one {PEM} between {PEM} two") + assert two == f"one ----…[{len(PEM)} bytes] between ----…[{len(PEM)} bytes] two", two + # Cut before its END line, the rest of the text is the key, so the rest goes. + assert redact.mask("x " + PEM[:80]) == "x ----…[80 bytes]" + + +def test_recall_redact_masks_a_key_behind_a_character_the_index_spells_out(home, src, capsys): + """The push gate reads raw bytes, and a BOM, a zero-width space or a DEL in + front of a key is not a word character there, so the key's `\\b` holds and + the gate finds it. The index holds `records.safe_text`'s spelling of each, + `\\ufeff`, `\\u200b`, `\\x7f`, and every spelling ends in a word character: + the boundary was gone and `recall --redact` printed all three whole. A `cat` + of an `.env` saved with a BOM is the ordinary way in. + + The password has a control inside its value, which a scan of the blanked + copy alone cuts to three characters and misses, so both scans stay.""" + ghp = "ghp_" + "a" * 36 + secret = "pass" + "word=abc" + "\x01" + "defghijklmn" + text = f"marmot \ufeff{AKIA} x\u200b{ghp} y\x7f{AKIA} z {secret} end" + line = {"type": "user", "uuid": "u1", "sessionId": "s1"} + line["message"] = {"role": "user", "content": text} + # Literal UTF-8, as Node's JSON.stringify writes it, which is what the gate reads. + with open(src, "wb") as fh: + fh.write(json.dumps(line, ensure_ascii=False).encode() + b"\n") + gate = [f.detector for f in redact.scan_bytes(Path(src).read_bytes())] + assert gate.count("aws_access_key_id") == 2 and "github_token" in gate, gate + assert "assigned_secret" in gate, gate + store.capture(src, "claude-code", "sess", home=home) + index.build(home) + capsys.readouterr() + assert AKIA in _recall_out(home, capsys, "marmot"), "premise: the default prints them" + out = _recall_out(home, capsys, "marmot", "--redact") + assert AKIA[4:10] not in out and ghp[4:20] not in out and "defghijklmn" not in out, out + assert "\\ufeffAKIA…[20 bytes] x\\u200bghp_…[40 bytes] y\\x7fAKIA…[20 bytes]" in out, out + + +def test_mask_stays_linear_on_a_page_of_key_headers_and_an_end_line_that_never_closes(): + """Two ways the PEM widening went quadratic on text an agent fetched. + + Each header searched for its own END line, so headers with none cost + headers x length: 16,000 of them (437 KB) took 2.4 s where `scan_bytes` + took 0.06 s, and a 2 MB tool_result of them kept `recall --redact` for + 41 s. And the END pattern's unbounded runs retried every `KEY` in a + `-----END KEY KEY …` line to its end: 125 KB took 4.1 s. + + At these sizes each would be about 15 s and 10 s. 1.5 s is several times + what the linear code costs here, and fails long before anyone would wait. + """ + header = "-----BEGIN " + "PRIVATE KEY-----\n" + for label, text in ( + ("headers with no END", header * 40_000), + ("an END line that never closes", header + "-----END " + "KEY " * 50_000), + ): + started = time.monotonic() + out = redact.mask(text) + elapsed = time.monotonic() - started + assert out == f"----…[{len(text)} bytes]", (label, out[:80]) + assert elapsed < 1.5, f"{label}: the PEM widening went quadratic again: {elapsed:.2f}s" + + +def test_recall_redact_keeps_a_subagent_turn_marked(home, tmp_path, capsys): + """The mask and the subagent mark compose. `--redact` changes what the + address and the text say, never which field role/kind is in or whose turn + a line is.""" + with_a_subagent(home, tmp_path) + capsys.readouterr() + out = _recall_out(home, capsys, "retry loop upload", "--redact") + assert [line.split()[3] for line in out.splitlines()] == [ + "user/text", + "subagent:user/text", + "assistant/text", + ], out diff --git a/tests/test_show.py b/tests/test_show.py index 1bf5db9..dabd362 100644 --- a/tests/test_show.py +++ b/tests/test_show.py @@ -15,9 +15,9 @@ import os import pytest -from test_index import assistant, user, write +from test_index import AKIA, PEM, PEM_BODY, assistant, user, write -from gitmemory import store +from gitmemory import redact, store from gitmemory.__main__ import main @@ -241,3 +241,51 @@ def test_show_on_no_store_and_a_negative_context_exit_2(tmp_path, capsys): with pytest.raises(SystemExit) as done: main(["--home", str(tmp_path), "show", "claude-code/sess/g00@0", "-C", "-1"]) assert done.value.code == 2 + + +def test_show_redact_masks_a_whole_turn_and_the_default_prints_it(home, src, capsys): + """What `recall --redact` masked one line of, whole. A private key's body is + the lines under its header, so a block is masked before it is split into + lines, never one line at a time. The address and a tool's name are masked + too, and the address stays one field, as recall's does.""" + ghp = "ghp_" + "a" * 36 # lowercase, because the store case-folds a session id + text = f"the deploy key:\n{PEM}\nand {AKIA} for staging" + tool = {"type": "tool_use", "id": "t", "name": ghp, "input": {}} + write(src, [user("u1", text), assistant("a2", [tool])]) + store.capture(src, "claude-code", ghp, home=home) + address = f"claude-code/{ghp}/g00@0" + code, out, _ = show(home, capsys, address, "-C", "1") + assert code == 0 and PEM_BODY in out and AKIA in out and out.count(ghp) == 3, out + code, out, err = show(home, capsys, address, "-C", "1", "--redact") + assert (code, err) == (0, "") + undated = "undated".ljust(17) + assert out.splitlines() == [ + f"> claude-code/ghp_…[40bytes]/g00@0 {undated} user", + " text", + " the deploy key:", + f" ----…[{len(PEM)} bytes]", + " and AKIA…[20 bytes] for staging", + f"- claude-code/ghp_…[40bytes]/g00@{size(user('u1', text))} {undated} assistant", + " tool_use ghp_…[40 bytes]", + ], out + assert ghp[4:20] not in out and AKIA[4:10] not in out and PEM_BODY[:8] not in out, out + + +def test_show_redact_joins_a_key_header_a_line_break_split(home, src, capsys): + """`-----BEGIN RSA\\nPRIVATE KEY-----` is no header until the line is + flattened. recall flattens a turn before it masks; show keeps the breaks, so + its mask flattens a copy itself and masks what that finds in the text as + written, leaving the lines around the key as they were. [E7b L3-F1]""" + split = PEM.replace("RSA PRIVATE", "RSA\nPRIVATE", 1) + text = f"first words\n\n {split}\nlast words" + assert redact.mask(text) == text, "premise: nothing matches with the break in" + write(src, [user("u1", text)]) + store.capture(src, "claude-code", "sess", home=home) + code, out, _ = show(home, capsys, "claude-code/sess/g00@0", "--redact") + assert code == 0 + assert out.splitlines()[2:] == [ + " first words", + " ", + f" ----…[{len(split)} bytes]", + " last words", + ], out diff --git a/tests/test_store.py b/tests/test_store.py index da7a0d8..28322e8 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -1994,7 +1994,8 @@ def test_a_credential_in_an_exception_message_is_masked_on_the_way_out(home, cap An `OSError` carries the path it failed on. Nobody asked to see that string — it is a diagnostic — and `recall`, which prints transcript content because - that is what it is for, is deliberately not masked. Only exception text is. + that is what it is for, is not masked unless the owner asks (`--redact`). + Exception text always is. """ from gitmemory.__main__ import main From 3bb0aff145a38f0bdc71c9c70833801409e96ce9 Mon Sep 17 00:00:00 2001 From: doronp <3586743+doronp@users.noreply.github.com> Date: Sun, 18 Oct 2026 13:27:55 +0300 Subject: [PATCH 09/13] feat: say captured sizes for people, keeping the exact count `watch` logged `captured 470 +43025169B commit=...` and `capture` printed ` +6202B size=11550 g00`, so a person had to count digits to know how much a pass took. Both now say the amount through `daemon.size_text()`: `+392.0 MB (391995999 B)`, or `+N B` under 1000 bytes. The units are decimal, so the rounded figure and the exact count in parentheses can be checked against each other by moving a point, and a script doing byte arithmetic still has the count. `size=` stays plain bytes: it is the end of `[0, size)`, the coordinate `verify` and every `@offset` are stated in, not an amount. The git commit subject keeps `+NB`, because it is the durable record and the history already written says it that way; the watch-line test pins that too. Three mutation rows, each caught by its named test (657 in all): the watch line and the capture line reverted to `+NB`, and the 999.95 guard loosened so 999950 B reads "1000.0 kB". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QPBLmJD2q1PVxdkWx8C16U Signed-off-by: doronp <3586743+doronp@users.noreply.github.com> --- CHANGELOG.md | 5 +++++ README.md | 8 ++++---- docs/REPRODUCE.md | 8 ++++---- docs/USAGE.md | 2 +- src/gitmemory/__main__.py | 5 +++-- src/gitmemory/daemon.py | 24 ++++++++++++++++++++++-- tests/mutate_index.py | 26 ++++++++++++++++++++++++-- tests/test_daemon.py | 35 ++++++++++++++++++++++++++++++++++- tests/test_store.py | 25 +++++++++++++++++++++++-- 9 files changed, 120 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bef05e..5f9e36d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,11 @@ tell you. 3.13 interpreter. A test reads the floor from `pyproject.toml` and holds every install line to it. See [README.md](README.md#quickstart). +- **Sizes for people.** `watch` and `capture` say how much a pass captured as + `+392.0 MB (391995999 B)` instead of `+391995999B`, keeping the exact count in + parentheses; under 1000 bytes it is `+N B`. A script that parsed `+NB` out of + either line needs the new form. `size=` stays an exact offset, and the commit + subject keeps `+NB`. See [USAGE.md](docs/USAGE.md#set-it-up-once). ### Fixed diff --git a/README.md b/README.md index 165a8b8..41c6bf8 100644 --- a/README.md +++ b/README.md @@ -210,13 +210,13 @@ real text, and that is a measurement rather than a suspicion. | What | How it was measured | Result | |---|---|---| | Hook cost in the agent's critical path | Timed against spawning `true` the same way, three runs of 400 | p50 **7.4 – 7.5 ms**, p99 **10.2 – 11.5 ms** | -| The suite | On a fresh checkout, no downloads | **1,119 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | -| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **654** negative controls | +| The suite | On a fresh checkout, no downloads | **1,128 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | +| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **657** negative controls | The two suite counts do not add up, and should not. Switching the corpora on -collects 1444, not 1447. Three conformance cases fill parametrisations that +collects 1453, not 1456. Three conformance cases fill parametrisations that collect as one empty placeholder each while the corpora are absent, so they -replace three of the 1119 rather than joining them. Every figure on this board +replace three of the 1128 rather than joining them. Every figure on this board is pinned by a test, which is how it stays true. **What is not measured**, and is shown as a row on the dashboard rather than diff --git a/docs/REPRODUCE.md b/docs/REPRODUCE.md index 6653f98..72fd876 100644 --- a/docs/REPRODUCE.md +++ b/docs/REPRODUCE.md @@ -484,11 +484,11 @@ uv run pytest -q --collect-only -p no:cacheprovider | tail -1 uv run pytest -q --collect-only -p no:cacheprovider | grep -c '\.jsonl\]$' ``` -**Expected:** `1119/1120 tests collected (1 deselected)`, then -`1444/1445 tests collected (1 deselected)`, then `328`. The deselected test is +**Expected:** `1128/1129 tests collected (1 deselected)`, then +`1453/1454 tests collected (1 deselected)`, then `328`. The deselected test is the LongMemEval corpus test, which is opt-in: `uv run pytest -q -m corpus` after `bench/fetch_longmemeval.sh`. The README's -paragraph on why 1,119 + 328 is 1,444 and not 1,447 is the arithmetic of these +paragraph on why 1,128 + 328 is 1,453 and not 1,456 is the arithmetic of these three numbers. All three were run while writing this guide. `tests/test_docs.py` holds the README to the first and third numbers, so a @@ -505,7 +505,7 @@ test fail. The count is the number of rows in `tests/mutate_index.py`. uv run python -c "import sys; sys.path.insert(0, 'tests'); from mutate_index import MUTANTS; print(len(MUTANTS))" ``` -**Expected:** `654`. Checked while writing this guide. +**Expected:** `657`. Checked while writing this guide. To run the controls rather than count them: diff --git a/docs/USAGE.md b/docs/USAGE.md index 72ee9f5..15d320f 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -55,7 +55,7 @@ To check the loop end to end without waiting, run one pass: ```console $ gitmemory watch --once -captured 1 +1167B commit=f355b10ace4b872ffe77794cbf4183afb1dcf478 +captured 1 +1.2 kB (1167 B) commit=f355b10ace4b872ffe77794cbf4183afb1dcf478 $ gitmemory verify 0 problem(s) ``` diff --git a/src/gitmemory/__main__.py b/src/gitmemory/__main__.py index b338c75..f982288 100644 --- a/src/gitmemory/__main__.py +++ b/src/gitmemory/__main__.py @@ -146,7 +146,7 @@ def _capture(args) -> int: ) # Said before the result line and on stderr, beside `diverged`, because it # is the same kind of fact: something about this store is not the ordinary - # case. A capture that reclaims an orphan reports `+0B`, so without this the + # case. A capture that reclaims an orphan reports `+0 B`, so without this the # loudest thing about the pass is how quiet it was. [E7 fs-F4] if cap.adopted: print( @@ -157,7 +157,8 @@ def _capture(args) -> int: if cap.diverged: print(f"diverged: {cap.diverged}", file=sys.stderr) print(f"sealed generation {cap.generation - 1}; now writing g{cap.generation:02d}") - print(f"{cap.manifest_path} +{cap.appended}B size={cap.size} g{cap.generation:02d}") + added = daemon.size_text(cap.appended) + print(f"{cap.manifest_path} +{added} size={cap.size} g{cap.generation:02d}") return 0 diff --git a/src/gitmemory/daemon.py b/src/gitmemory/daemon.py index 28b83cf..3815ae8 100644 --- a/src/gitmemory/daemon.py +++ b/src/gitmemory/daemon.py @@ -118,7 +118,8 @@ class Tick: # Segments this pass attested to without copying a byte — reclaimed from the # generation directory rather than read out of the source. The manifest # records them permanently; this is so the operator hears about it at the - # time, since a repair that shows up only as "+0B" reads like a quiet pass. + # time, since a repair that shows up only as a "+0B" commit reads like a + # quiet pass: with nothing captured, the log has no `captured` line at all. # Not an error: after a killed capture it is the correct outcome. [E7 fs-F4] adopted: list[str] = field(default_factory=list) @@ -1015,6 +1016,25 @@ def tick( return result +def size_text(n: int) -> str: + """How much a pass captured, for a person: `392.0 MB (391995999 B)`. + + `+391995999B` made the reader count digits. The exact count stays beside the + rounded one for anyone doing byte arithmetic, and the units are decimal so + the two can be checked against each other by moving a point. Amounts only: + `size=` and every offset are coordinates, and stay plain bytes. + """ + if n < 1000: + return f"{n} B" + x = float(n) + for unit in ("kB", "MB", "GB", "TB"): + x /= 1000 + # 999.95 and not 1000: `.1f` would round 999.96 kB up to "1000.0 kB". + if x < 999.95 or unit == "TB": + break + return f"{x:.1f} {unit} ({n} B)" + + def run( home: str | None = None, *, @@ -1176,7 +1196,7 @@ def run( else "" ) log( - f"captured {len(result.captured)} +{result.appended}B " + f"captured {len(result.captured)} +{size_text(result.appended)} " f"commit={result.commit}{dropped}" ) if once: diff --git a/tests/mutate_index.py b/tests/mutate_index.py index d6a9138..df7063b 100644 --- a/tests/mutate_index.py +++ b/tests/mutate_index.py @@ -4769,7 +4769,7 @@ # anchor appear twice and the harness would skip itself. [E7] "the docstring's row count drifts away from the index again", "tests/mutate_index.py", - "a full pass is 6" + "54 mutants", + "a full pass is 6" + "57 mutants", "a full pass is 14" + "6 mutants", "test_the_row_count_in_the_docstring_is_the_row_count", ), @@ -5864,6 +5864,28 @@ 'f"launchctl bootout {job} && rm {shlex.quote(target)}"', "test_the_printed_removal_works_when_the_job_was_never_started", ), + # --- a captured amount said for a person --- + ( + "the watch line goes back to a bare byte count", + "daemon.py", + "+{size_text(result.appended)} ", + "+{result.appended}B ", + "test_the_watch_line_says_how_much_in_units_a_person_reads", + ), + ( + "capture's result line goes back to a bare byte count", + "__main__.py", + " added = daemon.size_text(cap.appended)", + ' added = f"{cap.appended}B"', + "test_cli_capture_says_how_much_in_units_and_where_in_bytes", + ), + ( + "a size just under the next unit rounds up to 1000.0 of this one", + "daemon.py", + ' if x < 999.95 or unit == "TB":', + ' if x < 1000 or unit == "TB":', + "test_a_byte_count_is_said_for_a_person_and_exactly", + ), ] @@ -6016,7 +6038,7 @@ def _select(test: str) -> list[str]: def main() -> int: """Run every mutant, or only those whose name contains an argument. - The filter exists because a full pass is 654 mutants × two suite runs — + The filter exists because a full pass is 657 mutants × two suite runs — long enough that adding one row and checking it used to mean either waiting for the other 567 or trusting the new one untested. diff --git a/tests/test_daemon.py b/tests/test_daemon.py index bf599ca..14f0d9a 100644 --- a/tests/test_daemon.py +++ b/tests/test_daemon.py @@ -869,6 +869,39 @@ def test_run_reports_a_failing_pass_in_its_exit_code(tmp_path): os.chmod(bad, 0o600) +@pytest.mark.parametrize( + "n, said", + [ + (0, "0 B"), + (999, "999 B"), + (1000, "1.0 kB (1000 B)"), + (999_949, "999.9 kB (999949 B)"), + (999_950, "1.0 MB (999950 B)"), # not "1000.0 kB" + (391_995_999, "392.0 MB (391995999 B)"), + (10**15, "1000.0 TB (1000000000000000 B)"), # the last unit has no next one + ], +) +def test_a_byte_count_is_said_for_a_person_and_exactly(n, said): + assert daemon.size_text(n) == said + + +def test_the_watch_line_says_how_much_in_units_a_person_reads(tmp_path): + """`captured 470 +43025169B` made a person count digits to read the pass. + + The commit subject keeps `+NB`: it is the durable record, the history + already written says it that way, and nobody reads it to size a pass. + """ + home = str(tmp_path / "home") + root = tmp_path / "proj" + _write(str(root / "a.jsonl"), TURN * 40) # 2400 bytes + _config(home, [root]) + lines: list[str] = [] + assert daemon.run(home, once=True, interval=0, log=lines.append) == 0 + said = [line for line in lines if line.startswith("captured ")] + assert len(said) == 1 and said[0].startswith("captured 1 +2.4 kB (2400 B) commit="), lines + assert _subject(home).endswith(" +2400B"), _subject(home) + + # --- the seam the hook writes to ------------------------------------------- @@ -1963,7 +1996,7 @@ def test_a_fifo_with_a_writer_is_not_a_record_even_though_it_reads(tmp_path): def test_an_adoption_only_pass_names_what_it_reclaimed(tmp_path): - """`result.captured` is empty and `+0B` is the headline, so nothing else says it. + """`result.captured` is empty and the commit's `+0B` is the headline, so nothing else says it. Adoption attests to bytes found in the generation directory rather than copied out of the source. The manifest records that permanently; this is so diff --git a/tests/test_store.py b/tests/test_store.py index 28322e8..0e4c4aa 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -577,6 +577,27 @@ def test_cli_capture_records_the_compaction_boundary_the_adapter_found(tmp_path, assert main(["--home", home, "verify"]) == 0 +def test_cli_capture_says_how_much_in_units_and_where_in_bytes(tmp_path, capsys): + """`+` is an amount, so a person gets units; `size=` is a coordinate, so it stays bytes. + + `size=` is the end of `[0, size)`, the range `verify` and every `@offset` + are stated in. The second capture is there so the two numbers differ. + """ + from gitmemory.__main__ import main + + home, src = str(tmp_path / "h"), str(tmp_path / "s.jsonl") + os.mkdir(home) + base = transcript(src, 20) + assert main(["--home", home, "capture", src, "--session-id", "sess"]) == 0 + end = transcript(src, 20, start=20) + assert (base, end) == (1500, 3020) + capsys.readouterr() + + assert main(["--home", home, "capture", src, "--session-id", "sess"]) == 0 + out = capsys.readouterr().out + assert out.endswith(" +1.5 kB (1520 B) size=3020 g00\n"), out + + def test_cli_verify_exits_nonzero_on_a_hole(tmp_path): from gitmemory.__main__ import main @@ -2650,7 +2671,7 @@ def test_a_manifest_whose_adopted_is_not_a_list_is_refused(home, src): def test_cli_capture_says_when_it_only_found_the_bytes(tmp_path, capsys): - """A capture that reclaims an orphan reports `+0B` and is otherwise silent. + """A capture that reclaims an orphan reports `+0 B` and is otherwise silent. The manifest is where the fact lives for ever, but a person running `gitmemory capture` after a crash is not reading manifests — and the @@ -2675,7 +2696,7 @@ def test_cli_capture_says_when_it_only_found_the_bytes(tmp_path, capsys): out = capsys.readouterr() assert f"adopted 1 segment(s) found on disk, not copied from the source: {name}" in out.err - assert "+0B" in out.out, "the line has to be beside a result that looks like nothing happened" + assert "+0 B" in out.out, "the line has to be beside a result that looks like nothing happened" # --- E7 fs-F5: the proof tree reached through a link ------------------------- From 0434189783334707109dab9919c1c33bfa87e5e2 Mon Sep 17 00:00:00 2001 From: doronp <3586743+doronp@users.noreply.github.com> Date: Sun, 18 Oct 2026 13:25:34 +0300 Subject: [PATCH 10/13] docs: say how much disk the store takes, and why A store came to 1.75x what it had captured (684 MB for 392 MB), and no document said why. ARCHITECTURE.md said the store "holds each byte the agent wrote exactly once", and DESIGN.md's first consequence of appending was "Zero duplication", which are true of raw/ only: .git keeps a second, zlib-compressed copy (gitrepo.init says so), and index/ holds the searchable text a third time, uncompressed, beside its FTS5 postings. docs/watching.md gets a "How much disk it takes" section: budget about twice what you capture, what each directory costs, a gzip one-liner that estimates .git for a store holding what is under the Claude Code root now (and what that count leaves out), and the measurements behind the advice (du -sk on synthetic 470-transcript corpora, 2.3x for prose and code, 3.4x for base64 tool output, 1.2-1.5x for the part that cannot be rebuilt). It says index/ is the part to delete for space, because gitmemory index rebuilds it from the store, and that recall and dashboard refuse until it is rebuilt. It also says a rebuild needs about twice index/'s usual space while it runs: index.build writes the new database beside the old one and renames it into place, measured at 1.92x of capture steady and 3.82x just before the rename. So when space is tight, delete index/ first. The one real store is a reported data point, not proof of why real transcripts index smaller. The README points at the section, ARCHITECTURE.md scopes "exactly once" to raw/, and DESIGN.md gets a dated bracketed amendment. The ratios are not pinned in tests: they move with block size, git version, gc timing and corpus. The new doc test pins the advice instead. Deleting index/ is safe only while the store's .gitignore ignores index/ and tracks raw/ and sessions/. It also checks that the README's, ARCHITECTURE.md's and DESIGN.md's anchors resolve to a watching.md heading, reading headings with fenced blocks cut out so a `# comment` in a recipe cannot stand in for one. On this branch the section also points at the main-sessions-only pattern as the way to keep subagent and workflow transcripts out of the store, with its cost, and says that `show` reads the store and keeps working while index/ is gone. The new doc test moves the README count to 1,129 and the corpus arithmetic to 1454 and 1457; docs/REPRODUCE.md's collect lines are re-measured to match. No mutation row is added, so 657 stands. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QPBLmJD2q1PVxdkWx8C16U Signed-off-by: doronp <3586743+doronp@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ README.md | 13 +++++++++--- docs/ARCHITECTURE.md | 7 ++++--- docs/DESIGN.md | 3 +++ docs/REPRODUCE.md | 6 +++--- docs/watching.md | 50 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_docs.py | 39 ++++++++++++++++++++++++++++++++++ 7 files changed, 116 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f9e36d..cfde789 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,13 @@ tell you. and only in what the two commands print; see [docs/USAGE.md](docs/USAGE.md#masking-credentials-in-what-it-prints) for what it does not cover. +- **How much disk it takes.** Budget about twice what you capture: `raw/` + holds the bytes once, `.git/` a compressed copy, and `index/` the searchable + text again, which `gitmemory index` rebuilds (and, while it runs, needs room + for the old index and the new). Measured at 2.3x to 3.4x on synthetic + corpora; see [docs/watching.md](docs/watching.md#how-much-disk-it-takes). + ARCHITECTURE.md said the store holds each byte exactly once, and DESIGN.md + that it has zero duplication; both are true of `raw/` only. ### Changed diff --git a/README.md b/README.md index 41c6bf8..e40ffe8 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,13 @@ agent search the store and quote what it finds; the watcher still does the capturing. For watcher flags and troubleshooting, see [docs/watching.md](docs/watching.md). +Budget disk for about twice what the store captures: git keeps a compressed +second copy, and the search index holds the text again. `gitmemory index` +rebuilds the index if you delete it; rebuilding over an existing one briefly +needs room for both. +[docs/watching.md](docs/watching.md#how-much-disk-it-takes) has the +measurements. + **What to do with it next** — reading `recall` output, handing it back to an agent after compaction, derived key ideas, the dashboard — is in [docs/USAGE.md](docs/USAGE.md). @@ -210,13 +217,13 @@ real text, and that is a measurement rather than a suspicion. | What | How it was measured | Result | |---|---|---| | Hook cost in the agent's critical path | Timed against spawning `true` the same way, three runs of 400 | p50 **7.4 – 7.5 ms**, p99 **10.2 – 11.5 ms** | -| The suite | On a fresh checkout, no downloads | **1,128 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | +| The suite | On a fresh checkout, no downloads | **1,129 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | | Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **657** negative controls | The two suite counts do not add up, and should not. Switching the corpora on -collects 1453, not 1456. Three conformance cases fill parametrisations that +collects 1454, not 1457. Three conformance cases fill parametrisations that collect as one empty placeholder each while the corpora are absent, so they -replace three of the 1128 rather than joining them. Every figure on this board +replace three of the 1129 rather than joining them. Every figure on this board is pinned by a test, which is how it stays true. **What is not measured**, and is shown as a row on the dashboard rather than diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index d8088e4..c71a710 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -97,9 +97,10 @@ $GITMEMORY_HOME/ # default ~/.gitmemory; a git repository ``` A capture commits **only the bytes appended since the last recorded offset**, -so the store holds each byte the agent wrote exactly once. If the source is -rewritten (its prefix no longer matches), capture starts a new *generation* -`g` rather than treating the rewrite as corruption. §2.5a. +so `raw/` holds each byte the agent wrote exactly once; git's object store keeps +its own compressed copy ([how much disk](watching.md#how-much-disk-it-takes)). +If the source is rewritten (its prefix no longer matches), capture starts a new +*generation* `g` rather than treating the rewrite as corruption. §2.5a. ## Trust boundaries diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 6d08684..15be97e 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -202,6 +202,9 @@ $GITMEMORY_HOME/ # default ~/.gitmemory Each capture commits **only the bytes appended since the last recorded offset**. Consequences: 1. **Zero duplication.** Stored bytes == bytes the agent ever wrote, not a multiple. + **[2026-10-18 — true of `raw/`, not of the store on disk. `.git/` keeps a zlib-compressed copy + of the same bytes and `index/` holds their text again, so budget about twice what you capture; + [watching.md](watching.md#how-much-disk-it-takes) has the measurements.]** 2. **Plain text, never pre-compressed.** Let git's zlib + delta work on text it understands. Pre-compressing defeats git's native packing. 3. **Reconstruction is `cat` in filename order**, checked against a recorded whole-file sha256. diff --git a/docs/REPRODUCE.md b/docs/REPRODUCE.md index 72fd876..4c422dd 100644 --- a/docs/REPRODUCE.md +++ b/docs/REPRODUCE.md @@ -484,11 +484,11 @@ uv run pytest -q --collect-only -p no:cacheprovider | tail -1 uv run pytest -q --collect-only -p no:cacheprovider | grep -c '\.jsonl\]$' ``` -**Expected:** `1128/1129 tests collected (1 deselected)`, then -`1453/1454 tests collected (1 deselected)`, then `328`. The deselected test is +**Expected:** `1129/1130 tests collected (1 deselected)`, then +`1454/1455 tests collected (1 deselected)`, then `328`. The deselected test is the LongMemEval corpus test, which is opt-in: `uv run pytest -q -m corpus` after `bench/fetch_longmemeval.sh`. The README's -paragraph on why 1,128 + 328 is 1,453 and not 1,456 is the arithmetic of these +paragraph on why 1,129 + 328 is 1,454 and not 1,457 is the arithmetic of these three numbers. All three were run while writing this guide. `tests/test_docs.py` holds the README to the first and third numbers, so a diff --git a/docs/watching.md b/docs/watching.md index 5a72d43..6360165 100644 --- a/docs/watching.md +++ b/docs/watching.md @@ -243,3 +243,53 @@ anywhere useful. The two failures worth watching for are different shapes: There is no discovery counter: a pass prints only when it captured something, so "found 12 transcripts, all unchanged" and "found nothing" are the same silence. Listing the roots yourself is currently the way to tell them apart. + +## How much disk it takes + +Budget about twice what you capture. The store keeps the same bytes three ways: + +- **`raw/`** holds each captured byte once, so it is the size of the capture. + Each segment fills whole filesystem blocks, so a store grown in many small + passes runs a little over. +- **`.git/`** holds them again, zlib-compressed and packed by `git gc --auto`. + That is roughly the capture divided by what gzip gets out of your + transcripts. For a store that holds what is under `~/.claude/projects` now, + `find ~/.claude/projects -name '*.jsonl' -exec cat {} + | gzip | wc -c` + estimates it, a little low. That count leaves out bytes the store captured + before a transcript was rewritten or deleted, and any other root you watch. +- **`index/`** holds the searchable text a third time, uncompressed, beside its + FTS5 postings. Git ignores it and `gitmemory index` rebuilds it from the + store, so it is the one to delete when you need the space back. `recall` + and `dashboard` refuse until you rebuild it; `show` reads the store and + does not need it. A rebuild writes the new database beside the old one and + renames it into place when it is done, so while it runs `index/` takes + about twice its usual space. When space is tight, delete `index/` first, + then rebuild. + +`sessions/` is one small manifest per generation. `derived/`, if you run +`derive`, adds a little more: 0.09x on the first corpus below. + +Measured with `du -sk` (APFS, 4 KiB blocks) on synthetic Claude-Code-shaped +corpora of 470 transcripts, each captured in one pass and then indexed, as +multiples of the bytes captured: + +| Captured | `raw/` | `.git/` | `index/` | `sessions/` | Total | +|---|---|---|---|---|---| +| 43 MB of prose and source code (gzip 4.6x) | 1.02 | 0.24 | 1.02 | 0.04 | 2.3x | +| 163 MB, the same mix | 1.01 | 0.22 | 1.03 | 0.01 | 2.3x | +| 44 MB of base64 tool output (gzip 2.6x) | 1.02 | 0.40 | 1.94 | 0.04 | 3.4x | + +The part you cannot rebuild, `raw/`, `.git/` and `sessions/`, came to 1.2-1.5x +in each. Captured in ten passes instead of one, that part of the first corpus +came to 1.6x: smaller segments round up to more blocks, and git held ten packs +it had not yet merged. + +Only the text of a turn is indexed. The JSON around it is not, and an image goes +in as its shape, never its payload, so the more of that your transcripts carry, +the smaller `index/` is next to them. One real store was reported at 1.75x, +684 MB for 392 MB captured. Yours depends on how well your transcripts +compress, how much of them is text, and when git last packed. + +To capture less in the first place, [main sessions only](#main-sessions-only-claude-code) +keeps subagent and workflow transcripts out of the store, at the cost that +those turns are never captured and the dashboard's spend undercounts. diff --git a/tests/test_docs.py b/tests/test_docs.py index 10cabb4..3e1d24a 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -216,6 +216,45 @@ def test_the_documented_manifest_fields_are_the_fields(): assert field in design, f"DESIGN.md does not mention the manifest field {field!r}" +def test_the_disk_section_calls_rebuildable_only_what_is_gitignored(): + """ARCHITECTURE.md said the store "holds each byte the agent wrote exactly + once", and DESIGN.md "zero duplication". That is `raw/`. `.git` keeps a + second, compressed copy and `index/` the text a third time, so one store + came to 1.75x what it had captured, and no document said why. + + The ratios are not pinned: they move with the filesystem's block size, the + git version, when `gc --auto` last packed, and the corpus. What is pinned is + the advice. Deleting `index/` to get space back is safe only while git + ignores `index/` and tracks `raw/` and `sessions/`, so that is checked + against the store's own `.gitignore`. The anchors pointing at the section + are checked against its heading, so renaming it breaks a test and not just a link. + Headings are read with fenced blocks cut out: a `# comment` in a shell or + TOML recipe is not a heading, and counting it let a link to no heading pass. + """ + from gitmemory import gitrepo + + watching = re.sub(r"^```.*?^```", "", _read("docs/watching.md"), flags=re.S | re.M) + section = re.search(r"^## How much disk it takes\n(.*?)(?=^## |\Z)", watching, re.S | re.M) + assert section, "docs/watching.md has no 'How much disk it takes' section" + body = section.group(1) + assert "`index/`" in body and "`gitmemory index`" in body, "it no longer says what rebuilds it" + + ignored = gitrepo.GITIGNORE.splitlines() + assert "/index/" in ignored, "the section says to delete a directory git may be tracking" + for durable in ("/raw/", "/sessions/"): + assert durable not in ignored, f"{durable} is ignored, so git no longer holds it" + + slugs = { + re.sub(r"[^\w\- ]", "", h.strip().lower()).replace(" ", "-") + for h in re.findall(r"^#+ (.+)$", watching, re.M) + } + for rel in ("README.md", "docs/ARCHITECTURE.md", "docs/DESIGN.md"): + anchors = re.findall(r"watching\.md#([\w-]+)", _read(rel)) + assert "how-much-disk-it-takes" in anchors, f"{rel} does not point at the section" + for anchor in anchors: + assert anchor in slugs, f"{rel} links watching.md#{anchor}, which no heading makes" + + def test_the_documented_shim_length_is_the_shim_length(): """ "~20 lines POSIX sh" outlived the sentence by a factor of five. From 7a10f228b27df50d3938c490a19c0ee3e3a60f23 Mon Sep 17 00:00:00 2001 From: doronp <3586743+doronp@users.noreply.github.com> Date: Sun, 18 Oct 2026 15:46:04 +0300 Subject: [PATCH 11/13] chore: release 0.2.0 Claude Code keeps an installed plugin's copy until the plugin's version changes, so the recall skill, gitmemory show, --redact and the other changes under skills/ and hooks/ since 0.1.0 reach nobody who installed the plugin until the version moves. This bumps it in the five places CONTRIBUTING names, in one commit: pyproject.toml, uv.lock, __version__, .claude-plugin/plugin.json and the release the recall skill says it is written for. The tests pin the first four equal. CHANGELOG's [Unreleased] becomes [0.2.0] - 2026-10-18 with a compare link from v0.1.0, and [Unreleased] now starts at v0.2.0. The CHANGELOG and hook/README.md text that said the skill would arrive with the next release now says, in the past tense, why 0.1.0 installs lacked it, and that 0.2.0's bump brings it on update. DESIGN.md 2.8's "As built" said "the CLI only"; it now names the plugin's recall skill too, as a search the agent runs rather than an injection, so the no-injection statement stays true. The plugin.json description is cut from 620 to 449 characters, keeping the install command at --python 3.13, which the install-line test requires. Not tagged and not pushed: tag v0.2.0 once this is on main, since hook/README.md tells users to pin both installs to the tag. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QPBLmJD2q1PVxdkWx8C16U Signed-off-by: doronp <3586743+doronp@users.noreply.github.com> --- .claude-plugin/plugin.json | 4 ++-- CHANGELOG.md | 12 ++++++++---- docs/DESIGN.md | 12 ++++++++---- hook/README.md | 11 ++++++----- pyproject.toml | 2 +- skills/recall/SKILL.md | 2 +- src/gitmemory/__init__.py | 2 +- uv.lock | 2 +- 8 files changed, 28 insertions(+), 19 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 2b2cab1..249e3d0 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "gitmemory", - "version": "0.1.0", - "description": "Registers gitmemory's hook shim on PreCompact and SessionEnd, so a session is captured at the compaction boundary instead of at the watcher's next sweep, and adds a recall skill that has the agent search the captured sessions with gitmemory recall and quote what was said, with the address of each line. The hook is only a doorbell, and the skill writes nothing but the search index: nothing is captured unless the gitmemory watcher is installed (uv tool install --python 3.13 git+https://github.com/doronp/gitmemory), configured (gitmemory init claude-code) and running, and without the hook gitmemory is still correct.", + "version": "0.2.0", + "description": "Registers gitmemory's hook shim on PreCompact and SessionEnd, so a session is captured at the compaction boundary, and adds a recall skill that has the agent search past sessions with gitmemory recall and quote each line with its address. Neither captures anything: the gitmemory watcher must be installed (uv tool install --python 3.13 git+https://github.com/doronp/gitmemory), configured (gitmemory init claude-code) and running (gitmemory watch).", "author": { "name": "doronp" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index cfde789..1a33575 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ tell you. ## [Unreleased] +## [0.2.0] - 2026-10-18 + ### Added - **`gitmemory init claude-code`** (or `pi`) adds that agent's standard @@ -40,9 +42,10 @@ tell you. index is missing or stale, and writes nothing else. See [docs/USAGE.md](docs/USAGE.md#put-it-back-in-front-of-the-agent). What it reads goes to the model provider, past no redaction gate; `recall --redact` - (below) masks credential shapes in it, and blocks nothing. The plugin's - version is still 0.1.0, so a copy installed from 0.1.0 does not get the skill - from `claude plugin update`; reinstall it, or wait for the next release + (below) masks credential shapes in it, and blocks nothing. The skill was + added after 0.1.0 without a version bump, so a copy installed from 0.1.0 + lacks it; this release moves the plugin's version to 0.2.0, so + `claude plugin update` brings it ([hook/README.md](hook/README.md#as-a-claude-code-plugin)). - **`gitmemory --version`** prints the installed release. The recall skill runs it first and tells you when the CLI is older than the plugin; when the @@ -156,5 +159,6 @@ The first public release. What it contains: - Two review rounds before publication: E7 (68 findings) and E7b (22 findings). See [SECURITY.md](SECURITY.md). -[Unreleased]: https://github.com/doronp/gitmemory/compare/v0.1.0...HEAD +[Unreleased]: https://github.com/doronp/gitmemory/compare/v0.2.0...HEAD +[0.2.0]: https://github.com/doronp/gitmemory/compare/v0.1.0...v0.2.0 [0.1.0]: https://github.com/doronp/gitmemory/releases/tag/v0.1.0 diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 15be97e..c568254 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -640,10 +640,14 @@ answers the token-burn objection.)* **UNVERIFIED, must be tested before claiming coexistence:** whether multiple `UserPromptSubmit` hooks' `additionalContext` outputs concatenate or last-wins. -**As built:** the CLI only. There is no MCP server and no `UserPromptSubmit` hook in `src/`, so -nothing is injected; injection cost is carried as `NOT BUILT` on the dashboard (§2.9). `recall` -prints each hit's address, and `show` resolves one to the whole turn from the store, not the -index, so a stale or tampered index cannot change what the address is shown to say. +**As built:** the CLI, and the Claude Code plugin's recall skill (`skills/recall/SKILL.md`, +`/gitmemory:recall`), which has the agent run `recall` and `show` itself and quote what they +print with each line's address. The skill is a search the agent runs when the user refers to an +earlier decision or runs the skill, not an injection: there is no MCP server and no +`UserPromptSubmit` hook in `src/`, so nothing is injected; injection cost is carried as +`NOT BUILT` on the dashboard (§2.9). `recall` prints each hit's address, and `show` resolves one +to the whole turn from the store, not the index, so a stale or tampered index cannot change what +the address is shown to say. ### 2.9 Dashboard diff --git a/hook/README.md b/hook/README.md index 3d476d8..cc64d3b 100644 --- a/hook/README.md +++ b/hook/README.md @@ -118,11 +118,12 @@ does not read `raw/`. It masks; it is not a gate, and [USAGE.md](../docs/USAGE.md#masking-credentials-in-what-it-prints) lists what it does not cover. -An install made before the skill was added does not have it. Claude Code keeps -a plugin's cached copy until its version changes, and the skill was added -without a version bump, so `claude plugin update gitmemory@gitmemory` reports -that it is already at 0.1.0. The next release, which bumps the version, brings -the skill; until then, `claude plugin uninstall gitmemory@gitmemory` and then +An install made from 0.1.0 does not have the skill. Claude Code keeps a +plugin's cached copy until its version changes, and the skill was added after +0.1.0 without a version bump, so until 0.2.0 an update reported that the +plugin was already at 0.1.0. Release 0.2.0 bumps the version, so +`claude plugin update gitmemory@gitmemory` brings the skill; if it still says +0.1.0, `claude plugin uninstall gitmemory@gitmemory` and then `claude plugin install gitmemory@gitmemory` fetch the current copy. The plugin and the CLI are installed separately, and nothing makes them one diff --git a/pyproject.toml b/pyproject.toml index 41f13b6..612462d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "gitmemory" -version = "0.1.0" +version = "0.2.0" description = "Git-versioned, contiguity-checked memory for agents that keep a local, append-only, line-delimited transcript" readme = "README.md" requires-python = ">=3.13" diff --git a/skills/recall/SKILL.md b/skills/recall/SKILL.md index 1cb8782..eaa3979 100644 --- a/skills/recall/SKILL.md +++ b/skills/recall/SKILL.md @@ -31,7 +31,7 @@ in its place from memory. ## The CLI's version The plugin and the CLI are installed separately, so they can come from -different releases. This skill is written for gitmemory 0.1.0. Once per +different releases. This skill is written for gitmemory 0.2.0. Once per session, before the first search, run: ```sh diff --git a/src/gitmemory/__init__.py b/src/gitmemory/__init__.py index fe8ef51..37e1562 100644 --- a/src/gitmemory/__init__.py +++ b/src/gitmemory/__init__.py @@ -3,4 +3,4 @@ """gitmemory — git-versioned, contiguity-checked memory for agents that keep a local, append-only, line-delimited transcript.""" -__version__ = "0.1.0" +__version__ = "0.2.0" diff --git a/uv.lock b/uv.lock index fd4cdb2..0df6143 100644 --- a/uv.lock +++ b/uv.lock @@ -433,7 +433,7 @@ wheels = [ [[package]] name = "gitmemory" -version = "0.1.0" +version = "0.2.0" source = { editable = "." } [package.optional-dependencies] From b7e352175f26150e4dd67b679572aa68dc421374 Mon Sep 17 00:00:00 2001 From: doronp <3586743+doronp@users.noreply.github.com> Date: Sun, 18 Oct 2026 16:55:52 +0300 Subject: [PATCH 12/13] fix: answer the Kimi K3 cross-review of the install-feedback round Kimi K3 reviewed each item commit of the round on its own, against main. Every finding was then re-checked against the integrated branch, with a live probe where one could run. Of 53, 21 led to a change, 6 are true and declined, and 26 do not hold; several of those came from seeing one commit without the rest. docs/reviews/install-feedback-round.md records each one with its evidence. The one behavioural fix: two `gitmemory init`s at once read the same config.toml and the later rename dropped the other's table, while both said "added" (46 of 60 concurrent pairs, measured). init now re-reads the file under a lock in .locks/ just before the rename and refuses, exit 1, when it moved; 0 losses in 60 since. Both reads go through one helper, because the re-read alone also refused an unreadable file, and the existing control for that guard had stopped biting. One new negative control (658), and watching.md and the CHANGELOG say what is and is not covered. User-visible wording: the stale-spool line now says "N hook record(s) uncollected, the oldest for ", since the age was only the oldest record's (code, skill, USAGE, watching.md and tests together), and the --redact help says empty is off as well as 0. Tests that now hold what they claimed: the install-floor scan reads a command split across code spans; REPRODUCE.md's counts are held to the README's (a new test; with init's, the suite is 1,131); the disk section's delete advice is checked against the store's .gitignore, and CHANGELOG's anchors are checked too; the init temp test asks git about the writer's real temp; the PEM-header linearity test counts searches instead of timing them; a key inside a PEM body is pinned; the _ago 2-hour and 2-day edges are pinned; a daemon test no longer depends on /tmp/x being absent. Docs and skill: show's echo rule matches the header's shape; the skill says to search again with -k 50 when subagent lines crowd the results, and not to list sessions/ for a masked address; USAGE says the service starts the watcher at every login and names linger; "about twice" the capture is bounded for transcripts that compress poorly; the stale suite sizes in mutate_index's docstrings are dated rather than re-pinned. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QPBLmJD2q1PVxdkWx8C16U Signed-off-by: doronp <3586743+doronp@users.noreply.github.com> --- CHANGELOG.md | 5 +- README.md | 16 +-- docs/DESIGN.md | 5 +- docs/REPRODUCE.md | 20 +-- docs/USAGE.md | 18 +-- docs/reviews/install-feedback-round.md | 163 +++++++++++++++++++++++++ docs/watching.md | 12 +- skills/recall/SKILL.md | 21 ++-- src/gitmemory/__main__.py | 33 +++-- tests/mutate_index.py | 47 ++++--- tests/test_daemon.py | 8 +- tests/test_docs.py | 70 +++++++++-- tests/test_index.py | 65 +++++++--- tests/test_init.py | 42 +++++-- tests/test_show.py | 3 + 15 files changed, 425 insertions(+), 103 deletions(-) create mode 100644 docs/reviews/install-feedback-round.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a33575..f9a2cf4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,9 @@ tell you. named the agent. It appends without changing a byte already there, because the file also holds the `[remote.*]` push opt-ins. It does nothing if the table is already there, and it refuses, writing nothing and exiting 1, a root - the watcher would skip or another agent already watches, and a `config.toml` - that is a symlink or a hard link. See + the watcher would skip or another agent already watches, a `config.toml` + that is a symlink or a hard link, and one another `init` changed while it + ran. See [docs/watching.md](docs/watching.md#configure-what-to-watch). - **`gitmemory service install`** writes a systemd user unit (Linux) or a LaunchAgent (macOS) that keeps `gitmemory watch` running, and prints the diff --git a/README.md b/README.md index e40ffe8..31b9f84 100644 --- a/README.md +++ b/README.md @@ -111,10 +111,10 @@ agent search the store and quote what it finds; the watcher still does the capturing. For watcher flags and troubleshooting, see [docs/watching.md](docs/watching.md). -Budget disk for about twice what the store captures: git keeps a compressed -second copy, and the search index holds the text again. `gitmemory index` -rebuilds the index if you delete it; rebuilding over an existing one briefly -needs room for both. +Budget disk for about twice what the store captures, and more if your +transcripts compress poorly: git keeps a compressed second copy, and the +search index holds the text again. `gitmemory index` rebuilds the index if +you delete it; rebuilding over an existing one briefly needs room for both. [docs/watching.md](docs/watching.md#how-much-disk-it-takes) has the measurements. @@ -217,13 +217,13 @@ real text, and that is a measurement rather than a suspicion. | What | How it was measured | Result | |---|---|---| | Hook cost in the agent's critical path | Timed against spawning `true` the same way, three runs of 400 | p50 **7.4 – 7.5 ms**, p99 **10.2 – 11.5 ms** | -| The suite | On a fresh checkout, no downloads | **1,129 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | -| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **657** negative controls | +| The suite | On a fresh checkout, no downloads | **1,131 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | +| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **658** negative controls | The two suite counts do not add up, and should not. Switching the corpora on -collects 1454, not 1457. Three conformance cases fill parametrisations that +collects 1456, not 1459. Three conformance cases fill parametrisations that collect as one empty placeholder each while the corpora are absent, so they -replace three of the 1129 rather than joining them. Every figure on this board +replace three of the 1131 rather than joining them. Every figure on this board is pinned by a test, which is how it stays true. **What is not measured**, and is shown as a row on the dashboard rather than diff --git a/docs/DESIGN.md b/docs/DESIGN.md index c568254..cc1ab4e 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -203,8 +203,9 @@ Each capture commits **only the bytes appended since the last recorded offset**. 1. **Zero duplication.** Stored bytes == bytes the agent ever wrote, not a multiple. **[2026-10-18 — true of `raw/`, not of the store on disk. `.git/` keeps a zlib-compressed copy - of the same bytes and `index/` holds their text again, so budget about twice what you capture; - [watching.md](watching.md#how-much-disk-it-takes) has the measurements.]** + of the same bytes and `index/` holds their text again, so budget about twice what you capture, + more if it compresses poorly; [watching.md](watching.md#how-much-disk-it-takes) has the + measurements.]** 2. **Plain text, never pre-compressed.** Let git's zlib + delta work on text it understands. Pre-compressing defeats git's native packing. 3. **Reconstruction is `cat` in filename order**, checked against a recorded whole-file sha256. diff --git a/docs/REPRODUCE.md b/docs/REPRODUCE.md index 4c422dd..d647814 100644 --- a/docs/REPRODUCE.md +++ b/docs/REPRODUCE.md @@ -484,17 +484,18 @@ uv run pytest -q --collect-only -p no:cacheprovider | tail -1 uv run pytest -q --collect-only -p no:cacheprovider | grep -c '\.jsonl\]$' ``` -**Expected:** `1129/1130 tests collected (1 deselected)`, then -`1454/1455 tests collected (1 deselected)`, then `328`. The deselected test is +**Expected:** `1131/1132 tests collected (1 deselected)`, then +`1456/1457 tests collected (1 deselected)`, then `328`. The deselected test is the LongMemEval corpus test, which is opt-in: `uv run pytest -q -m corpus` after `bench/fetch_longmemeval.sh`. The README's -paragraph on why 1,129 + 328 is 1,454 and not 1,457 is the arithmetic of these +paragraph on why 1,131 + 328 is 1,456 and not 1,459 is the arithmetic of these three numbers. All three were run while writing this guide. -`tests/test_docs.py` holds the README to the first and third numbers, so a -drifted count fails the suite. To run the suite rather than count it: -`uv run pytest -q`, which CONTRIBUTING.md gives as about a minute and a half, -no network. +`tests/test_docs.py` holds the README to all three numbers, the second and +third only where the corpora are cloned, and this guide, §5c's figure +included, to the README, so a drifted count fails the suite. To run the suite +rather than count it: `uv run pytest -q`, which CONTRIBUTING.md gives as about +a minute and a half, no network. ### 5c. Negative controls @@ -505,7 +506,7 @@ test fail. The count is the number of rows in `tests/mutate_index.py`. uv run python -c "import sys; sys.path.insert(0, 'tests'); from mutate_index import MUTANTS; print(len(MUTANTS))" ``` -**Expected:** `657`. Checked while writing this guide. +**Expected:** `658`. Checked while writing this guide. To run the controls rather than count them: @@ -518,7 +519,8 @@ uv run python tests/mutate_index.py "verify" # only mutants whose name contain and `bench/` in place and restores each one after its row, so run it on a clean checkout that nothing else is editing. Its docstring estimates a full pass at one to two and a half hours, from three rows timed at 3.7 s, 17.1 s and -17.2 s. It exits 0 only if every mutant is caught by its intended test. +17.2 s on a smaller suite. It exits 0 only if every mutant is caught by its +intended test. ## What cannot be reproduced exactly, and why diff --git a/docs/USAGE.md b/docs/USAGE.md index 15d320f..6e12d4f 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -42,8 +42,10 @@ repository, and lets you search it. So: ``` How it stays running is your choice. `gitmemory service install` writes a - systemd user unit (Linux) or a LaunchAgent (macOS) that keeps it running - across logins, and prints the line that starts it; it starts nothing itself + systemd user unit (Linux) or a LaunchAgent (macOS) that starts it at every + login, and prints the line that starts it now; it starts nothing itself. On + a Linux machine you reach over SSH, the `loginctl enable-linger` line it + prints keeps it running after you log out ([watching.md](watching.md#keep-it-running)). A terminal you leave open works too. [watching.md](watching.md#run-it) covers the flags and how to tell a misconfigured watcher from an idle one. @@ -92,7 +94,7 @@ newest first (see [When memories disagree](#when-memories-disagree)): If the [hook](../hook/README.md) has rung and no watcher has collected the record for an hour, `recall` says so on stderr, before the results it still prints: -``1 hook record(s) uncollected for 3 hours: is `gitmemory watch` running for ? These results may be missing everything said since.`` +``1 hook record(s) uncollected, the oldest for 3 hours: is `gitmemory watch` running for ? These results may be missing everything said since.`` A running watcher drains the spool on every pass, so none has run on that store since: start `gitmemory watch` with the hook's `$GITMEMORY_HOME`, and `gitmemory service install` to keep it running @@ -148,11 +150,11 @@ $ gitmemory recall --redact "staging key" ``` A masked span keeps its first four bytes and its length; in an address the -marker drops its space, so each line keeps its columns and each header its -fields. A private key is masked from its BEGIN line to its END line, or to the -end of the text when there is none. `show --redact` masks each block of the -turn whole before it splits it into lines, so a key's body goes with its -header, and it masks a tool's name too. It keeps the turn's line breaks, where +marker drops its space, so the address stays one field of a `recall` line or +a `show` header. A private key is masked from its BEGIN line to its END +line, or to the end of the text when there is none. `show --redact` masks +each block of the turn whole before it splits it into lines, so a key's body +goes with its header, and it masks a tool's name too. It keeps the turn's line breaks, where `recall` flattens a line before it masks, so it also finds a header that a line break split, as the flattened line would have shown it. A masked address is not one `show` can look up. diff --git a/docs/reviews/install-feedback-round.md b/docs/reviews/install-feedback-round.md new file mode 100644 index 0000000..6065143 --- /dev/null +++ b/docs/reviews/install-feedback-round.md @@ -0,0 +1,163 @@ +# Install-feedback round: the Kimi K3 cross-review + +The round answered ten items of feedback from a first install, on one branch +off `main`, one signed-off commit per item and then the 0.2.0 release commit. +Two channels reviewed it: + +- **An independent Opus reviewer for each item commit**, with no part in + writing it. Its findings were fixed in that item's commit before the item was + integrated, so they are not repeated here. +- **Kimi K3, as a pair programmer, on each item commit.** One pack per item, + `git diff -U8 main...`, and one review per pack. The redaction item was + split into a code pack and a docs pack to fit. + +**Kimi saw each commit alone**, against `main` and before integration, and +some of its findings come from exactly that. It called the redaction docs +unbacked by code, and the code unbacked by docs, because each half of the +split pack held what the other said was missing. It read a fourteen-test jump +in `docs/REPRODUCE.md` as unexplained, when `main`'s copy was already ten +behind the README. It could not see `main()`'s exception handler, the +`## Echoes` heading or `tests/conftest.py`, because none of them was in the +hunk. Each is recorded below with what it turned out to be, not dropped. + +Every finding was re-checked against the integrated branch at `7a10f22` before +anything changed. The lead refuted the first batch from a code read or a live +probe. A skeptic then took each of the rest and tried to break it, with a probe +wherever one could run. One finding, install-3, was on neither list, and was +checked while the fixes were applied. The fixes landed in one commit on top, +the one that adds this file. + +## The ten items + +| Item | What | Commit | +|---|---|---| +| 1 | Every install command names `--python 3.13` | `59865e1` | +| 7c | `gitmemory --version`; the recall skill recognises an older CLI | `56327cf` | +| 5 | latest-user stops taking a subagent's prompt for the user's words | `656136c` | +| 2 | `gitmemory init ` writes the standard `[[watch]]` table | `beffcf2` | +| 3a | `gitmemory service install` writes a user unit or LaunchAgent | `b39449a` | +| 3b | `recall` says when hook records have waited an hour for a watcher | `5c6a874` | +| 4 | `gitmemory show` prints the turn at a recall address | `dba5db7` | +| 6 | `recall --redact` and `show --redact` | `b9a5f57` | +| 7a | Captured sizes for people, with the exact count kept | `3bb0aff` | +| 7b | How much disk the store takes, and why | `0434189` | +| | Release 0.2.0 | `7a10f22` | + +## Status + +**53 findings: 21 fixed, 6 declined, 26 refuted.** *Fixed* means something +landed: code, test or text. *Declined* means the claim is true and not worth +the change, with the reason written down. *Refuted* means the claim does not +hold on the integrated branch. + +| Severity | Raised | Fixed | Declined | Refuted | +|---|---|---|---|---| +| High | 5 | — | — | 5 | +| Medium | 13 | 7 | — | 6 | +| Low | 35 | 14 | 6 | 15 | +| **Total** | **53** | **21** | **6** | **26** | + +All five Highs are artifacts of the commit seen alone: four from the split +redaction pack, and one (the detector count, redact-docs-4) a correction of a +number `main`'s DESIGN.md had wrong. Fifteen of the 32 that changed nothing +rest on something outside the hunk Kimi was given. + +One change of behaviour came out of the round: two `gitmemory init`s run at +once could lose a table silently (init-1). It carries a test that failed +first and a new negative control, which brings the index to **658**. The rest +are one reworded message (stale-1), one help string (redact-code-3), tests, +wording and pinned numbers. + +## Findings + +| ID | Sev | Claim | Disposition | Evidence or reason | +|---|---|---|---|---| +| install-1 | med | The floor test skips `` `uv tool install` git+… ``, a command split across spans | fixed | Probed: such a line in hook/README.md passed. A span holding the tool alone now reads on when the next word is a source or a flag. Kimi's "rest of the line" rule would have failed three prose lines in hook/README.md | +| install-2 | low | A `#` ends a command, so a `#egg=` fragment hides a later `--python` | declined | Fails loud, with a message naming the line. Documented in the test. No scanned command has a fragment | +| install-3 | low | The `derive` install hint in derive.py has no mutation row | declined | The floor test scans every tracked file: with `--python 3.13` removed from the hint, it fails on `derive.py:142`. Rows are for fixes, and item 1 was docs | +| install-4 | low | A trailing `\` joins prose lines into one command | declined | Documented in both docstrings. Every Markdown line ending in `\` is a fenced shell continuation, and a join can only add a failure, never hide one | +| version-1 | med | The skill's "step 1" does not exist | refuted | The numbered setup list is in SKILL.md; step 1 is the install line (lead) | +| version-2 | med | Rerunning `uv tool install` does not move an install forward | refuted | Live probe, uv 0.11.3: rerunning the unpinned line replaced the install's commit with the remote's tip; `uv tool upgrade` said "Nothing to upgrade" (lead) | +| version-3 | low | A fix with no mutation row | refuted | A `feat:` adding one flag; rows are required for fixes. Deleting the flag fails two skill tests, measured | +| subagents-1 | med | A doc test writes into the real `~/.claude/projects` | refuted | conftest's autouse `_no_real_home` points HOME at `tmp_path` for every test (lead) | +| subagents-2 | low | REPRODUCE's counts jump by 14 for 4 tests | fixed | The jump was `main` already ten behind the README. The real gap is that nothing pinned the guide, so a test now holds it to the README | +| subagents-3 | low | The skill never says what to do when `subagent:` lines crowd the results | fixed | Probed: 20 subagents hid the user's rule at `-k 10` and `-k 30`, and it came first at `-k 50`. The skill now says to search again with `-k 50`. `main_only` is a user-written recipe, so the skill does not point at it | +| subagents-4 | low | A docstring claims "byte-for-byte" and the test checks one field | fixed | Probed: a patched address and text still passed. The docstring now says what is asserted | +| init-1 | med | Read, then rename: an edit between them is lost | fixed | The racer that hits it is a second `init`: 46 of 60 concurrent pairs lost a table, both saying "added". `init` now re-reads under a lock before the rename and refuses, exit 1, if the file moved. 0 losses in 60 since. An editor takes no lock and is not covered, as watching.md now says | +| init-2 | low | The temp-name test checks a typed name, not the writer's | fixed | Probed: it passed with the writer's temp renamed. The test now asks `git check-ignore` about the temp the writer really made, while it exists | +| init-3 | low | The invalid-TOML refusal prints the raw exception | refuted | `push` and the watcher print the same error the same way. tomllib quotes key names and single characters, never a value, and the credential is a value | +| init-4 | low | The refusal could print an empty reason | declined | Every skip a written table can reach logs with the prefix the filter wants. A fallback would only guard a future watcher bug | +| service-1 | med | Other control characters make an invalid plist and unpasteable commands | refuted | plistlib refuses them before anything is written (exit 2). The legal ones pass `plutil -lint`. The printed commands hold no store path | +| service-2 | low | USAGE says the unit keeps the watcher running across logins | fixed | Without linger, systemd stops user units at the last logout. Reworded to "at every login", with the `loginctl enable-linger` line for SSH-only machines | +| service-3 | low | The removal line lacks `daemon-reload` | refuted | `disable` reloads, and a stopped, disabled unit is garbage-collected, so `status` reports it gone. From systemd's documentation; not probed live, as there was no systemd to hand | +| service-4 | low | `main()` may not map ValueError and RuntimeError to exit 2 | refuted | It does (lead) | +| stale-1 | med | "N records uncollected for " gives the oldest's age to all N | fixed | Probed: a record from seconds earlier was reported 3 days old. Now `N hook record(s) uncollected, the oldest for `, in the code, the skill, USAGE and watching.md | +| stale-2 | low | REPRODUCE's counts jump by 13 for 3 tests | fixed | As subagents-2. One fix serves both; see below | +| stale-3 | low | The skill test assumes `resolve_home` is the identity on a relative path | refuted | Every sibling message also goes through `resolve_home`, and pytest's `tmp_path` is already resolved | +| stale-4 | low | The `_ago` boundaries and the `>= STALE_RECORD` edge are unpinned | fixed | The 2-hour and 2-day edges are now pinned 30 s either side. Probed: four mutants of them that had survived are caught. The `>=` edge is left: it differs only at exactly 3600.0 s, and pinning it needs an injected clock | +| show-1 | med | Store-read errors escape `show`'s exit contract | refuted | `main()` turns OSError, ValueError, RuntimeError and sqlite3.Error into a masked `error:`, exit 2 (lead) | +| show-2 | low | mutate_index's docstrings keep stale counts | fixed | "the other 567", "955-test", "1280-test". The number is gone, and the timings are dated as taken when the suites were 955 and 1280 tests, not re-pinned to today's | +| show-3 | low | Role and kind are printed uncollapsed | refuted | Both adapters write only literal roles and kinds (lead) | +| show-4 | low | The skill's echo rule also skips a real turn that quotes an address | fixed | Probed: read literally, it skipped the user's newer reversal. Now the header's shape, marker then address then date, and a test pins the wording | +| show-5 | low | The `## Echoes` heading may not exist | refuted | It is on `main` (lead) | +| show-6 | low | `g5` and `@0007` are accepted | declined | Probed: same output as the canonical address, and the header prints the canonical one. The address never becomes a path | +| show-7 | low | The success path prints a credential-shaped session id unmasked | refuted | It does by design: masking is opt-in, and `--redact` masks it. The case-fold residue is already documented at the call site and in USAGE | +| redact-code-1 | high | No SKILL.md change ships | refuted | It was in the docs half of the split pack (lead) | +| redact-code-2 | med | The linearity test asserts on wall clock | fixed | The page of headers is now counted: one END search, against 2000 under the mutant. The never-closing END line is one `re` search, which cannot be counted, so it stays timed, as the suite's other regex-cost tests are | +| redact-code-3 | low | The `--redact` help says "set and not 0", but empty is off too | fixed | Now "set to anything but empty or 0" | +| redact-code-4 | low | `safe_path` now widens a PEM header to the end of a diagnostic | refuted | The owner's decision, pinned by a test and a mutation row. It only ever masks more | +| redact-code-5 | low | The flag is registered only for `recall` | refuted | Integrated, `show` registers it too, and those are the two commands that print transcript text | +| redact-code-6 | low | The docs for the env var are absent | refuted | They were in the docs half of the split pack (lead) | +| redact-docs-1 | high | No `--redact` in the CLI | refuted | In the code half (lead) | +| redact-docs-2 | high | No `mask()`, no `safe_path` delegation | refuted | In the code half (lead) | +| redact-docs-3 | high | Counts bumped with no tests or rows | refuted | The tests and rows were in the code half (lead) | +| redact-docs-4 | high | Detector counts edited to twelve and four with no code change | refuted | redact.py has 12 HIGH and 4 SUSPECT on `main` and here. `main`'s DESIGN.md said nine and three, which was stale | +| redact-docs-5 | med | The skill's env check may differ from the CLI's | refuted | Same predicate: unset, empty or `0` is off. Probed with `00`, `false` and padded values: both read them as on | +| redact-docs-6 | med | A masked address can be opened neither with `show` nor as a manifest | fixed | The first half was already fixed in integration. The skill now says the manifest cannot be found from one either, and not to list `/sessions/`, whose names are the ids unmasked | +| redact-docs-7 | med | The PEM edges are untested | fixed | Three of the four were already pinned. The fourth, a second key inside the body, had no test, and `_merged`'s `max(` held it by accident: dropping it leaked the body's tail. It now has an assertion | +| redact-docs-8 | low | "Measured examples" should be pinned | refuted | Re-run: all ten examples still print unmasked. No rule says a measurement must be pinned, and drift could only understate coverage | +| redact-docs-9 | low | "Keeps its columns" overclaims | fixed | Alignment is not kept; the field count is. Now "stays one field of a `recall` line or a `show` header" | +| sizes-1 | low | mutate_index's docstring keeps "1280" and "955" | fixed | As show-2 | +| sizes-2 | low | `#set-it-up-once` may not exist | refuted | USAGE.md has `## Set it up once` (lead) | +| sizes-3 | low | The skill or hook README may show old `+NB` output | refuted | Neither has an output sample. The only `+NB` left is the commit subject, kept by decision | +| disk-1 | med | The test never reads which directory the section says to delete | fixed | Probed: "delete `sessions/`" passed. Every directory a sentence says to delete must now be one the store's `.gitignore` ignores | +| disk-2 | low | The CHANGELOG's link is outside the anchor loop | fixed | Added. A broken anchor there now fails | +| disk-3 | low | "About twice" sits beside a table of 2.3x and 3.4x | fixed | Bounded in watching.md, the README and DESIGN.md: "more if your transcripts compress poorly" | +| disk-4 | low | Three behavioural claims are unverified | refuted | Each is true and pinned: recall and dashboard refuse without an index, `show` does not need one, a rebuild renames into place, `gc --auto` runs | +| disk-5 | low | Only column-0 backtick fences are stripped | declined | No tracked Markdown has a `~~~` fence. An indented fence indents its `#` lines too, and the heading pattern reads column 0 only | + +## Decided while applying + +**One pin for two findings.** subagents-2 and stale-2 each proposed holding +`docs/REPRODUCE.md` to the counts: the first by adding assertions to the four +count tests, against the collected numbers, and the second with one new test +holding the guide to the README. The second is the one that landed. The README +is already held to the suite, so it needs no second collection, and it runs on +a machine without the corpora too. The first would have held two of the +guide's four numbers only where the corpora are cloned. The cost is one test. + +**The re-read made an old control blind.** init-1's second read, under the +lock, raises on a file it cannot read by itself. So the row "init takes a +config.toml it cannot read for a missing one", which mutates the first read's +`except FileNotFoundError` to `except OSError`, survived: the second read +still refused. Both reads now go through one helper, the row's anchor moved +into it, and it is caught again. + +## Two findings from outside Kimi's table + +| ID | Claim | Disposition | Evidence or reason | +|---|---|---|---| +| opus-item1 | The README's `uv python install 3.13` advice is wrong where uv may not fetch Python | refuted | Item 1 had already limited it to `python-downloads = "manual"`, and says to put a 3.13 on `PATH` under `"never"`. Probed both, offline, uv 0.11.3 | +| daemon-tmp-x | A test on `main` uses `/tmp/x` as a watch root | fixed | Probed: a stray regular file there empties the result and fails the test. The root is now under `tmp_path`, never created, which also pins that a root that does not exist yet is kept | + +## Verified + +At the state this file describes: + +| | | +|---|---| +| Full suite, corpora off | **1114 passed, 17 skipped**, 1 deselected: 1131, the README's number | +| Full suite, corpora on | **1455 passed, 1 skipped**, 1 deselected | +| Collected | 1131 off and 1456 on, 328 conformance cases: the README and §5b of REPRODUCE.md | +| `ruff check` | clean | +| Mutation rows | **658**. The 23 rows that name a test this commit changed or re-pinned, the new one included: **23 of 23 caught**. Three of them, the README-count rows, bite only with the corpora present, which is what they check; they were run with them | diff --git a/docs/watching.md b/docs/watching.md index 6360165..01bea0d 100644 --- a/docs/watching.md +++ b/docs/watching.md @@ -27,7 +27,11 @@ would skip (listed below; `init` prints the watcher's own message), a file that is not valid TOML or already has a `watch` of another shape, and a `config.toml` that is a symlink or has another hard link, because the rename would leave the other name holding the old file. A `config.toml` it cannot read -is an error, exit 2. A running watcher picks the new table up on its next pass. +is an error, exit 2. Two `init`s run at the same moment cannot drop each +other's table: the later one sees the file change after it read it, leaves it +as the other one wrote it, and exits 1, so run it again. An editor saving the +file in that same instant is not covered. A running watcher picks the new table +up on its next pass. By hand, or for any other agent or root, `$GITMEMORY_HOME/config.toml`: @@ -235,7 +239,7 @@ anywhere useful. The two failures worth watching for are different shapes: |---|---| | `no .../config.toml; watching nothing` | The file is not where the watcher is looking. Check `$GITMEMORY_HOME`, or run `gitmemory init ` to write it there. | | Watcher quiet, store empty, no complaint | A root that **does not exist** is kept, not refused — before an agent has run for the first time that is the normal state, and `run` re-reads the config every pass, so it starts working the moment the directory appears. The cost is that a typo'd root and a not-yet-used one look identical. Check the path by hand; `gitmemory init` says so when the root it writes does not exist yet. | -| `recall` says `N hook record(s) uncollected for ` | No watcher has run a pass on that store for that long: it is stopped, it watches a store other than the hook's `$GITMEMORY_HOME` (a service's store is the `--home` written into it), or it runs a pass less often than once an hour (`--poll` of 3600 or more, or a supervisor re-running `--once` that seldom). Start `gitmemory watch` with the hook's `$GITMEMORY_HOME` and keep it running with `gitmemory service install` ([above](#keep-it-running)), or run passes more often. A service that is installed but not started, or failing, shows here too; its check and log lines are above. A pass drains the spool even when no watch covers the record, so silence here means a pass ran, not that this agent is captured. | +| `recall` says `N hook record(s) uncollected, the oldest for ` | No watcher has run a pass on that store for that long: it is stopped, it watches a store other than the hook's `$GITMEMORY_HOME` (a service's store is the `--home` written into it), or it runs a pass less often than once an hour (`--poll` of 3600 or more, or a supervisor re-running `--once` that seldom). Start `gitmemory watch` with the hook's `$GITMEMORY_HOME` and keep it running with `gitmemory service install` ([above](#keep-it-running)), or run passes more often. A service that is installed but not started, or failing, shows here too; its check and log lines are above. A pass drains the spool even when no watch covers the record, so silence here means a pass ran, not that this agent is captured. | | `compact_boundaries: []` everywhere | Adapter missing or `--no-parse`. The bytes are intact; only the boundaries are absent. | | `source changed while parsing` | The transcript was rewritten between the parse and the copy — a compaction landing mid-pass. The bytes are captured; that one generation has no boundaries. Nothing to do. | | `was captured from , not ; pass a distinct --session-id` | Two transcripts collided on one session name, and the store refuses to mix them rather than duplicating both for ever. The advice in the message is for the CLI and there is no `--session-id` to pass here: rename or move either file — the name is derived from the path — and the next pass captures both. Until then the second one is **not being captured**, so this is not a warning to leave running. | @@ -246,7 +250,9 @@ silence. Listing the roots yourself is currently the way to tell them apart. ## How much disk it takes -Budget about twice what you capture. The store keeps the same bytes three ways: +Budget about twice what you capture, and more if your transcripts compress +poorly: the synthetic corpora below came to 2.3x and 3.4x. The store keeps the +same bytes three ways: - **`raw/`** holds each captured byte once, so it is the size of the capture. Each segment fills whole filesystem blocks, so a store grown in many small diff --git a/skills/recall/SKILL.md b/skills/recall/SKILL.md index eaa3979..1737978 100644 --- a/skills/recall/SKILL.md +++ b/skills/recall/SKILL.md @@ -94,9 +94,10 @@ recalls match the same words. Skip these hits; they are not sources: score, `N generation(s)` or `no matches`; - a `gitmemory --version` tool call and the one line it printed; - a `gitmemory show ...` tool call and its result, text that starts `> ` or - `- ` and an `//gNN@` address. It holds the whole - turn, dated later than the turn was: the source is the turn at the address - in its `> ` header, not the echo's own address; + `- ` immediately followed by an `//gNN@` address + and a date, which is `show`'s header. It holds the whole turn, dated later + than the turn was: the source is the turn at the address in its `> ` + header, not the echo's own address; - an earlier answer to the same question, and the lines it quoted. If they crowd out the rest, search again with `-k 30`. Quote the original turn. @@ -118,7 +119,7 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. - The watcher captures; it does not index. If a search misses something the user says was said recently, the index may be older than the capture: run `gitmemory index` and search once more. -- `` hook record(s) uncollected for : is `gitmemory watch` running for ? These results may be missing everything said since.`` +- `` hook record(s) uncollected, the oldest for : is `gitmemory watch` running for ? These results may be missing everything said since.`` on stderr, before the results, exit 0: the hook has rung and no watcher has run a pass on that store since, so nothing said after that is being captured there. Answer from the results, say they may be missing everything since, and @@ -152,9 +153,12 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. `gitmemory show --context 2
` for the two turns either side too, and `gitmemory show --redact
` if you passed `--redact` to `recall`. A masked address (one with `…[` in it) is not one `show` can - look up: say the line stops there and give the address as printed. A - header starts `>` for the turn at the address and `-` for a neighbour, and - only headers start in column 0; quote the text under one without its indent. + look up, and its manifest (below) cannot be found from it: say the line + stops there and give the address as printed. Do not list + `/sessions/` to find it: the names there are the session ids, + unmasked. A header starts `>` for the turn at the address and `-` for a + neighbour, and only headers start in column 0; quote the text under one + without its indent. `subagent:user` in a header is a brief from the agent that started the subagent, not the user's words. If the CLI is older than `show` (exit 2, `invalid choice: 'show'`), tell the user as [When it fails](#when-it-fails) @@ -173,7 +177,8 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. agent, another session or another project. - Say so, too, when a quote is a `subagent:` line, and never give one as what the user said. When it and a main-session line say the same thing, quote the - main-session line. + main-session line. If `subagent:` lines crowd out the rest, search again + with `-k 50`: the order is applied to the top `-k` matches only. - The store holds whatever was typed or pasted into a watched session, tool output included. Quote what the question needs and no more. diff --git a/src/gitmemory/__main__.py b/src/gitmemory/__main__.py index f982288..1621396 100644 --- a/src/gitmemory/__main__.py +++ b/src/gitmemory/__main__.py @@ -63,6 +63,17 @@ def _refuse(why: str) -> int: return 1 +def _config_bytes(path: str) -> bytes: + """`config.toml` as it is now, or nothing when there is no file. One it + cannot read is not a missing one: taking it for absent would replace it, + push opt-ins and all, so that is main's `error:` and exit 2.""" + try: + with open(path, "rb") as fh: + return fh.read() + except FileNotFoundError: + return b"" + + def _init(args) -> int: """Add the standard `[[watch]]` table for one agent to `config.toml`. @@ -86,10 +97,9 @@ def _init(args) -> int: # name a new inode, and the other name keeps the old bytes. if st.st_nlink > 1: return _refuse(f"{path} has another hard link; add the table by hand") - with open(path, "rb") as fh: - old = fh.read() except FileNotFoundError: - old = b"" + pass # no file yet, so the table is the whole of it + old = _config_bytes(path) try: before = tomllib.loads(old.decode("utf-8")) except ValueError as exc: # `TOMLDecodeError`, and `UnicodeDecodeError` on a bad byte @@ -126,7 +136,15 @@ def _init(args) -> int: ): mine = f"{path} [[watch]] #{len(have) + 1}:" return _refuse("; ".join(n for n in notes if n.startswith(mine))) - store._write_atomic(path, new) # 0600, and a running watcher reads old or new + # Two inits at once both read the same bytes, and the later rename dropped + # the other's table while both said "added". Each holds this across a second + # read and the rename, so the later one sees the file move and says so. Not + # taken earlier: it makes `.locks/`, and a refusal writes nothing. An editor + # saving in the same instant takes no lock, and is not covered. + with store._lockfile(os.path.join(home, ".locks", "config.toml.lock")): + if _config_bytes(path) != old: + return _refuse(f"{path} changed while init was reading it; run it again") + store._write_atomic(path, new) # 0600, and a running watcher reads old or new print(f"{'added to' if old else 'wrote'} {path}: watch {root} for {args.agent}") if not os.path.isdir(real): print(f"note: {root} does not exist yet; it is watched once it does", file=sys.stderr) @@ -437,8 +455,9 @@ def _recall(args) -> int: waiting, age = daemon.uncollected(home) if waiting and age >= daemon.STALE_RECORD: print( - f"{waiting} hook record(s) uncollected for {_ago(age)}: is `gitmemory watch` " - f"running for {home}? These results may be missing everything said since.", + f"{waiting} hook record(s) uncollected, the oldest for {_ago(age)}: is " + f"`gitmemory watch` running for {home}? These results may be missing " + "everything said since.", file=sys.stderr, ) db = index.open_db(path) @@ -590,7 +609,7 @@ def _redact_flag(parser: argparse.ArgumentParser) -> None: action="store_true", default=os.environ.get("GITMEMORY_REDACT", "") not in ("", "0"), help="mask the credential shapes the push gate detects in what is printed " - "(default on when $GITMEMORY_REDACT is set and not 0)", + "(default on when $GITMEMORY_REDACT is set to anything but empty or 0)", ) diff --git a/tests/mutate_index.py b/tests/mutate_index.py index df7063b..87f576a 100644 --- a/tests/mutate_index.py +++ b/tests/mutate_index.py @@ -4769,7 +4769,7 @@ # anchor appear twice and the harness would skip itself. [E7] "the docstring's row count drifts away from the index again", "tests/mutate_index.py", - "a full pass is 6" + "57 mutants", + "a full pass is 6" + "58 mutants", "a full pass is 14" + "6 mutants", "test_the_row_count_in_the_docstring_is_the_row_count", ), @@ -5783,8 +5783,8 @@ # Unreadable taken for absent: the file is replaced by the table alone. "init takes a config.toml it cannot read for a missing one", "__main__.py", - ' except FileNotFoundError:\n old = b""', - ' except OSError:\n old = b""', + ' except FileNotFoundError:\n return b""', + ' except OSError:\n return b""', "test_cli_init_says_error_and_exits_2_when_it_could_not_look", ), ( @@ -5812,6 +5812,15 @@ " continue", "test_cli_init_refuses_a_root_another_agent_already_watches", ), + ( + # Two inits read the same bytes; the later rename drops the other's + # table, and both say "added". + "init renames over a config another init wrote after it read it", + "__main__.py", + " if _config_bytes(path) != old:", + " if False:", + "test_cli_init_says_so_when_another_init_wrote_first", + ), ( "a killed init's temp copy of config.toml is committed", "gitrepo.py", @@ -5916,12 +5925,13 @@ def run(args: list[str]) -> int: figure depends on something the docstring used to get wrong. `addopts` carries `-m 'not corpus'`, but only one test is marked `corpus`; the ~328 conformance cases are gated on the `.conformance/` clones being present, not - on a marker. So this is the 955-test offline suite on a machine without the - clones and the 1280-test suite on a machine with them — **measured**, 81 s - for the former on the author's laptop. It said "about a second" from E3 - until somebody read it next to the "about a minute" nine lines down and - noticed the same call described twice, an order of magnitude apart. - [E3; review: docs, claims] + on a marker. So this is the offline suite on a machine without the clones + and the corpus-on suite on a machine with them — **measured**, 81 s for the + former on the author's laptop, when the two were 955 and 1280 tests; both + have grown since and it has not been re-timed. It said "about a second" + from E3 until somebody read it next to the "about a minute" nine lines + down and noticed the same call described twice, an order of magnitude + apart. [E3; review: docs, claims] `ANCHOR_TEST` as well, for the reason written where it is defined: it is the one test in the suite that is *supposed* to fail while a mutant is applied, @@ -6038,19 +6048,20 @@ def _select(test: str) -> list[str]: def main() -> int: """Run every mutant, or only those whose name contains an argument. - The filter exists because a full pass is 657 mutants × two suite runs — + The filter exists because a full pass is 658 mutants × two suite runs — long enough that adding one row and checking it used to mean either waiting - for the other 567 or trusting the new one untested. + for every other row or trusting the new one untested. Cost: **measured**, three rows timed end to end at 3.7 s, 17.1 s and 17.2 s, on a machine with the `.conformance/` clones — so against the - 1280-test suite, not the 955-test offline one (`run` explains why the two - differ). **Estimated** from those, a full pass is roughly one to two and a - half hours. The spread is not noise and does not average away — the first - of the two runs is `-x`, so a row costs however long the suite takes to - *reach* the test the mutant breaks, and that is a position in collection - order, not a property of the row. A row the suite misses entirely pays for - the whole suite instead. + corpus-on suite, not the offline one (`run` explains why the two differ), + when they were 1280 and 955 tests. **Estimated** from those, a full pass + was then roughly one to two and a half hours; the suite and the index have + both grown since, and neither figure has been re-timed. The spread is not + noise and does not average away — the first of the two runs is `-x`, so a + row costs however long the suite takes to *reach* the test the mutant + breaks, and that is a position in collection order, not a property of the + row. A row the suite misses entirely pays for the whole suite instead. Every figure in this paragraph had drifted: "41 s a row", "1,126-test", "71 verdicts in 49 minutes" and "about six hours" were all carried forward diff --git a/tests/test_daemon.py b/tests/test_daemon.py index 14f0d9a..e3e22b6 100644 --- a/tests/test_daemon.py +++ b/tests/test_daemon.py @@ -126,11 +126,13 @@ def test_a_malformed_config_means_watch_nothing_not_watch_everything(tmp_path): def test_a_watch_missing_its_roots_is_skipped_and_the_rest_survive(tmp_path): + home = str(tmp_path / "home") + root = tmp_path / "proj" # never created: a root that does not exist yet is kept _write( - os.path.join(str(tmp_path), "config.toml"), - '[[watch]]\nagent = "a"\n\n[[watch]]\nagent = "b"\nroots = ["/tmp/x"]\n', + os.path.join(home, "config.toml"), + f'[[watch]]\nagent = "a"\n\n[[watch]]\nagent = "b"\nroots = [{json.dumps(str(root))}]\n', ) - got = daemon.load_watches(str(tmp_path)) + got = daemon.load_watches(home) assert [w.agent for w in got] == ["b"] diff --git a/tests/test_docs.py b/tests/test_docs.py index 3e1d24a..f7209ce 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -185,6 +185,35 @@ def test_the_readme_arithmetic_paragraph_is_the_arithmetic(): assert f"replace three of the {offline}" in text +def test_the_reproduce_guide_states_the_readme_counts(): + """docs/REPRODUCE.md repeats the board's four numbers, and nothing held them. + + So they drifted: main said 1027 tests beside the README's 1,037, and an + install-feedback commit left 1043 beside 1,063. The README is pinned to the + suite by the tests above, so holding the guide to the README holds it to + the suite, with no second collection, and on a machine without the corpora + too. [review] + """ + board = _read("README.md") + guide = " ".join(_read("docs/REPRODUCE.md").split()) + + def number(pattern: str) -> int: + found = re.search(pattern, board) + assert found, pattern + return int(found.group(1).replace(",", "")) + + tests = number(r"\*\*([\d,]+) tests\*\*") + cases = number(r"(\d[\d,]*) conformance cases") + online = number(r"collects (\d+), not \d+\.") + controls = number(r"\*\*([\d,]+)\*\* negative controls") + assert ( + f"**Expected:** `{tests}/{tests + 1} tests collected (1 deselected)`, then " + f"`{online}/{online + 1} tests collected (1 deselected)`, then `{cases}`." + ) in guide, "docs/REPRODUCE.md §5b does not expect the README's counts" + assert f"why {tests:,} + {cases:,} is {online:,} and not {tests + cases:,}" in guide + assert f"**Expected:** `{controls}`." in guide, "§5c does not expect the README's count" + + def test_the_design_document_does_not_promise_a_file_that_is_not_written(): """`manifest.json` was the name in every version of the verification recipe. @@ -224,10 +253,11 @@ def test_the_disk_section_calls_rebuildable_only_what_is_gitignored(): The ratios are not pinned: they move with the filesystem's block size, the git version, when `gc --auto` last packed, and the corpus. What is pinned is - the advice. Deleting `index/` to get space back is safe only while git - ignores `index/` and tracks `raw/` and `sessions/`, so that is checked - against the store's own `.gitignore`. The anchors pointing at the section - are checked against its heading, so renaming it breaks a test and not just a link. + the advice. Every directory a sentence of the section tells you to delete + must be one the store's own `.gitignore` ignores, and `raw/` and `sessions/` + must stay tracked, because deleting either loses the only copy. The anchors + pointing at the section are checked against its heading, so renaming it + breaks a test and not just a link. Headings are read with fenced blocks cut out: a `# comment` in a shell or TOML recipe is not a heading, and counting it let a link to no heading pass. """ @@ -240,7 +270,15 @@ def test_the_disk_section_calls_rebuildable_only_what_is_gitignored(): assert "`index/`" in body and "`gitmemory index`" in body, "it no longer says what rebuilds it" ignored = gitrepo.GITIGNORE.splitlines() - assert "/index/" in ignored, "the section says to delete a directory git may be tracking" + told = [ + d + for sentence in re.split(r"(?<=\.)\s+", " ".join(body.split())) + if re.search(r"\b(?:delete|remove)\b", sentence, re.I) + for d in re.findall(r"`([\w.-]+/)`", sentence) + ] + assert told, "the section no longer names the directory to delete" + for d in told: + assert f"/{d}" in ignored, f"the section says to delete `{d}`, which git does not ignore" for durable in ("/raw/", "/sessions/"): assert durable not in ignored, f"{durable} is ignored, so git no longer holds it" @@ -248,7 +286,7 @@ def test_the_disk_section_calls_rebuildable_only_what_is_gitignored(): re.sub(r"[^\w\- ]", "", h.strip().lower()).replace(" ", "-") for h in re.findall(r"^#+ (.+)$", watching, re.M) } - for rel in ("README.md", "docs/ARCHITECTURE.md", "docs/DESIGN.md"): + for rel in ("README.md", "CHANGELOG.md", "docs/ARCHITECTURE.md", "docs/DESIGN.md"): anchors = re.findall(r"watching\.md#([\w-]+)", _read(rel)) assert "how-much-disk-it-takes" in anchors, f"{rel} does not point at the section" for anchor in anchors: @@ -326,7 +364,10 @@ def test_the_design_document_cites_nothing_in_tmp(): def _installs(text: str): """(line number, tool, its words, as written) for each install command in - `text`. A line that ends in a backslash goes on into the next, as in sh.""" + `text`. A line that ends in a backslash goes on into the next, as in sh. A + tool whose span holds nothing else takes the rest of the line, up to the + next tool, when that starts with a source or a flag: `` `uv tool install` + git+… `` is one command split across spans, not prose.""" lines, n = text.splitlines(), 0 while n < len(lines): start, line = n + 1, lines[n] @@ -335,7 +376,13 @@ def _installs(text: str): line = line[:-1] + " " + lines[n] n += 1 for m in _INSTALL.finditer(line): - yield start, m.group(1), m.group(2).split(), " ".join(m.group(0).split()) + words = m.group(2).split() + if not words: + rest = re.split(_INSTALLERS, line[m.end() :], maxsplit=1)[0] + rest = rest.replace("`", " ").split() + if rest and rest[0].strip("\"'").startswith(("git+", "gitmemory", "-")): + words = rest + yield start, m.group(1), words, " ".join([m.group(1), *words]) def _names_python(tool: str, words: list[str], want: str) -> bool: @@ -366,7 +413,9 @@ def test_every_documented_install_names_the_python_floor(): names the tool with nothing after it ("if you installed with `uv tool install`,") has no source, so it is not one. Prose with words after the tool and no code span around it reads as a command and fails, so put the - tool in a code span or write the whole command. `tests/` is not scanned, + tool in a code span or write the whole command; a span holding the tool + alone still reads on when the next word is a source or a flag + (`` `uv tool install` git+… ``). `tests/` is not scanned, because its strings are fixtures and this one names the tools; nor are the records, which quote what was run at the time. [install feedback 1] """ @@ -384,6 +433,7 @@ def test_every_documented_install_names_the_python_floor(): f"uv tool install \\\n --python {v} git+x\n" f"`pipx install --python python{v} git+x` `pipx install -p python{v} git+x`\n" "if you installed with `uv tool install`,\n" + f"`uv tool install` git+x, then uv tool install `--python {v} git+x`\n" ) found = [(n, t, _names_python(t, w, want[t])) for n, t, w, _ in _installs(probe) if w] assert found == [ @@ -394,6 +444,8 @@ def test_every_documented_install_names_the_python_floor(): (3, "uv tool install", True), (5, "pipx install", True), (5, "pipx install", False), + (7, "uv tool install", False), + (7, "uv tool install", True), ], found records = ("tests/", "docs/reviews/", "docs/tasks/", "docs/DESIGN-v0.md") diff --git a/tests/test_index.py b/tests/test_index.py index eb95941..25ab5cc 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -1599,8 +1599,8 @@ def test_a_subagent_prompt_is_not_the_users_word(home, tmp_path): def test_recall_marks_a_subagent_turn(home, tmp_path, capsys): """A prefix on the role, not a new column: `split()[3]` is role/kind to - whatever parses these lines, this file included, and a main session's line - is byte-for-byte what it was.""" + whatever parses these lines, this file included, and a main session's + role/kind carries no prefix.""" with_a_subagent(home, tmp_path) capsys.readouterr() assert main(["--home", home, "recall", "retry loop upload"]) == 0 @@ -1671,16 +1671,24 @@ def test_recall_says_the_hook_rang_and_no_watcher_answered(home, src, capsys): out, err = capsys.readouterr() assert "marmoset" in out and "uncollected" not in out, out assert err == ( - "2 hook record(s) uncollected for 3 hours: is `gitmemory watch` running for " - f"{home}? These results may be missing everything said since.\n" + "2 hook record(s) uncollected, the oldest for 3 hours: is `gitmemory watch` " + f"running for {home}? These results may be missing everything said since.\n" ), err assert main(["--home", home, "recall", "aardvark"]) == 0 err = capsys.readouterr().err assert err.index("uncollected") < err.index("no matches"), err - for age, said in ((90 * 60, "90 minutes"), (3 * 86400, "3 days")): + # Each unit starts at 2, so the number is never 1 and its plural always fits. + for age, said in ( + (90 * 60, "90 minutes"), + (2 * 3600 - 30, "119 minutes"), + (2 * 3600 + 30, "2 hours"), + (2 * 86400 - 30, "47 hours"), + (2 * 86400 + 30, "2 days"), + (3 * 86400, "3 days"), + ): _ring(home, age) assert main(["--home", home, "recall", "marmoset"]) == 0 - assert f"uncollected for {said}:" in capsys.readouterr().err + assert f"the oldest for {said}:" in capsys.readouterr().err def test_recall_says_nothing_about_a_spool_a_watcher_could_still_be_draining( @@ -1714,7 +1722,8 @@ def test_one_watcher_pass_is_what_clears_it(home, src, capsys): os.utime(stuck, (time.time() - 3 * 3600,) * 2) capsys.readouterr() assert main(["--home", home, "recall", "marmoset"]) == 0 - assert capsys.readouterr().err.startswith("1 hook record(s) uncollected for 3 hours:") + err = capsys.readouterr().err + assert err.startswith("1 hook record(s) uncollected, the oldest for 3 hours:"), err assert main(["--home", home, "watch", "--once"]) == 0 assert os.path.isdir(stuck), "the pass removed the directory; this test no longer bites" capsys.readouterr() @@ -1839,6 +1848,9 @@ def test_mask_returns_clean_text_unchanged_and_keeps_the_words_around_a_key(): assert two == f"one ----…[{len(PEM)} bytes] between ----…[{len(PEM)} bytes] two", two # Cut before its END line, the rest of the text is the key, so the rest goes. assert redact.mask("x " + PEM[:80]) == "x ----…[80 bytes]" + # A second key in the body is inside the header's span, and must not end it. + inner = PEM.replace(PEM_BODY + "\n", f"{PEM_BODY}\n{AKIA}\n", 1) + assert redact.mask(f"x {inner} y") == f"x ----…[{len(inner)} bytes] y" def test_recall_redact_masks_a_key_behind_a_character_the_index_spells_out(home, src, capsys): @@ -1871,7 +1883,9 @@ def test_recall_redact_masks_a_key_behind_a_character_the_index_spells_out(home, assert "\\ufeffAKIA…[20 bytes] x\\u200bghp_…[40 bytes] y\\x7fAKIA…[20 bytes]" in out, out -def test_mask_stays_linear_on_a_page_of_key_headers_and_an_end_line_that_never_closes(): +def test_mask_stays_linear_on_a_page_of_key_headers_and_an_end_line_that_never_closes( + monkeypatch, +): """Two ways the PEM widening went quadratic on text an agent fetched. Each header searched for its own END line, so headers with none cost @@ -1880,19 +1894,32 @@ def test_mask_stays_linear_on_a_page_of_key_headers_and_an_end_line_that_never_c 41 s. And the END pattern's unbounded runs retried every `KEY` in a `-----END KEY KEY …` line to its end: 125 KB took 4.1 s. - At these sizes each would be about 15 s and 10 s. 1.5 s is several times - what the linear code costs here, and fails long before anyone would wait. + The page of headers is counted, not timed: one header searches, and every + later one sits inside the span that search found. The END line is one + `search` whose cost is inside `re` and cannot be counted, so it is timed, + as `test_a_separator_free_megabyte_does_not_hang_the_build` is: about 10 s + at this size when quadratic, and 1.5 s is several times the linear cost. """ header = "-----BEGIN " + "PRIVATE KEY-----\n" - for label, text in ( - ("headers with no END", header * 40_000), - ("an END line that never closes", header + "-----END " + "KEY " * 50_000), - ): - started = time.monotonic() - out = redact.mask(text) - elapsed = time.monotonic() - started - assert out == f"----…[{len(text)} bytes]", (label, out[:80]) - assert elapsed < 1.5, f"{label}: the PEM widening went quadratic again: {elapsed:.2f}s" + real, searched = redact._PEM_END, [] + + class Counted: + def search(self, data, pos): + searched.append(pos) + return real.search(data, pos) + + with monkeypatch.context() as m: + m.setattr(redact, "_PEM_END", Counted()) + text = header * 2_000 + assert redact.mask(text) == f"----…[{len(text)} bytes]" + assert searched == [len(header) - 1], f"{len(searched)} headers searched for an END line" + + text = header + "-----END " + "KEY " * 50_000 + started = time.monotonic() + out = redact.mask(text) + elapsed = time.monotonic() - started + assert out == f"----…[{len(text)} bytes]", out[:80] + assert elapsed < 1.5, f"the END line pattern went quadratic again: {elapsed:.2f}s" def test_recall_redact_keeps_a_subagent_turn_marked(home, tmp_path, capsys): diff --git a/tests/test_init.py b/tests/test_init.py index 09ef86f..05a6678 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -176,6 +176,24 @@ def test_cli_init_says_error_and_exits_2_when_it_could_not_look(tmp_path, capsys assert kept.read_text() == body +def test_cli_init_says_so_when_another_init_wrote_first(tmp_path, capsys, monkeypatch): + """Two inits at once read the same bytes, and the later rename dropped the + other's table while both said "added". The other one is run here between + this one's read and its write: this one now sees the file move, and says so.""" + home = str(tmp_path / "store") + watches_from = daemon.watches_from + + def racer(*args): + monkeypatch.setattr(daemon, "watches_from", watches_from) + assert main(["--home", home, "init", "pi"]) == 0 + return watches_from(*args) + + monkeypatch.setattr(daemon, "watches_from", racer) + assert main(["--home", home, "init", "claude-code"]) == 1 + assert "changed while init was reading it; run it again" in capsys.readouterr().err + assert [w.agent for w in daemon.load_watches(home)] == ["pi"] + + def test_cli_init_refuses_a_root_another_agent_already_watches(tmp_path, capsys, _no_real_home): """Two watches on one root at the same depth: `discover` keeps the first, so the table init would add is dead on arrival. Spelled absolute and with a @@ -207,14 +225,24 @@ def test_cli_init_knows_only_adapters_that_ship_and_roots_the_docs_name(): assert done.value.code == 2 -def test_cli_init_leaves_no_temp_the_watcher_would_commit(tmp_path): +def test_cli_init_leaves_no_temp_the_watcher_would_commit(tmp_path, monkeypatch): """`init` publishes `config.toml` by rename, so an init killed between the write and the `os.replace` leaves a full copy of the file — push credential - included — beside it, and the watcher's `git add --all` would commit it.""" + included — beside it, and the watcher's `git add --all` would commit it. + Asked of the temp the writer really made, while it exists: a name typed into + the test passed against a writer that spelled its temp any other way.""" home = str(tmp_path / "store") gitrepo.init(home) - stray = "config.toml.tmp.4242.7f" - with open(os.path.join(home, stray), "w", encoding="utf-8") as fh: - fh.write('[remote.origin]\nurl = "https://example.invalid/x.git"\n') - ignored = subprocess.run(["git", "-C", home, "check-ignore", "-q", stray], check=False) - assert ignored.returncode == 0 + asked = [] + rename = os.replace + + def spy(src, dst): + if os.path.basename(dst) == "config.toml": + rel = os.path.relpath(src, home) + ask = subprocess.run(["git", "-C", home, "check-ignore", "-q", rel], check=False) + asked.append((rel, ask.returncode)) + return rename(src, dst) + + monkeypatch.setattr(os, "replace", spy) + assert main(["--home", home, "init", "pi"]) == 0 + assert [code for _, code in asked] == [0], asked diff --git a/tests/test_show.py b/tests/test_show.py index dabd362..55db2ca 100644 --- a/tests/test_show.py +++ b/tests/test_show.py @@ -233,6 +233,9 @@ def test_a_captured_show_result_is_an_echo_the_skill_lists(home, tmp_path, capsy with open(os.path.join(root, "skills", "recall", "SKILL.md")) as fh: echoes = fh.read().split("\n## Echoes\n")[1].split("\n## ")[0] assert "`gitmemory show ...`" in echoes and "`> `" in echoes, echoes + # The header's shape, not "a quote or a bullet that names an address": + # a real turn can open with either and cite one further on. + assert "`- ` immediately followed by an `/" in " ".join(echoes.split()), echoes def test_show_on_no_store_and_a_negative_context_exit_2(tmp_path, capsys): From f0ebb41284a0242518f8418f3aef7708f89b17cb Mon Sep 17 00:00:00 2001 From: doronp <3586743+doronp@users.noreply.github.com> Date: Sun, 18 Oct 2026 18:16:54 +0300 Subject: [PATCH 13/13] fix: answer the final review gate of the install-feedback round Kimi K3's final pass, over the fix commit and the skill as it now reads whole, and an Opus gate of five lenses over the whole branch, with two refute-by-default skeptics per finding, raised 33 findings between them. This answers them. The record is the "Final gate" section of docs/reviews/install-feedback-round.md, which also corrects show-7 and moves service-3 to fixed. Four changes of behaviour, each with a test that failed first and a new mutation row: - show --redact no longer stops with exit 2 partway through a turn that holds a lone surrogate. mask() encodes with surrogatepass, as records hashes; surrogateescape round-trips only U+DC80-U+DCFF. - recall --redact masks as show does, with the mask's own collapse, and flattens and cuts after it. str.split() broke a value at a no-break space before the mask saw it, so recall printed what show and the gate flag. A header a line break split is still joined. - show refuses a masked address with a line of its own, still exit 2, not with "as recall prints it" about what recall printed. - The printed systemd removal ends with daemon-reload. Two simplifications: init refuses in the except of its re-parse and drops `have`, and uncollected() loses a clock nothing passed. Docs: the --redact caveat list and the settings.json snippet are kept in USAGE.md only, and SECURITY.md and hook/README.md link them. The list now says the store lowercases a session id, so a credential-shaped one prints unmasked, and that a key split across two blocks is masked to the end of the first. The --redact sample comes from its own transcript. While masking, the skill reads nothing under the store by hand; it puts -- before a topic that starts with -, and states each rule once, with one retry, -k 50. REPRODUCE 5b sets HOME as the suite does, so a generic account collects what the README says. The CHANGELOG, DESIGN.md, watching.md and the docs index follow. A2, the push gate missing a key right after a JSON escape, is on main and left for its own issue. Pins move to 1,134 tests and 662 negative controls. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QPBLmJD2q1PVxdkWx8C16U Signed-off-by: doronp <3586743+doronp@users.noreply.github.com> --- CHANGELOG.md | 43 +++++---- README.md | 8 +- SECURITY.md | 30 ++---- docs/DESIGN.md | 8 +- docs/README.md | 2 +- docs/REPRODUCE.md | 19 ++-- docs/USAGE.md | 35 ++++--- docs/reviews/install-feedback-round.md | 125 +++++++++++++++++++++++-- docs/watching.md | 25 +++-- hook/README.md | 22 ++--- skills/recall/SKILL.md | 66 ++++++------- src/gitmemory/__main__.py | 29 +++--- src/gitmemory/daemon.py | 4 +- src/gitmemory/redact.py | 14 ++- src/gitmemory/service.py | 6 +- tests/mutate_index.py | 50 ++++++++-- tests/test_index.py | 19 +++- tests/test_service.py | 6 +- tests/test_show.py | 39 ++++++++ 19 files changed, 382 insertions(+), 168 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9a2cf4..b6616d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,8 @@ tell you. [docs/watching.md](docs/watching.md#configure-what-to-watch). - **`gitmemory service install`** writes a systemd user unit (Linux) or a LaunchAgent (macOS) that keeps `gitmemory watch` running, and prints the - commands that start it, check it, follow its log, restart it after an + commands that start it, keep it running after you log out (Linux: + `loginctl enable-linger`), check it, follow its log, restart it after an upgrade and remove it. It starts nothing itself, and it refuses, exiting 1, to overwrite a file that is already there. The file names the Python, the store and git's directory outright, because a user service inherits none of @@ -74,28 +75,30 @@ tell you. index, so it needs no index and a stale one cannot change the answer. `--context N` (`-C N`) adds N turns either side; a turn cut by a segment boundary comes back whole; an offset inside a turn shows that turn; a - subagent's turn is headed `subagent:`. An address that is not one exits - 2, and one the store does not hold exits 1. The recall skill uses it in place - of reading `raw/` by hand. See + subagent's turn is headed `subagent:`. An address that is not one, or + is masked, exits 2, and one the store does not hold exits 1. The recall + skill uses it in place of reading `raw/` by hand. See [USAGE.md](docs/USAGE.md#read-the-whole-turn). -- **`recall --redact`** and **`show --redact`** mask the credential shapes - the push gate detects in what they print, the text and the address, as +- **`recall --redact`** and **`show --redact`** mask the credential shapes the + push gate detects in what they print, the text and the address (except, in + a session id, a key the store's lowercasing hides from its detector), as `AKIA…[20 bytes]`; a private key is masked down to its END line, or to the - end of the text without one. `show` masks each block whole before it splits - it into lines, so a key's body goes with its header. Off by default. + end of the text without one. `show` masks each block whole before it splits it into + lines, so a key's body goes with its header. Off by default. `GITMEMORY_REDACT` makes it standing for both (unset, empty or `0` is off), - and the plugin's skill passes the flag to both whenever it is set and not - `0`, and then never reads a turn from `raw/`. It masks only those shapes, - and only in what the two commands print; see + and the plugin's skill passes the flag to both whenever it is set to + anything but empty or `0`, and then reads nothing under the store by hand. + It masks only those shapes, and only in what the two commands print; see [docs/USAGE.md](docs/USAGE.md#masking-credentials-in-what-it-prints) for what it does not cover. -- **How much disk it takes.** Budget about twice what you capture: `raw/` - holds the bytes once, `.git/` a compressed copy, and `index/` the searchable - text again, which `gitmemory index` rebuilds (and, while it runs, needs room - for the old index and the new). Measured at 2.3x to 3.4x on synthetic - corpora; see [docs/watching.md](docs/watching.md#how-much-disk-it-takes). - ARCHITECTURE.md said the store holds each byte exactly once, and DESIGN.md - that it has zero duplication; both are true of `raw/` only. +- **How much disk it takes.** Budget about twice what you capture, and more if + your transcripts compress poorly: `raw/` holds the bytes once, `.git/` a + compressed copy, and `index/` the searchable text again, which + `gitmemory index` rebuilds (and, while it runs, needs room for the old index + and the new). Measured at 2.3x to 3.4x on synthetic corpora; see + [docs/watching.md](docs/watching.md#how-much-disk-it-takes). ARCHITECTURE.md + said the store holds each byte exactly once, and DESIGN.md that it has zero + duplication; both are true of `raw/` only. ### Changed @@ -115,6 +118,10 @@ tell you. parentheses; under 1000 bytes it is `+N B`. A script that parsed `+NB` out of either line needs the new form. `size=` stays an exact offset, and the commit subject keeps `+NB`. See [USAGE.md](docs/USAGE.md#set-it-up-once). +- The watcher's warning for a root inside the store now reads + `root is inside the store; skipped`. A root that contains the store + still says `contains the store itself`. `gitmemory init` prints the same + sentence as its reason when it refuses such a root. ### Fixed diff --git a/README.md b/README.md index 31b9f84..d398b40 100644 --- a/README.md +++ b/README.md @@ -217,13 +217,13 @@ real text, and that is a measurement rather than a suspicion. | What | How it was measured | Result | |---|---|---| | Hook cost in the agent's critical path | Timed against spawning `true` the same way, three runs of 400 | p50 **7.4 – 7.5 ms**, p99 **10.2 – 11.5 ms** | -| The suite | On a fresh checkout, no downloads | **1,131 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | -| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **658** negative controls | +| The suite | On a fresh checkout, no downloads | **1,134 tests**, and **328 conformance cases** against three third-party corpora, one gated on the LongMemEval download and one on `pip install -e '.[serve]'` | +| Whether the tests hold anything | Every fix mutated to remove its behaviour; the named test must fail | **662** negative controls | The two suite counts do not add up, and should not. Switching the corpora on -collects 1456, not 1459. Three conformance cases fill parametrisations that +collects 1459, not 1462. Three conformance cases fill parametrisations that collect as one empty placeholder each while the corpora are absent, so they -replace three of the 1131 rather than joining them. Every figure on this board +replace three of the 1134 rather than joining them. Every figure on this board is pinned by a test, which is how it stays true. **What is not measured**, and is shown as a row on the dashboard rather than diff --git a/SECURITY.md b/SECURITY.md index 63201e4..24168a0 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -98,25 +98,11 @@ on the masking below. That includes text another watched agent never sent to that provider, and a credential `push` would refuse to ship. The skill quotes what the question needs and no more, which is a rule it follows, not a gate. -`recall --redact` and `show --redact`, or `GITMEMORY_REDACT` set to anything -but empty or `0`, mask the gate's shapes in what those two commands print, and -the skill passes the flag to both whenever the variable is set and not `0`, -and then does not read `raw/`. It masks and never refuses, and it is off -unless you turn it on. What it does not guarantee: - -- It masks only the sixteen shapes in [`redact.py`](src/gitmemory/redact.py): - twelve issuer-prefixed keys and four SUSPECT patterns. Anything else prints. - Measured examples: a bare AWS secret key, `PGPASSWORD=…`, a custom - `GH_TOKEN=…`, `glpat-` and `npm_` tokens, an Azure `AccountKey=…`, a password - in prose, `curl -u user:pass`, a `.netrc` line. -- Nothing is decoded. Base64, URL-encoding and a token wrapped across lines get - through. JSON `\u` escapes are masked, because both commands read the parsed - text. -- It is credential masking, not confidentiality. Hostnames, customer data, - personal data and code are untouched. -- It covers only what `recall` and `show` print. `raw/`, the index database, - the dashboard, `derived/` and any file the agent opens are not masked by it. - The environment setting is a floor, not a boundary: an agent with a shell can - unset it, or read `raw/`. -- A mask reveals the first four bytes and the length. -- The SUSPECT patterns also mask some prose about configuration. +`recall --redact` and `show --redact` mask the gate's shapes in what those two +commands print. `GITMEMORY_REDACT` set to anything but empty or `0` turns it on +for both, and the skill then passes the flag to both and reads nothing under +the store by hand. It masks and never refuses, and it is off unless you turn it +on. What it does not guarantee is listed in +[USAGE.md](docs/USAGE.md#what-it-does-not-guarantee): it masks sixteen shapes +and nothing else, decodes nothing, and covers only what those two commands +print. diff --git a/docs/DESIGN.md b/docs/DESIGN.md index cc1ab4e..f9fb892 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -223,11 +223,9 @@ opt-in per remote and refuses unless the redaction gate passed.** Default config not a gate. Off by default, because a store on the owner's own disk is still not an egress. The owner turns it on, or makes it standing with `GITMEMORY_REDACT`, when their output goes on to a model provider through the plugin's skill. Both tiers are masked, unlike `derived/` below, because a -mask costs the span and not the sentence; a PEM block from its BEGIN line to its END line, or to -the end of the text when there is none; and the session id in the address. `recall` masks after -the whitespace collapse and before the 160-character cut. `show` masks each block whole before it -splits it into lines, and, because it keeps the line breaks, also scans a copy with each whitespace -run collapsed. Neither refuses anything. `raw/` and the index stay verbatim.]** +mask costs the span and not the sentence. Neither refuses anything, and `raw/` and the index stay +verbatim. [USAGE.md](USAGE.md#masking-credentials-in-what-it-prints) says what is masked, and +what is not.]** Not "anything that leaves the machine" — that claim was wider than the code and is withdrawn **[E7]**. The store is an ordinary git repository in a directory the owner can read. A `git push` diff --git a/docs/README.md b/docs/README.md index d15a9d6..4ecf443 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,7 +9,7 @@ | [USAGE.md](USAGE.md) | Getting value out of it: the daily loop, reading `recall`, handing memory back to an agent | | [RESULTS.md](RESULTS.md) | Every published result, including the ones it loses | | [REPRODUCE.md](REPRODUCE.md) | The command behind each result, what it needs, and how closely a rerun matches | -| [watching.md](watching.md) | Configuring `gitmemory watch`: roots, flags, and telling a misconfigured watcher from an idle one | +| [watching.md](watching.md) | Configuring `gitmemory watch`: roots, flags, telling a misconfigured watcher from an idle one, keeping it running as a user service, and how much disk the store takes | | [../hook/README.md](../hook/README.md) | Installing the optional hook shim, and its measured cost | | [agents.md](agents.md) | Which agents can be captured, which cannot, and why | diff --git a/docs/REPRODUCE.md b/docs/REPRODUCE.md index d647814..e044edc 100644 --- a/docs/REPRODUCE.md +++ b/docs/REPRODUCE.md @@ -475,22 +475,29 @@ which is the load and not the shim. conformance cases the three third-party corpora add. ```sh -# the fresh-checkout count: point both corpus variables at nothing +# the fresh-checkout count: point both corpus variables at nothing, and HOME +# at a new directory +HOME=$(mktemp -d)/isolated-home \ GITMEMORY_CC_FIXTURES=$PWD/no-such-corpus GITMEMORY_PI_FIXTURES=$PWD/no-such-corpus \ uv run pytest -q --collect-only -p no:cacheprovider | tail -1 # with the corpora cloned by tests/fetch_fixtures.sh -uv run pytest -q --collect-only -p no:cacheprovider | tail -1 +HOME=$(mktemp -d)/isolated-home uv run pytest -q --collect-only -p no:cacheprovider | tail -1 uv run pytest -q --collect-only -p no:cacheprovider | grep -c '\.jsonl\]$' ``` -**Expected:** `1131/1132 tests collected (1 deselected)`, then -`1456/1457 tests collected (1 deselected)`, then `328`. The deselected test is +**Expected:** `1134/1135 tests collected (1 deselected)`, then +`1459/1460 tests collected (1 deselected)`, then `328`. The deselected test is the LongMemEval corpus test, which is opt-in: `uv run pytest -q -m corpus` after `bench/fetch_longmemeval.sh`. The README's -paragraph on why 1,131 + 328 is 1,456 and not 1,459 is the arithmetic of these +paragraph on why 1,134 + 328 is 1,459 and not 1,462 is the arithmetic of these three numbers. All three were run while writing this guide. +The `HOME` is the one the suite gives itself. The owner-data scan adds two +cases named after your account, the last part of `$HOME`, and leaves them out +when that name is shorter than four characters or a generic one such as +`ubuntu`, `runner` or `root`. Without it, such an account collects two fewer. + `tests/test_docs.py` holds the README to all three numbers, the second and third only where the corpora are cloned, and this guide, §5c's figure included, to the README, so a drifted count fails the suite. To run the suite @@ -506,7 +513,7 @@ test fail. The count is the number of rows in `tests/mutate_index.py`. uv run python -c "import sys; sys.path.insert(0, 'tests'); from mutate_index import MUTANTS; print(len(MUTANTS))" ``` -**Expected:** `658`. Checked while writing this guide. +**Expected:** `662`. Checked while writing this guide. To run the controls rather than count them: diff --git a/docs/USAGE.md b/docs/USAGE.md index 6e12d4f..dbefd85 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -140,30 +140,33 @@ exits 1. `--redact` masks credentials in what it prints `recall` and `show` print what the transcript says, credentials included, because the store is yours and on your disk. When their output is going further, to an agent and with it to the agent's model provider, `--redact` -masks every shape the push gate knows, in the text and in the address. That is +masks every shape the push gate knows, in the text and in the address, with +one exception for a session id ([below](#what-it-does-not-guarantee)). That is twelve issuer-prefixed keys and four SUSPECT patterns, listed in -[`redact.py`](../src/gitmemory/redact.py): +[`redact.py`](../src/gitmemory/redact.py). This sample is from a second +transcript, a one-turn session of its own: ```console $ gitmemory recall --redact "staging key" - -0.000 2026-09-28T10:02Z claude-code/0f3a9c2e-…-fc59494095426389/g00@0 user/text Use AKIA…[20 bytes] for the staging key, never the prod one. + -0.000 2026-10-02T09:41Z claude-code/7c41d2e9-…-61a9a184ea213ec3/g00@0 user/text Use AKIA…[20 bytes] for the staging key, never the prod one. ``` A masked span keeps its first four bytes and its length; in an address the marker drops its space, so the address stays one field of a `recall` line or -a `show` header. A private key is masked from its BEGIN line to its END -line, or to the end of the text when there is none. `show --redact` masks -each block of the turn whole before it splits it into lines, so a key's body -goes with its header, and it masks a tool's name too. It keeps the turn's line breaks, where -`recall` flattens a line before it masks, so it also finds a header that a -line break split, as the flattened line would have shown it. A masked address -is not one `show` can look up. +a `show` header. A private key is masked from its BEGIN line to its END line, +or to the end of the text when there is none. Both commands mask the text as +written, and also find a header that a line break split, as a flattened line +would show it; `recall` then flattens its line and cuts it at 160 characters. +`show --redact` masks each block of the turn whole before it splits it into +lines, so a key's body goes with its header, and it masks a tool's name too. +A masked address is not one `show` can look up: it refuses it, exit 2, and +`recall` prints the full address when it is not masking. `GITMEMORY_REDACT=1` makes it standing for both: unset, empty or `0` is off, and any other value is on. There is no flag that turns a standing setting off; for one call, run `GITMEMORY_REDACT=0 gitmemory recall ...`. -What it does not guarantee: +### What it does not guarantee - It masks only those sixteen shapes. Anything else prints as written: a bare AWS secret key, `PGPASSWORD=…`, a custom `GH_TOKEN=…`, `glpat-` and `npm_` @@ -178,6 +181,14 @@ What it does not guarantee: the dashboard, `derived/` and any file an agent opens are not masked by it. The environment setting is a floor, not a boundary: an agent with a shell can unset it, or read `raw/`. +- A private key whose BEGIN line and body are in two blocks of one turn, such + as two tool results, is masked to the end of the first block only; `show` + prints the body under the second. +- The store lowercases session ids, so an AWS key id, a Google API key or a + JWT used as a session id, by `capture --session-id` or as a transcript's + file name, prints in the address lowercased and unmasked: those detectors + match only the case the issuer writes. The manifest's `source_path` keeps + it as it was. - A mask shows the first four bytes and the length. - The SUSPECT patterns also mask some prose about configuration. @@ -309,7 +320,7 @@ Code's own environment, in `~/.claude/settings.json`: The skill then passes `--redact` to every `recall` and `show` it runs, so a CLI too old to know the flag stops with an error instead of printing unmasked, -and it does not read a turn from `raw/` by hand. +and it reads nothing under the store by hand. ## Derive key ideas and decisions diff --git a/docs/reviews/install-feedback-round.md b/docs/reviews/install-feedback-round.md index 6065143..d0517be 100644 --- a/docs/reviews/install-feedback-round.md +++ b/docs/reviews/install-feedback-round.md @@ -45,7 +45,7 @@ the one that adds this file. ## Status -**53 findings: 21 fixed, 6 declined, 26 refuted.** *Fixed* means something +**53 findings: 22 fixed, 6 declined, 25 refuted.** *Fixed* means something landed: code, test or text. *Declined* means the claim is true and not worth the change, with the reason written down. *Refuted* means the claim does not hold on the integrated branch. @@ -54,13 +54,15 @@ hold on the integrated branch. |---|---|---|---|---| | High | 5 | — | — | 5 | | Medium | 13 | 7 | — | 6 | -| Low | 35 | 14 | 6 | 15 | -| **Total** | **53** | **21** | **6** | **26** | +| Low | 35 | 15 | 6 | 14 | +| **Total** | **53** | **22** | **6** | **25** | All five Highs are artifacts of the commit seen alone: four from the split redaction pack, and one (the detector count, redact-docs-4) a correction of a -number `main`'s DESIGN.md had wrong. Fifteen of the 32 that changed nothing -rest on something outside the hunk Kimi was given. +number `main`'s DESIGN.md had wrong. Fifteen of the 31 that changed nothing +rest on something outside the hunk Kimi was given. service-3 was refuted here +and is counted fixed: Kimi raised it again in its final pass, and the +[final gate](#final-gate) took the reload. One change of behaviour came out of the round: two `gitmemory init`s run at once could lose a table silently (init-1). It carries a test that failed @@ -89,7 +91,7 @@ wording and pinned numbers. | init-4 | low | The refusal could print an empty reason | declined | Every skip a written table can reach logs with the prefix the filter wants. A fallback would only guard a future watcher bug | | service-1 | med | Other control characters make an invalid plist and unpasteable commands | refuted | plistlib refuses them before anything is written (exit 2). The legal ones pass `plutil -lint`. The printed commands hold no store path | | service-2 | low | USAGE says the unit keeps the watcher running across logins | fixed | Without linger, systemd stops user units at the last logout. Reworded to "at every login", with the `loginctl enable-linger` line for SSH-only machines | -| service-3 | low | The removal line lacks `daemon-reload` | refuted | `disable` reloads, and a stopped, disabled unit is garbage-collected, so `status` reports it gone. From systemd's documentation; not probed live, as there was no systemd to hand | +| service-3 | low | The removal line lacks `daemon-reload` | fixed | On the second report (final gate, F2). First refuted from systemd's documentation, not a probe: `disable` reloads, and a stopped, disabled unit is garbage-collected. The reload is harmless and conventional, so the printed removal now ends with one | | service-4 | low | `main()` may not map ValueError and RuntimeError to exit 2 | refuted | It does (lead) | | stale-1 | med | "N records uncollected for " gives the oldest's age to all N | fixed | Probed: a record from seconds earlier was reported 3 days old. Now `N hook record(s) uncollected, the oldest for `, in the code, the skill, USAGE and watching.md | | stale-2 | low | REPRODUCE's counts jump by 13 for 3 tests | fixed | As subagents-2. One fix serves both; see below | @@ -101,7 +103,7 @@ wording and pinned numbers. | show-4 | low | The skill's echo rule also skips a real turn that quotes an address | fixed | Probed: read literally, it skipped the user's newer reversal. Now the header's shape, marker then address then date, and a test pins the wording | | show-5 | low | The `## Echoes` heading may not exist | refuted | It is on `main` (lead) | | show-6 | low | `g5` and `@0007` are accepted | declined | Probed: same output as the canonical address, and the header prints the canonical one. The address never becomes a path | -| show-7 | low | The success path prints a credential-shaped session id unmasked | refuted | It does by design: masking is opt-in, and `--redact` masks it. The case-fold residue is already documented at the call site and in USAGE | +| show-7 | low | The success path prints a credential-shaped session id unmasked | refuted | It does by design: masking is opt-in, and `--redact` masks it, except an upper-case shape the store has lowercased. This row first said that residue was documented at the call site and in USAGE. It was only at the call site; the final gate found the gap (A5, A6, A9), and USAGE's list of what masking does not guarantee now names it | | redact-code-1 | high | No SKILL.md change ships | refuted | It was in the docs half of the split pack (lead) | | redact-code-2 | med | The linearity test asserts on wall clock | fixed | The page of headers is now counted: one END search, against 2000 under the mutant. The never-closing END line is one `re` search, which cannot be counted, so it stays timed, as the suite's other regex-cost tests are | | redact-code-3 | low | The `--redact` help says "set and not 0", but empty is off too | fixed | Now "set to anything but empty or 0" | @@ -161,3 +163,112 @@ At the state this file describes: | Collected | 1131 off and 1456 on, 328 conformance cases: the README and §5b of REPRODUCE.md | | `ruff check` | clean | | Mutation rows | **658**. The 23 rows that name a test this commit changed or re-pinned, the new one included: **23 of 23 caught**. Three of them, the README-count rows, bite only with the corpora present, which is what they check; they were run with them | + +## Final gate + +Two more passes read the branch at `b7e3521`, and one commit on top answers +both, the one that adds this section. + +- **Kimi K3, three packs:** the code half of the fix commit, its docs half, + and SKILL.md as it now reads whole. On the code half its verdict was ship, + with no findings. +- **An Opus gate of five lenses** over the whole branch: security, end to end, + docs, CI from a fresh clone, and simplicity. Two skeptics took each finding + and tried to refute it by default, with a probe wherever one could run. 18 + survived both, and 2 were refuted. + +**33 findings: 18 fixed, 4 documented, 4 declined, 1 pre-existing and out of +scope, 6 refuted.** *Documented* means the behaviour stays and the docs now +say so. Four are changes of behaviour, A1, A4, A7 and F2, each with a test that +failed first and a new negative control. + +### Kimi K3 + +| ID | Sev | Claim | Disposition | Evidence or reason | +|---|---|---|---|---| +| F1 | high | The fix commit re-ran only the rows whose test it changed, not the rows anchored in the hunks it changed | declined | Answered by a full pass over every row instead of a better selection. A missing anchor already fails: `test_every_mutation_row_anchors_exactly_once` on CI, and `SKIP` in a run. This commit also re-ran each row anchored in or beside a hunk it touched | +| F2 | med | service-3's refutation is wrong: after the `rm`, the unit stays loaded until a `daemon-reload` | fixed | service-3's second report, and its refutation rested on systemd's documentation, not a probe. The reload is harmless and conventional: the printed removal ends `; systemctl --user daemon-reload`, in the code, its test, a new row and watching.md | +| F3 | med | The CHANGELOG's list of what `service install` prints has no `loginctl enable-linger` line, which USAGE says it prints | fixed | It prints one on Linux, and the CHANGELOG now lists it | +| F4 | low | Pin redact-docs-8's measured examples | declined | Each is a shape `--redact` does not mask, so a pin would hold a weakness in place | +| F5 | low | A reflow left one overlong line in USAGE's `--redact` paragraph | fixed | Rewrapped, with A4's new order | +| S1 | med | Rerunning step 1 does not update the CLI | refuted | Live probe, uv 0.11.3, as version-2: rerunning the unpinned line replaced the install's commit with the remote's tip | +| S2 | med | A CLI from before `--version` does not print `the following arguments are required: cmd` | refuted | Measured on `c352a40`: it prints exactly that | +| S3 | med | A topic that starts with `-` is read as a flag, and the error reads as an older CLI | fixed | Probed: `recall "--bogus"` exits 2, `the following arguments are required: query`. The skill now puts `--` before such a topic, after any flags; `recall -k 30 --policy latest --redact -- "--bogus"` parses | +| S4 | low | Drop the `echo "${GITMEMORY_REDACT:-}"` check, since the CLI already defaults to masking | declined | The owner's decision: an explicit `--redact` makes a CLI too old for the flag fail loudly, where the default would print unmasked | +| S5 | low | `${CLAUDE_SESSION_ID}` may not exist | refuted | It is Claude Code's documented substitution in a skill, and the sentence predates this round | +| S6 | low | A masked address cannot occur | refuted | `capture --session-id` takes a credential-shaped id, and `recall --redact` masks it (A5) | +| S7 | low | Split the by-hand `raw/` lookup into numbered steps | declined | It is the fallback for a CLI older than `show`, and is left as it is | +| S8 | low | Rules are repeated: masking three times, the subagent cautions twice, and `-k 30` beside `-k 50` | fixed | Each is stated once: masking in the opening paragraph, what `subagent:` means in the field table, and one retry, `-k 50`, with its reason, that the order applies to the top `-k` matches only | + +### The Opus gate + +| ID | Lens | Sev | Claim | Disposition | Evidence or reason | +|---|---|---|---|---|---| +| A1 | security | med | `show --redact` stops, exit 2, partway through a turn that holds a lone surrogate | fixed | `mask` encoded with `surrogateescape`, which round-trips only U+DC80-U+DCFF; a JSON `\ud83d`, half an emoji, is any other. It encodes with `surrogatepass` now, as `records` hashes. Tested at the address and in a `-C` neighbour | +| A2 | security | med | The push gate misses a `\b`-anchored key right after a JSON `\n` or `\t` escape | pre-existing, out of scope | Reproduced. The HIGH table is `main`'s, and `push` does not send yet. Left for its own issue; the owner's unpushed egress branch adds a scan of the decoded text that targets this class | +| A3 | security | low | `show --redact` prints a private key's body when its BEGIN line is in an earlier block of the turn | documented | No span is carried across blocks. USAGE's list says a key split across two blocks is masked to the end of the first | +| A4 | security | low | `recall --redact` prints a SUSPECT value with a Unicode space in it, which `show --redact` and the gate flag | fixed | `str.split()` broke the value at a no-break space before the mask saw it. recall now masks as `show` does, with the mask's own collapse, then flattens and cuts | +| A5 | security | low | The address's case-fold residue is not in USAGE, so show-7's evidence is wrong | documented | A bullet in USAGE's list; show-7 corrected above | +| A6 | e2e | low | An AWS or Google key used as the session id prints unmasked, though USAGE, DESIGN and the CHANGELOG say the address is masked | documented | As A5. USAGE and the CHANGELOG now make the exception for a session id, and the DESIGN amendment no longer lists what is masked | +| A7 | e2e | low | `show`'s refusal of a masked address says `as recall prints it` about what recall printed | fixed | A one-line refusal of its own, `a masked address: ...`, still exit 2 | +| A8 | e2e | low | USAGE's `--redact` sample cannot come from the transcript every sample on the page comes from | fixed | It comes from a second transcript now, with its own session, and says so | +| A9 | docs | med | `--redact` leaves a credential-shaped session id unmasked, because ids are stored lowercased; show-7 cites USAGE text that does not exist | documented | As A5 and A6. The ruling was to document it and leave the code | +| A10 | docs | low | While masking, the skill still reads a hit's manifest, whose `source_path` is unmasked and in its original case | fixed | While masking, the skill reads nothing under the store by hand and goes by the address alone | +| A11 | docs | low | USAGE's `--redact` sample puts a different turn at another sample's address | fixed | As A8 | +| A12 | docs | low | The CHANGELOG and SECURITY say the skill passes the flag when the variable is "set and not `0`"; empty is off too | fixed | "set to anything but empty or `0`" in both, as redact-code-3 did for the help | +| A13 | docs | low | The CHANGELOG misses the reworded skip message and keeps the unbounded "about twice" | fixed | Both added | +| A14 | docs | low | The `--redact` caveats and the `settings.json` snippet are kept twice, and the copies differ; some passages run long | fixed | The caveat list and the snippet are in USAGE only; SECURITY and hook/README link them. watching.md's init paragraph and the DESIGN amendment are cut to what a reader acts on | +| A15 | docs | low | The docs index describes watching.md as configuration and flags only | fixed | The row names the service and the disk section | +| A16 | ci | low | REPRODUCE §5b's count is two low on an account named `ubuntu`, `runner` or `root` | fixed | Its commands set `HOME` as the suite's conftest does, and a paragraph says why | +| A17 | simplicity | low | init's join check cannot fire once the appended config parses | fixed | It refuses in the `except`; `have` is gone, and the new table's number is the length of the parsed `watch` array | +| A18 | simplicity | low | `uncollected`'s injected clock has no caller | fixed | Dropped. The `waiting and` the finding called optional is left: it changes nothing, and two rows anchor on it | +| R1 | docs | — | "Nothing leaves the machine" and "the only way anything leaves is `push`" contradict text this branch added | refuted | The tension was on `main` already, where SECURITY.md said the skill's quotes reach the provider unscanned; the added text puts that egress in the agent's path, not in anything gitmemory sends | +| R2 | simplicity | — | `test_recall_redact_keeps_a_subagent_turn_marked` repeats `test_recall_marks_a_subagent_turn` | refuted | By mutation: dropping the mark under `--redact` only fails this test and no other | + +### Decided while applying + +**A4 moves where recall masks.** The owner's decision put the mask after the +whitespace collapse, so that it would see a PEM header a line break split. But +`str.split()` also breaks at a no-break space, and that collapse was what cut +the value. recall now does as `show` does: the mask's own collapse, of ASCII +whitespace, finds the split header, and recall flattens and cuts after it. The +decision's purpose holds, and recall without `--redact` prints what it did. + +**What A1 costs.** In a block that also holds a mask, a lone surrogate prints +as replacement characters, where the same block unmasked prints `?`. + +**Rows.** One new row for each change of behaviour brings the index to +**662**. A4 moved three rows' anchors: the two on recall's head line, and +`show`'s collapse row, whose anchor recall's new line repeated. Every row +anchored in or beside a hunk this commit touched was run, the new ones +included: **45 of 45 caught** by their own test. + +| | | +|---|---| +| Full suite, corpora off | **1117 passed, 17 skipped**, 1 deselected: 1134, the README's number | +| Full suite, corpora on | **1458 passed, 1 skipped**, 1 deselected | +| Collected | 1134 off and 1459 on, 328 conformance cases: the README and §5b of REPRODUCE.md | +| `ruff check` | clean | + +### The full mutation pass + +F1 asked for every row, not a better choice of rows, so every row was run: +twice over, on two tips, each split six ways across fresh clones, with no +corpora. + +| Tip | Rows | Caught by their own test | Not | +|---|---|---|---| +| `b7e3521`, the Kimi fix commit | 658 | 642 | 16 MISSED, 0 SURVIVED, 0 SKIP | +| this commit | 662 | 646 | 15 of the same 16, MISSED or SURVIVED; 0 SKIP | + +None of the sixteen is this branch's. Each was run again on `main`, under the +same two setups the passes used, and each gave the verdict it gave here, row +for row. They are rows whose test cannot run offline here: the README and +conformance counts need the corpora, the dashboard's need the `serve` extra, +and the rest fail the same way on `main`. The sixteenth, the suite-wide home +isolation, is caught on this tip with the real `HOME`: it misses only when the +pass itself overrides `HOME`, so that verdict was the harness's, not the row's. +Every row this branch added or moved is caught by its own test. + +The sixteen are a finding about `main`, and they are left for a round of +their own. diff --git a/docs/watching.md b/docs/watching.md index 01bea0d..c1fe6ec 100644 --- a/docs/watching.md +++ b/docs/watching.md @@ -19,19 +19,14 @@ you, with the root [agents.md](agents.md) gives for that agent. That is still not a default: the root is written because you named the agent, and the watcher reads only the file. `init` appends to an existing `config.toml` and never changes a byte already there, since the file also holds any `[remote.*]` push -opt-in. It writes the result to a new file and renames it into place, so a -running watcher reads the old file or the new one, and the new one is mode -0600. It does nothing if the table is already there. It refuses, writing -nothing and exiting 1, a root another agent already watches, a root the watcher -would skip (listed below; `init` prints the watcher's own message), a file that -is not valid TOML or already has a `watch` of another shape, and a -`config.toml` that is a symlink or has another hard link, because the rename -would leave the other name holding the old file. A `config.toml` it cannot read -is an error, exit 2. Two `init`s run at the same moment cannot drop each -other's table: the later one sees the file change after it read it, leaves it -as the other one wrote it, and exits 1, so run it again. An editor saving the -file in that same instant is not covered. A running watcher picks the new table -up on its next pass. +opt-in, and it does nothing if the table is already there. When it cannot add +the table safely it writes nothing, prints why, and exits 1: a root another +agent watches or the watcher would skip (listed below); a file that is not +valid TOML, has a `watch` of another shape, is a symlink or has a second hard +link; or one another `init` changed while it ran, which a second run settles. +An editor saving the file at that moment is not covered. A `config.toml` it +cannot read is an error, exit 2. A running watcher picks the new table up on +its next pass. By hand, or for any other agent or root, `$GITMEMORY_HOME/config.toml`: @@ -195,7 +190,7 @@ loginctl enable-linger # keep it running after you log ou systemctl --user status gitmemory.service # check it journalctl --user -u gitmemory.service -f # its log systemctl --user restart gitmemory.service # after upgrading gitmemory -systemctl --user disable --now gitmemory.service; rm ~/.config/systemd/user/gitmemory.service +systemctl --user disable --now gitmemory.service; rm ~/.config/systemd/user/gitmemory.service; systemctl --user daemon-reload ``` On macOS: @@ -223,6 +218,8 @@ launchctl bootout gui/$(id -u)/io.github.doronp.gitmemory; rm ~/Library/LaunchAg - **Removing it stops it, then removes the file whether or not that worked.** A file written and never started has nothing to stop: `launchctl bootout` says `No such process` and exits 3, and the `rm` after the `;` runs anyway. + On Linux a `daemon-reload` after the `rm` has systemd drop the unit it had + loaded. ## Check it diff --git a/hook/README.md b/hook/README.md index cc64d3b..7f7d8b3 100644 --- a/hook/README.md +++ b/hook/README.md @@ -102,21 +102,13 @@ there too, and be absolute, as above. What the skill reads from the store goes to the model provider with the rest of the conversation, and no redaction gate stands in that path ([SECURITY.md](../SECURITY.md#if-a-credential-lands-in-the-store)). To have `recall` and `show` mask the credential shapes the gate knows in what -they print, set `GITMEMORY_REDACT` in that environment, in -`~/.claude/settings.json`: - -```json -{ - "env": { - "GITMEMORY_REDACT": "1" - } -} -``` - -The skill then passes `--redact` to every `recall` and `show` it runs, and -does not read `raw/`. It masks; it is not a gate, and -[USAGE.md](../docs/USAGE.md#masking-credentials-in-what-it-prints) lists what -it does not cover. +they print, set `GITMEMORY_REDACT=1` in that environment, in +`~/.claude/settings.json` as +[USAGE.md](../docs/USAGE.md#put-it-back-in-front-of-the-agent) shows. The +skill then passes `--redact` to every `recall` and `show` it runs. It masks; it +is not a gate, and +[USAGE.md](../docs/USAGE.md#what-it-does-not-guarantee) lists what it does not +cover. An install made from 0.1.0 does not have the skill. Claude Code keeps a plugin's cached copy until its version changes, and the skill was added after diff --git a/skills/recall/SKILL.md b/skills/recall/SKILL.md index 1737978..203ec91 100644 --- a/skills/recall/SKILL.md +++ b/skills/recall/SKILL.md @@ -17,8 +17,9 @@ credential shapes that gate knows in what `recall` and `show` print, as `sk-a…[103 bytes]`. Pass it to both whenever `echo "${GITMEMORY_REDACT:-}"` prints anything but nothing or `0`, and whenever the user asks you to keep credentials out of the conversation. Masking covers only what those two print, -so while you are masking, do not read a turn from `raw/` by hand, and if -either command rejects the flag, do not run it without it. +so while you are masking, read nothing under the store by hand: `raw/`, the +manifests under `sessions/` and the index all hold it unmasked. If either +command rejects the flag, do not run it without it. The store is `$GITMEMORY_HOME`, or `~/.gitmemory` when that is unset; the CLI reads it, so leave it as this session has it. If the user keeps the store @@ -56,6 +57,10 @@ gitmemory recall "" # 10 lines by default; -k N for more or fe gitmemory recall --redact "" # the same, masked ``` +If the topic starts with `-`, put `--` before it, after any flags, as in +`gitmemory recall --redact -- ""`. Otherwise `recall` reads it as a +flag, and the error that follows says nothing about the CLI's age. + One line per turn: what the user said first, newest first, then everything else newest first: @@ -70,7 +75,7 @@ else newest first: | `claude-code//g00` | Agent, session and generation | | `@346` | Byte offset of the turn (its JSONL line) in that generation's raw bytes | | `assistant/text` | Role and block kind | -| `subagent:user/text` | A subagent's turn, marked `subagent:` before any role/kind (`subagent:assistant/text` too). Its user/text is the instructions the parent agent gave it, not the user's words | +| `subagent:user/text` | A subagent's turn, marked `subagent:` before any role/kind (`subagent:assistant/text` too, and `subagent:user` in a `show` header). Its user text is the instructions the parent agent gave it, not the user's words | | the rest | The block's first 160 characters, whitespace collapsed | `no matches` on stderr, exit 0, is an answer. Try once more with other words @@ -81,7 +86,9 @@ If two lines disagree, the later date is usually the current decision: say so, and quote both. An agent's newer proposal does not overrule the user's older rule, and neither does a subagent's line; a newer user line does. If the user has set `GITMEMORY_POLICY` or asks for another order, leave it; `--policy` is -theirs to choose. +theirs to choose. The order is applied to the top `-k` matches only, so if +echoes (below) or `subagent:` lines crowd out the rest, search again with +`-k 50`. ## Echoes @@ -100,15 +107,15 @@ recalls match the same words. Skip these hits; they are not sources: header, not the echo's own address; - an earlier answer to the same question, and the lines it quoted. -If they crowd out the rest, search again with `-k 30`. Quote the original turn. +Quote the original turn. ## When it fails - A `gitmemory` command exits 2, what it prints starts `usage: gitmemory`, and its last line says `invalid choice`, or `unrecognized arguments` naming a - flag (a word starting with `-`): the CLI is older than this skill and lacks - that subcommand or flag. Do not run it again without the flag or under - another name. Tell the user which one is missing, and that running step 1 + flag (a word starting with `-`) that is not your topic: the CLI is older + than this skill and lacks that subcommand or flag. Do not run it again + without the flag or under another name. Tell the user which one is missing, and that running step 1 below again updates the CLI, a pinned install included. If `unrecognized arguments` names only plain words, the CLI is not the problem: the topic went in unquoted. Put it in one pair of quotes and search again. @@ -153,40 +160,35 @@ If they crowd out the rest, search again with `-k 30`. Quote the original turn. `gitmemory show --context 2
` for the two turns either side too, and `gitmemory show --redact
` if you passed `--redact` to `recall`. A masked address (one with `…[` in it) is not one `show` can - look up, and its manifest (below) cannot be found from it: say the line - stops there and give the address as printed. Do not list - `/sessions/` to find it: the names there are the session ids, - unmasked. A header starts `>` for the turn at the address and `-` for a - neighbour, and only headers start in column 0; quote the text under one - without its indent. - `subagent:user` in a header is a brief from the agent that started the - subagent, not the user's words. If the CLI is older than `show` (exit 2, - `invalid choice: 'show'`), tell the user as [When it fails](#when-it-fails) - says. If you are masking, stop there: `raw/` is unmasked, so say the line - stops at 160 characters and give the address. Otherwise read the turn by - hand: it is the JSON line that starts `offset - start` bytes into the - segment under `/raw///g/` whose name, the byte - range `start-end` with the end excluded, holds the offset, and it may run - on into the next segment. -- This session is `${CLAUDE_SESSION_ID}`. A hit's manifest, - `/sessions///g.json`, has its transcript's - `source_path`: `/.jsonl`, or for a subagent (a session + look up: say the line stops there and give the address as printed. A + header starts `>` for the turn at the address and `-` for a neighbour, and + only headers start in column 0; quote the text under one without its + indent. If the CLI is older than `show` (exit 2, `invalid choice: 'show'`), + tell the user as [When it fails](#when-it-fails) says. If you are masking, + say the line stops at 160 characters and give the address. Otherwise read + the turn by hand: it is the JSON line that starts `offset - start` bytes + into the segment under `/raw///g/` whose name, + the byte range `start-end` with the end excluded, holds the offset, and it + may run on into the next segment. +- This session is `${CLAUDE_SESSION_ID}`. Unless you are masking, a hit's + manifest, `/sessions///g.json`, has its + transcript's `source_path`: `/.jsonl`, or for a subagent (a session named `agent--...`) `//subagents/.../agent-.jsonl`. Both belong to ``. `` is the working directory, with `/` - and other punctuation turned into `-`. Say so when a hit is from another - agent, another session or another project. + and other punctuation turned into `-`. While masking, go by the address + alone. Say so when a hit is from another agent, another session or another + project. - Say so, too, when a quote is a `subagent:` line, and never give one as what the user said. When it and a main-session line say the same thing, quote the - main-session line. If `subagent:` lines crowd out the rest, search again - with `-k 50`: the order is applied to the top `-k` matches only. + main-session line. - The store holds whatever was typed or pasted into a watched session, tool output included. Quote what the question needs and no more. ## Read-only Run `gitmemory --version`, `gitmemory recall`, `gitmemory show` and -`gitmemory index`, and read files under ``, but not `raw/` while you -are masking. `index` writes only the derived index under `/index/`. Do +`gitmemory index`, and, when you are not masking, read files under +``. `index` writes only the derived index under `/index/`. Do not run `gitmemory init`, `gitmemory capture`, `gitmemory watch`, `gitmemory service`, `gitmemory derive`, `gitmemory push` or `gitmemory dashboard`, and never `gitmemory dashboard --expose`. Do not edit diff --git a/src/gitmemory/__main__.py b/src/gitmemory/__main__.py index 1621396..90749f4 100644 --- a/src/gitmemory/__main__.py +++ b/src/gitmemory/__main__.py @@ -104,7 +104,6 @@ def _init(args) -> int: before = tomllib.loads(old.decode("utf-8")) except ValueError as exc: # `TOMLDecodeError`, and `UnicodeDecodeError` on a bad byte return _refuse(f"{path} is not valid TOML ({exc}); fix it first") - have = before.get("watch", []) table = ( f"# added by `gitmemory init {args.agent}`\n" f'[[watch]]\nagent = "{args.agent}"\nroots = ["{root}"]\n' @@ -113,11 +112,10 @@ def _init(args) -> int: new = old + sep + table.encode() # Re-parsed whole, because appending is not always joining: after a plain # `[watch]` or an inline `watch = [...]` the result does not parse at all. + # When it parses, the header added one table, last in the `watch` array. try: after = tomllib.loads(new.decode("utf-8")) except ValueError: - after = {} - if not isinstance(have, list) or len(after.get("watch") or []) != len(have) + 1: return _refuse(f"{path} has a `watch` a [[watch]] table cannot join; add it by hand") for w in daemon.watches_from(before, path, home, lambda _m: None): if real in w.roots: @@ -134,7 +132,7 @@ def _init(args) -> int: w.agent == args.agent and real in w.roots for w in daemon.watches_from(after, path, home, notes.append) ): - mine = f"{path} [[watch]] #{len(have) + 1}:" + mine = f"{path} [[watch]] #{len(after['watch'])}:" return _refuse("; ".join(n for n in notes if n.startswith(mine))) # Two inits at once both read the same bytes, and the later rename dropped # the other's table while both said "added". Each holds this across a second @@ -477,13 +475,16 @@ def _recall(args) -> int: shown = list(resolve(hits)) except Exception as exc: # a custom policy is the user's code: report it, no traceback raise ValueError(f"policy {args.policy!r} failed: {exc!r}") from exc - # After the policy, so a custom one cannot route around it. After the - # collapse, which can join a PEM header a newline split, and before the cut: + # After the policy, so a custom one cannot route around it. Before the cut: # a key that straddles character 160 leaves a prefix no detector matches. - # `graph._label` measured both. The address too: a session id can be a key. - shown_as = redact.mask if args.redact else str + # Before the collapse too, with the mask's own, as `show`'s: it joins a PEM + # header a newline split, where `str.split()` also broke on a no-break + # space, which the patterns read as part of a token, and cut the value under + # its floor. `graph._label` measured the cut and the split header. The + # address too: a session id can be a key. + shown_as = functools.partial(redact.mask, collapse=True) if args.redact else str for h in shown: - head = shown_as(" ".join(h.text.split()))[:160] + head = " ".join(shown_as(h.text).split())[:160] # The date on every line, whatever the policy: it is what lets the # reader see that two disagreeing turns are three weeks apart. date = _date(h) @@ -537,6 +538,12 @@ def _show(args) -> int: except ValueError: m = None if not m: + # The marker fits `_ADDRESS` and fails `identity`: the user copied it + # whole, from a recall that was masking, and did not mistype it. + if "…[" in args.address: + hint = "recall prints one show can look up when it is not masking" + print(f"a masked address: {redact.safe_path(args.address)!r}; {hint}", file=sys.stderr) + return 2 want = "expected //gNN@, as recall prints it" print(f"not an address: {redact.safe_path(args.address)!r}; {want}", file=sys.stderr) return 2 @@ -580,8 +587,8 @@ def _show(args) -> int: # # `--redact` masks as recall's does, on each block whole, before the split: # a private key's body is the lines under its header. These lines keep - # their breaks, where recall's are flattened first, so the mask is asked to - # join `-----BEGIN RSA` and `PRIVATE KEY-----` across one itself. + # their breaks, so only the mask's own collapse joins `-----BEGIN RSA` and + # `PRIVATE KEY-----` across one. shown_as = functools.partial(redact.mask, collapse=True) if args.redact else str lo = max(0, at - args.context) for i, t in enumerate(turns[lo : at + args.context + 1], start=lo): diff --git a/src/gitmemory/daemon.py b/src/gitmemory/daemon.py index 3815ae8..dc6c56e 100644 --- a/src/gitmemory/daemon.py +++ b/src/gitmemory/daemon.py @@ -495,7 +495,7 @@ def drain_spool( return wanted, tick -def uncollected(home: str, now: float | None = None) -> tuple[int, float]: +def uncollected(home: str) -> tuple[int, float]: """`(complete records in the spool, age of the oldest in seconds)`. `drain_spool`'s rule for what a record is: a dot-name is a hook still @@ -504,7 +504,7 @@ def uncollected(home: str, now: float | None = None) -> tuple[int, float]: mtime (hook/README.md). Read-only and total, because `recall` asks: anything it cannot list is `(0, 0.0)`. """ - now = time.time() if now is None else now + now = time.time() ages = [] try: with os.scandir(os.path.join(home, SPOOL)) as entries: diff --git a/src/gitmemory/redact.py b/src/gitmemory/redact.py index dbcd886..2489e49 100644 --- a/src/gitmemory/redact.py +++ b/src/gitmemory/redact.py @@ -210,14 +210,18 @@ def mask(text: str, *, collapse: bool = False) -> str: are the text's. The first stays, because a spelling can sit inside a match (`password=abc\\x01defghijklmn`) that the blanked copy cuts short. - `collapse` is for a caller that prints the line breaks. `recall` flattens - each line before it masks, and `-----BEGIN RSA\\nPRIVATE KEY-----` is a - header only once flattened, so a caller that keeps the break asks for a - third scan, of the blanked copy with every whitespace run one space. Its END + `collapse` is for a caller that masks before it flattens, or never does. + `-----BEGIN RSA\\nPRIVATE KEY-----` is a header only once flattened, so such + a caller asks for a third scan, of the blanked copy with every whitespace + run one space. `recall` and `show` both do: flattening first with + `str.split()` broke on a no-break space, which is a token byte here. Its END line is still looked for as written, so one a break split is not found, and the mask runs to the end of the text. """ - data = text.encode("utf-8", "surrogateescape") + # `surrogatepass`, as `records` hashes: `show` masks the parsed text, where + # a JSON `\ud83d` is a lone surrogate, and `surrogateescape` raised on any + # outside U+DC80-U+DCFF. The patterns are bytes; the result decodes lossily. + data = text.encode("utf-8", "surrogatepass") plain = _SPELLED.sub(lambda m: b" " * len(m[0]), data) found = scan_bytes(data) + (scan_bytes(plain) if plain != data else []) if collapse: diff --git a/src/gitmemory/service.py b/src/gitmemory/service.py index 91416e8..d68592c 100644 --- a/src/gitmemory/service.py +++ b/src/gitmemory/service.py @@ -108,8 +108,10 @@ def render(platform: str, home: str) -> tuple[str, bytes, list[tuple[str, str]]] ("Check it", f"{ctl} status {UNIT}"), ("Its log", f"journalctl --user -u {UNIT} -f"), ("After upgrading gitmemory", f"{ctl} restart {UNIT}"), - # `;`: the file goes even when the manager cannot be reached. - ("Remove it", f"{ctl} disable --now {UNIT}; rm {shlex.quote(target)}"), + # `;`: the file goes even when the manager cannot be reached. The + # reload after it is so the manager drops the unit it had loaded. + ("Remove it", + f"{ctl} disable --now {UNIT}; rm {shlex.quote(target)}; {ctl} daemon-reload"), ] return target, body, steps target = os.path.expanduser(f"~/Library/LaunchAgents/{LABEL}.plist") diff --git a/tests/mutate_index.py b/tests/mutate_index.py index 87f576a..5aa1498 100644 --- a/tests/mutate_index.py +++ b/tests/mutate_index.py @@ -3906,17 +3906,27 @@ ( "recall --redact masks the head after the 160-character cut", "__main__.py", - ' head = shown_as(" ".join(h.text.split()))[:160]', + ' head = " ".join(shown_as(h.text).split())[:160]', ' head = shown_as(" ".join(h.text.split())[:160])', "test_recall_redact_masks_a_key_the_160_character_cut_would_have_halved", ), ( + # The mask joins a header a line break split itself, as show's does. "recall --redact masks the head before the collapse assembles a header", "__main__.py", - ' head = shown_as(" ".join(h.text.split()))[:160]', - ' head = " ".join(shown_as(h.text).split())[:160]', + " shown_as = functools.partial(redact.mask, collapse=True) if args.redact else str\n" + " for h in shown:", + " shown_as = redact.mask if args.redact else str\n for h in shown:", "test_recall_redact_masks_a_header_that_only_the_collapse_assembles", ), + ( + # `str.split()` breaks on a no-break space; the bytes patterns do not. + "recall --redact collapses a Unicode space out of a value before it masks", + "__main__.py", + ' head = " ".join(shown_as(h.text).split())[:160]', + ' head = shown_as(" ".join(h.text.split()))[:160]', + "test_recall_redact_masks_a_value_with_a_unicode_space_in_it", + ), ( "recall --redact masks a private key's header and prints its body", "redact.py", @@ -4008,13 +4018,30 @@ "test_show_redact_masks_a_whole_turn_and_the_default_prints_it", ), ( - # recall flattens a turn before it masks; show prints the line breaks. + # show prints the line breaks, so only the mask's own collapse joins one. "show --redact masks without joining a key header a line break split", "__main__.py", - "functools.partial(redact.mask, collapse=True)", - "functools.partial(redact.mask, collapse=False)", + "functools.partial(redact.mask, collapse=True) if args.redact else str\n" + " lo = max(0, at - args.context)", + "functools.partial(redact.mask, collapse=False) if args.redact else str\n" + " lo = max(0, at - args.context)", "test_show_redact_joins_a_key_header_a_line_break_split", ), + ( + # `surrogateescape` round-trips U+DC80-U+DCFF only; `\ud83d` raised. + "show --redact stops on a lone surrogate the mask cannot encode", + "redact.py", + ' data = text.encode("utf-8", "surrogatepass")', + ' data = text.encode("utf-8", "surrogateescape")', + "test_show_redact_prints_a_turn_that_holds_a_lone_surrogate", + ), + ( + "show refuses a masked address as one recall never printed", + "__main__.py", + ' if "…[" in args.address:', + " if False:", + "test_a_masked_address_is_refused_as_masked_not_as_mistyped", + ), ( # The copy is shorter by every run it narrowed, so its offsets are not # the text's: unmapped, the mask eats the bytes before the key. @@ -4769,7 +4796,7 @@ # anchor appear twice and the harness would skip itself. [E7] "the docstring's row count drifts away from the index again", "tests/mutate_index.py", - "a full pass is 6" + "58 mutants", + "a full pass is 6" + "62 mutants", "a full pass is 14" + "6 mutants", "test_the_row_count_in_the_docstring_is_the_row_count", ), @@ -5873,6 +5900,13 @@ 'f"launchctl bootout {job} && rm {shlex.quote(target)}"', "test_the_printed_removal_works_when_the_job_was_never_started", ), + ( + "the printed systemd removal leaves the manager holding the unit it removed", + "service.py", + "; rm {shlex.quote(target)}; {ctl} daemon-reload", + "; rm {shlex.quote(target)}", + "test_service_install_writes_a_systemd_user_unit_and_starts_nothing", + ), # --- a captured amount said for a person --- ( "the watch line goes back to a bare byte count", @@ -6048,7 +6082,7 @@ def _select(test: str) -> list[str]: def main() -> int: """Run every mutant, or only those whose name contains an argument. - The filter exists because a full pass is 658 mutants × two suite runs — + The filter exists because a full pass is 662 mutants × two suite runs — long enough that adding one row and checking it used to mean either waiting for every other row or trusting the new one untested. diff --git a/tests/test_index.py b/tests/test_index.py index 25ab5cc..8cc30a4 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -1778,8 +1778,8 @@ def test_recall_redact_masks_a_private_key_body_and_not_only_its_header(home, sr def test_recall_redact_masks_a_header_that_only_the_collapse_assembles(home, src, capsys): """`-----BEGIN RSA\\nPRIVATE KEY-----` scans clean as written and is a header once the line is flattened, and the flattened line is what recall - prints. So the mask runs after the collapse, as `graph._label` found it - had to. [E7b L3-F1]""" + prints. So the mask scans a flattened copy too, as `graph._label` found it + had to, and as `show`'s does. [E7b L3-F1]""" split = PEM.replace("RSA PRIVATE", "RSA\nPRIVATE", 1) assert redact.mask(split) == split, "premise: nothing matches before the collapse" built(home, src, [user("u1", "pemcase " + split)]).close() @@ -1788,6 +1788,21 @@ def test_recall_redact_masks_a_header_that_only_the_collapse_assembles(home, src assert PEM_BODY[:8] not in out and "PRIVATE KEY" not in out, out +def test_recall_redact_masks_a_value_with_a_unicode_space_in_it(home, src, capsys): + """`str.split()` breaks on a no-break space, and the gate's bytes patterns + read one as part of the token. Collapsing first cut the value below the + twelve-character floor, so `recall --redact` printed what `show --redact` + and the push gate both flag. The collapse that joins a split header is the + mask's own, on ASCII whitespace, as `show`'s is.""" + secret = "pass" + "word=abc defghijklmnop" + flat = " ".join(secret.split()) + assert redact.mask(flat) == flat and redact.mask(secret) != secret, "premise" + built(home, src, [user("u1", f"gnu note {secret} end")]).close() + capsys.readouterr() + out = _recall_out(home, capsys, "gnu", "--redact") + assert "defghijklmnop" not in out and "gnu note pass…[" in out, out + + def test_recall_redact_masks_a_session_id_that_is_itself_a_credential(home, src, capsys): """The address is printed too, and `_SAFE_RE` accepts a token as a session id. Lowercase, because the store case-folds ids. diff --git a/tests/test_service.py b/tests/test_service.py index 6de417d..f60026d 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -122,7 +122,8 @@ def test_service_install_writes_a_systemd_user_unit_and_starts_nothing( ctl = "systemctl --user" assert f"{ctl} daemon-reload && {ctl} enable --now gitmemory.service" in out assert "loginctl enable-linger" in out - assert f"{ctl} disable --now gitmemory.service; rm {shlex.quote(str(path))}" in out + removal = f"{ctl} disable --now gitmemory.service; rm {shlex.quote(str(path))}" + assert f" {removal}; {ctl} daemon-reload\n" in out def test_service_install_writes_a_launch_agent_and_starts_nothing( @@ -307,7 +308,8 @@ def test_a_unit_lands_under_dot_config_without_an_absolute_xdg_config_home( assert _words(_unit(path)["ExecStart"], dollars=True) == _expected(tmp_path / "store") assert not (tmp_path / "rel").exists() out = capsys.readouterr().out - assert out.startswith(f"wrote {path}\n") and f"; rm {shlex.quote(path)}\n" in out + removal = f"; rm {shlex.quote(path)}; systemctl --user daemon-reload\n" + assert out.startswith(f"wrote {path}\n") and removal in out @pytest.mark.parametrize("platform", ["linux", "darwin"]) diff --git a/tests/test_show.py b/tests/test_show.py index 55db2ca..fc47ac8 100644 --- a/tests/test_show.py +++ b/tests/test_show.py @@ -148,6 +148,24 @@ def test_a_malformed_or_escaping_address_is_not_an_address(home, src, capsys, ad assert err.count("\n") == 1 +def test_a_masked_address_is_refused_as_masked_not_as_mistyped(home, src, capsys): + """`recall --redact` masks a credential-shaped session id in the address, + and a masked address is not one `show` can look up. The refusal said + `expected //gNN@, as recall prints it` about the + string recall had just printed, so it now says which it is. Still exit 2.""" + ghp = "ghp_" + "a" * 36 # lowercase, because the store case-folds a session id + write(src, [user("u1", "walrus note")]) + store.capture(src, "claude-code", ghp, home=home) + assert main(["--home", home, "index"]) == 0 + capsys.readouterr() + assert main(["--home", home, "recall", "walrus", "--redact"]) == 0 + address = capsys.readouterr().out.split()[2] + assert address == "claude-code/ghp_…[40bytes]/g00@0", address + code, out, err = show(home, capsys, address) + assert (code, out) == (2, "") and err.count("\n") == 1, err + assert err.startswith("a masked address: ") and "as recall prints it" not in err, err + + def test_an_address_the_store_does_not_hold_is_an_answer_exit_1(home, src, capsys): write(src, [user("u1", "aardvark")]) store.capture(src, "claude-code", "sess", home=home) @@ -292,3 +310,24 @@ def test_show_redact_joins_a_key_header_a_line_break_split(home, src, capsys): f" ----…[{len(split)} bytes]", " last words", ], out + + +def test_show_redact_prints_a_turn_that_holds_a_lone_surrogate(home, src, capsys): + """Half an emoji is what `JSON.stringify` writes for a tool result cut + mid-pair, `\\ud83d`, and the parse keeps it as a lone surrogate. `show` + printed it as `?`. `show --redact` printed the headers and then stopped, + exit 2, because the mask encoded with `surrogateescape`, which round-trips + only U+DC80-U+DCFF. At the address and in a `-C` neighbour alike, and the + key beside it is still masked.""" + result = {"type": "tool_result", "tool_use_id": "t", "content": f"build ok \ud83d {AKIA}"} + first, second = user("u1", "the first turn"), user("u2", "") + second["message"]["content"] = [result] + write(src, [first, second]) + store.capture(src, "claude-code", "sess", home=home) + at = f"claude-code/sess/g00@{size(first)}" + code, out, err = show(home, capsys, at) + assert (code, err) == (0, "") and f" build ok ? {AKIA}\n" in out, out + for argv in ([at], ["-C", "1", "claude-code/sess/g00@0"]): + code, out, err = show(home, capsys, *argv, "--redact") + assert (code, err) == (0, ""), (argv, err) + assert " build ok " in out and out.endswith(" AKIA…[20 bytes]\n"), (argv, out)