From 554519d683d7a2c4aa68cedff275c2e705ac282b Mon Sep 17 00:00:00 2001 From: Ansu <230804140223@stu.nepu.edu.cn> Date: Sun, 18 Oct 2026 14:01:12 +0800 Subject: [PATCH 1/2] fix(sleep): stop a re-run hook template from filling the session cap (#319) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A local hook that summarises the previous session on a timer writes one single-turn transcript per run. Its only user prompt is the hook's own template: identical leading text, a fresh log appended. Neither the engine replay markers nor the sub-agent markers match it, and in the desktop app those runs carry `entrypoint: "claude-desktop"` like interactive sessions, so it was harvested as a user task. Because harvest walks newest first and stops at `--max-sessions`, an hourly hook filled the cap and pushed real sessions out of the night entirely. `harvest()` now drops single-turn sessions whose first 120 characters of prompt recur across at least three sessions of the same harvest window, before the cap is applied. The whole window is scanned first: a template can only be recognized by comparing it with the rest of the window, and the repeats must not be counted against the cap they used to fill. A session with a follow-up user turn is never matched, and neither is a prompt shorter than the prefix — a real one-shot request repeats entirely instead of sharing a long boilerplate header. `SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS` tunes the threshold (default 3) and `0` disables the filter. `harvest()` takes an optional `stats` out-parameter that `harvest` and the `run` / `dry-run` progress line report, so a thin night says why: `harvest done: sessions=120 (skipped: 297 machine-generated, 2 past the session cap)`. The harvest filters that already existed were undocumented — no .md file named `SKILLOPT_SLEEP_AGENT_MARKERS`, `SKILLOPT_SLEEP_NEG_FEEDBACK` or `SKILLOPT_SLEEP_POS_FEEDBACK`. They are now documented in the sleep guide and the Claude Code plugin README, with the hourly-summary-hook example. --- docs/sleep/README.md | 41 ++++ plugins/claude-code/README.md | 34 +++ skillopt_sleep/__main__.py | 7 +- skillopt_sleep/cycle.py | 10 +- skillopt_sleep/harvest.py | 135 +++++++++++- skillopt_sleep/harvest_sources.py | 15 +- tests/test_harvest_machine_repeats.py | 290 ++++++++++++++++++++++++++ 7 files changed, 524 insertions(+), 8 deletions(-) create mode 100644 tests/test_harvest_machine_repeats.py diff --git a/docs/sleep/README.md b/docs/sleep/README.md index 4142da1da..1cc56bfb0 100644 --- a/docs/sleep/README.md +++ b/docs/sleep/README.md @@ -201,6 +201,47 @@ An existing file that cannot be decoded, parsed, or read stops the command with a configuration error and exit code 2; it does not discard your settings and run with default budgets. Explicit CLI flags still override loaded values. +### Harvest filters (environment) + +Harvest keeps what you typed and drops what a machine wrote. Four environment +variables extend the built-in filters. They apply to the Claude Code source and +are read when the process starts, so set them in the shell or in whatever runs +the night: + +| Variable | Effect | +|---|---| +| `SKILLOPT_SLEEP_AGENT_MARKERS` | Comma-separated markers. A session whose **first** user prompt contains one is dropped entire — for prompts written by another tool's sub-agent (memory observers, critic agents, hook-driven runs). | +| `SKILLOPT_SLEEP_NEG_FEEDBACK` | Extra comma-separated phrases, in any language, that count as you disapproving of the previous answer. | +| `SKILLOPT_SLEEP_POS_FEEDBACK` | The same, for approval. | +| `SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS` | Sessions a prompt prefix must recur in before the repeats are treated as a hook re-running itself (default `3`; `0` turns the filter off). | + +The first three cover a tool whose prompt template you can quote, and the +filter they feed is exact: it checks the session's first user prompt against +each marker. A nightly hook that summarises the previous session — the shape +that fills a harvest window without your ever typing anything — is the case +they exist for: + +```bash +# the hook's own template, as its first line appears in the transcript +export SKILLOPT_SLEEP_AGENT_MARKERS="Below is a conversation log from a coding session" +``` + +`SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS` covers the templates you cannot quote. A +single-turn session that shares its first 120 characters of prompt with +`N` others in the same harvest window is a hook re-running one job; the filter +drops those repeats *before* `--max-sessions` is applied, so they cannot +consume the cap the way they did in #319. A session with a follow-up user turn +is never matched (it is interactive by definition), and neither is a prompt +shorter than the prefix — a real one-shot request repeats entirely rather than +sharing a long boilerplate header. + +`harvest` and the progress output of `run` / `dry-run` report what was dropped, +so a night that looks empty is explainable: + +```text +[sleep] 120 sessions -> 40 tasks (skipped: 297 machine-generated, 2 past the session cap) +``` + ### VS Code GitHub Copilot Chat Use `--source copilot` to harvest local VS Code GitHub Copilot Chat sessions. diff --git a/plugins/claude-code/README.md b/plugins/claude-code/README.md index 9d5f6b954..02b15cdc6 100644 --- a/plugins/claude-code/README.md +++ b/plugins/claude-code/README.md @@ -96,6 +96,40 @@ session data and provider policy before using a real backend on a sensitive project; the [shared integration guide](../README.md#data-boundary) describes a reviewed task-file workflow. +### Sessions you did not type + +Harvest keeps your prompts and drops machine-written transcripts. Three +environment variables extend what counts as machine-written, for the sessions +Claude Code gives no other tell: + +| Variable | Effect | +|---|---| +| `SKILLOPT_SLEEP_AGENT_MARKERS` | Comma-separated markers; a session whose **first** prompt contains one is dropped entire. | +| `SKILLOPT_SLEEP_NEG_FEEDBACK` / `SKILLOPT_SLEEP_POS_FEEDBACK` | Extra comma-separated phrases, in any language, that count as you rejecting or approving the previous answer. | + +The case they exist for is a local hook that summarises the previous session on +a timer. Its runs are single-turn, they are frequent enough to fill +`--max-sessions` and starve the night of your real sessions, and in the desktop +app they carry `entrypoint: "claude-desktop"` like interactive ones — so neither +`entrypoint` nor a replay marker identifies them. Quote the hook's first line: + +```bash +export SKILLOPT_SLEEP_AGENT_MARKERS="Below is a conversation log from a coding session" +``` + +A template you cannot quote is covered without configuration: a single-turn +session sharing its first 120 characters of prompt with three others in the same +harvest window is a hook re-running one job, and it is dropped before the +session cap is applied. Tune that with `SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS` +(default `3`, `0` disables). A session with a follow-up user turn is never +matched, and neither is a prompt shorter than the prefix — your own one-shot +requests repeat entirely rather than sharing a long boilerplate header. + +`/skillopt-sleep harvest` and the progress line of a `run` / `dry-run` report +what was dropped, so a thin night is explainable: +`harvest done: sessions=120 (skipped: 297 machine-generated, 2 past the session cap)`. +Full reference: [the sleep guide](../../docs/sleep/README.md#harvest-filters-environment). + ### Handoff mode (session answers the model calls) `--backend handoff` runs the cycle without any model subprocess: the engine diff --git a/skillopt_sleep/__main__.py b/skillopt_sleep/__main__.py index eebe5d457..696b15ecc 100644 --- a/skillopt_sleep/__main__.py +++ b/skillopt_sleep/__main__.py @@ -35,6 +35,7 @@ from skillopt_sleep.backend import CursorBackendError from skillopt_sleep.config import ConfigError, load_config from skillopt_sleep.cycle import _one_line_display_text, run_sleep_cycle +from skillopt_sleep.harvest import format_harvest_note from skillopt_sleep.harvest_sources import harvest_for_config from skillopt_sleep.mine import mine from skillopt_sleep.staging import ( @@ -788,13 +789,14 @@ def fail(code: int, kind: str, message: str, **extra: Any) -> int: def cmd_harvest(args) -> int: cfg = _cfg_from_args(args) session_limit = cfg.get("max_sessions_per_night", 0) or cfg.get("max_tasks_per_night", 40) * 3 + harvest_stats: Dict[str, int] = {} target_skill_path = cfg.managed_skill_path() if cfg.get("target_skill_path", "") else "" target_skill_text = _read_text(target_skill_path) if target_skill_path else "" max_tasks = cfg.get("max_tasks_per_night", 40) candidate_limit = max_tasks if cfg.get("target_task_filter", True) and target_skill_text: candidate_limit = max(max_tasks, max_tasks * 3) - digests = harvest_for_config(cfg, limit=session_limit) + digests = harvest_for_config(cfg, limit=session_limit, stats=harvest_stats) tasks = mine( digests, max_tasks=max_tasks, @@ -820,7 +822,8 @@ def cmd_harvest(args) -> int: json_payload["output"] = output_path print(json.dumps(json_payload, ensure_ascii=False, indent=2)) else: - print(f"[sleep] {len(digests)} sessions -> {len(tasks)} tasks") + print(f"[sleep] {len(digests)} sessions -> {len(tasks)} tasks" + f"{format_harvest_note(harvest_stats)}") if output_path: print(f"[sleep] wrote reviewed-task draft: {output_path}") for t in tasks: diff --git a/skillopt_sleep/cycle.py b/skillopt_sleep/cycle.py index 9621694c2..42a663628 100644 --- a/skillopt_sleep/cycle.py +++ b/skillopt_sleep/cycle.py @@ -18,7 +18,7 @@ import time import unicodedata from dataclasses import dataclass -from typing import List, Optional, Sequence +from typing import Dict, List, Optional, Sequence from skillopt_sleep import evidence from skillopt_sleep.backend import Backend, CursorBackendError, build_backend @@ -28,6 +28,7 @@ from skillopt_sleep.dream import dream_consolidate from skillopt_sleep.evidence import EvidenceLog from skillopt_sleep.handoff_backend import PendingCalls +from skillopt_sleep.harvest import format_harvest_note from skillopt_sleep.harvest_sources import harvest_for_config from skillopt_sleep.memory import ensure_skill_scaffold from skillopt_sleep.mine import group_tasks_by_skill_hint, mine @@ -798,6 +799,7 @@ def budget_exhausted() -> bool: _progress(cfg, f"using {len(tasks)} seeded tasks") else: since = state.last_harvest_for(project, harvest_source) + harvest_stats: Dict[str, int] = {} # On first run (no prior harvest), apply lookback_hours so we don't # scan the entire transcript history and trigger massive LLM mining. if since is None: @@ -819,9 +821,13 @@ def budget_exhausted() -> bool: cfg, since_iso=since, limit=max_sessions, + stats=harvest_stats, ) n_sessions = len(digests) - _progress(cfg, f"harvest done: sessions={n_sessions}") + _progress( + cfg, + f"harvest done: sessions={n_sessions}{format_harvest_note(harvest_stats)}", + ) if ev is not None: # The transcript end of the evidentiary chain: which sessions were # even considered, and what signals they carried into mining. diff --git a/skillopt_sleep/harvest.py b/skillopt_sleep/harvest.py index 7f3bee8f7..be4af85c2 100644 --- a/skillopt_sleep/harvest.py +++ b/skillopt_sleep/harvest.py @@ -236,6 +236,109 @@ def _is_headless_replay(digest: "SessionDigest") -> bool: return False +# ── Issue #319: single-turn transcripts a local hook re-runs verbatim ── + +# A hook that summarises the previous session on a timer (for example every +# hour) writes one single-turn transcript per run: a machine-written prompt +# and one answer. The prompt is the hook's own template, so its leading text +# is identical in every run while the payload it embeds differs. Neither +# `_is_headless_replay` nor `_is_agent_session` matches those templates — they +# are another tool's, not the engine's or a known sub-agent's — and in the +# Claude Code desktop app the headless runs carry entrypoint +# "claude-desktop", like interactive ones, so `entrypoint` cannot separate +# them either. They arrive at a fixed cadence, so they used to fill +# `--max-sessions` (harvest walks newest first and stops at the cap) and push +# real interactive sessions out of the night entirely. +# +# The signal available to a tool that cannot know another tool's templates is +# recurrence: one prompt prefix shared by several single-turn sessions of the +# same harvest window. Real one-shot prompts repeat verbatim too, so a +# candidate must additionally be long enough that the shared prefix is +# boilerplate — a machine template carries the run's own payload past it. +_REPEAT_PREFIX_CHARS = 120 + +# Sessions that must share a prompt prefix before it counts as a template. +_DEFAULT_REPEAT_MIN_SESSIONS = 3 + + +def _repeat_min_sessions() -> int: + """How many sessions a prompt prefix must recur in (Issue #319). + + ``SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS`` overrides the default of + ``_DEFAULT_REPEAT_MIN_SESSIONS``; ``0`` disables the filter, which is the + escape hatch for a project whose own one-shot prompts are long and + identical. Read per call so a long-running process — and the tests — can + change it without reloading the module. + """ + raw = os.environ.get("SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS", "").strip() + if not raw: + return _DEFAULT_REPEAT_MIN_SESSIONS + try: + return max(0, int(raw)) + except ValueError: + import logging + logging.getLogger("skillopt_sleep").warning( + "ignoring non-integer SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS=%r", raw + ) + return _DEFAULT_REPEAT_MIN_SESSIONS + + +def _repeat_prefix(digest: "SessionDigest") -> str: + """Normalized leading text of a single-turn session, or "" if ineligible. + + Multi-turn sessions are interactive by definition, and a prompt shorter + than ``_REPEAT_PREFIX_CHARS`` is dropped because then the *whole* prompt + would have to repeat — the shape of a real recurring one-liner, not of a + template with a fresh payload appended. + """ + if digest.n_user_turns != 1: + return "" + prompt = (digest.user_prompts[0] if digest.user_prompts else "").strip() + if len(prompt) < _REPEAT_PREFIX_CHARS: + return "" + # Case and whitespace are uninteresting here: a template re-rendered with + # different line wrapping is still the same template. + return " ".join(prompt[:_REPEAT_PREFIX_CHARS].lower().split()) + + +def _machine_generated_repeats( + digests: List["SessionDigest"], min_sessions: int +) -> set: + """Session ids whose prompt prefix recurs across `min_sessions` sessions.""" + if min_sessions <= 0: + return set() + groups: Dict[str, List[str]] = {} + for d in digests: + key = _repeat_prefix(d) + if key: + groups.setdefault(key, []).append(d.session_id) + repeated: set = set() + for ids in groups.values(): + if len(ids) >= min_sessions: + repeated.update(ids) + return repeated + + +def format_harvest_note(stats: Optional[Dict[str, int]]) -> str: + """One-line suffix describing what ``harvest()`` filtered out (#319). + + Empty when there is nothing worth reporting, so callers can append it + unconditionally. + """ + if not stats: + return "" + parts: List[str] = [] + repeats = int(stats.get("machine_generated_repeats", 0) or 0) + if repeats: + parts.append(f"{repeats} machine-generated") + over_cap = int(stats.get("skipped_by_cap", 0) or 0) + if over_cap: + parts.append(f"{over_cap} past the session cap") + if not parts: + return "" + return f" (skipped: {', '.join(parts)})" + + def digest_transcript(path: str) -> Optional[SessionDigest]: """Build a SessionDigest from one ``.jsonl`` transcript.""" session_id = os.path.splitext(os.path.basename(path))[0] @@ -385,6 +488,7 @@ def harvest( invoked_project: str = "", since_iso: Optional[str] = None, limit: int = 0, + stats: Optional[Dict[str, int]] = None, ) -> List[SessionDigest]: """Walk ~/.claude/projects and return digests matching scope/time. @@ -395,6 +499,17 @@ def harvest( invoked_project : str used when scope == "invoked" since_iso : str|None ISO8601; only sessions ending after this are kept limit : int cap number of digests (0 = no cap) + stats : dict|None filled in place with what was filtered out: + ``candidates`` (in scope and in window), + ``machine_generated_repeats`` (dropped as a + re-run template, see `_machine_generated_repeats`), + ``skipped_by_cap`` (in scope, but past `limit`), + and ``sessions`` (returned). Callers use + `format_harvest_note` to report them. + + The window is scanned in full before `limit` is applied: a repeated + template can only be recognized by comparing it with the rest of the + window, and it must not be counted against the cap it used to fill. """ digests: List[SessionDigest] = [] if not os.path.isdir(transcripts_dir): @@ -412,6 +527,7 @@ def harvest( # newest first by mtime paths.sort(key=_safe_mtime, reverse=True) + candidates: List[SessionDigest] = [] for p in paths: d = digest_transcript(p) if d is None: @@ -427,7 +543,20 @@ def harvest( # ended_at timestamp — mtime can diverge (copy/touch), so we # cannot break here; we must continue to check all files. continue - digests.append(d) - if limit and len(digests) >= limit: - break + candidates.append(d) + + # Issue #319: a local hook's single-turn summary sessions arrive hourly. + # They are transcript noise, not the user's tasks, so drop them before the + # cap rather than letting them consume it. + repeated = _machine_generated_repeats(candidates, _repeat_min_sessions()) + kept = [d for d in candidates if d.session_id not in repeated] + digests = kept[:limit] if limit else kept + + if stats is not None: + stats.update( + candidates=len(candidates), + machine_generated_repeats=len(repeated), + skipped_by_cap=len(kept) - len(digests), + sessions=len(digests), + ) return digests diff --git a/skillopt_sleep/harvest_sources.py b/skillopt_sleep/harvest_sources.py index 12506e7b2..e4d890566 100644 --- a/skillopt_sleep/harvest_sources.py +++ b/skillopt_sleep/harvest_sources.py @@ -13,7 +13,19 @@ from skillopt_sleep.types import SessionDigest -def harvest_for_config(cfg, *, since_iso: Optional[str] = None, limit: int = 0) -> list[SessionDigest]: +def harvest_for_config( + cfg, + *, + since_iso: Optional[str] = None, + limit: int = 0, + stats: Optional[dict] = None, +) -> list[SessionDigest]: + """Harvest with the configured source. + + ``stats`` is filled in place by the Claude source (see + `skillopt_sleep.harvest.harvest`); the other sources have no repeat + filter to report yet and leave it untouched. + """ source = cfg.get("transcript_source", "claude") scope = cfg.get("projects", "invoked") invoked_project = cfg.get("invoked_project", "") @@ -83,4 +95,5 @@ def harvest_for_config(cfg, *, since_iso: Optional[str] = None, limit: int = 0) invoked_project=invoked_project, since_iso=since_iso, limit=limit, + stats=stats, ) diff --git a/tests/test_harvest_machine_repeats.py b/tests/test_harvest_machine_repeats.py new file mode 100644 index 000000000..b09b7c3af --- /dev/null +++ b/tests/test_harvest_machine_repeats.py @@ -0,0 +1,290 @@ +"""Issue #319: single-turn transcripts a local hook re-runs verbatim. + +A hook that summarises the previous session on a timer writes one single-turn +transcript per run. Neither the engine's replay markers nor the sub-agent +markers match that template, and `entrypoint` is "claude-desktop" in the +desktop app, like interactive sessions — so before this filter they were +harvested as user tasks and, because harvest walks newest first and stops at +`--max-sessions`, they filled the cap and pushed real sessions out of the +night entirely. + +Pure-stdlib (unittest), deterministic, no API key, no third-party deps. +""" +from __future__ import annotations + +import json +import os +import tempfile +import unittest +from unittest import mock + +from skillopt_sleep.harvest import harvest +from skillopt_sleep.mine import heuristic_mine +from skillopt_sleep.types import SessionDigest + +# The shape from the report: one identical header, then the run's own payload. +SUMMARY_HEADER = ( + "Below is a conversation log from a coding session. Create a summary to " + "help the next session quickly understand the context.\n## Prioritize\n" + "- decisions\n- open items\n\nLOG:\n" +) + + +class _HarvestFixture: + """Writes Claude Code shaped transcripts under a temporary projects dir.""" + + def __init__(self, tmp: str, project: str | None = None): + self.transcripts = os.path.join(tmp, "projects") + self.project = project or os.path.join(tmp, "proj") + os.makedirs(self.project, exist_ok=True) + + def session(self, session_id: str, turns, mtime: float) -> str: + folder = os.path.join(self.transcripts, "proj") + os.makedirs(folder, exist_ok=True) + path = os.path.join(folder, session_id + ".jsonl") + with open(path, "w", encoding="utf-8") as f: + for i, (role, text) in enumerate(turns): + f.write(json.dumps({ + "type": role, + "cwd": self.project, + "sessionId": session_id, + "timestamp": f"2026-01-01T10:{i:02d}:00Z", + "message": {"role": role, "content": text}, + }) + "\n") + os.utime(path, (mtime, mtime)) + return path + + def summary_sessions(self, n: int, mtime: float = 2_000_000) -> None: + """Single-turn sessions whose prompts differ only after the header.""" + for i in range(n): + self.session( + f"summary-{i}", + [("user", SUMMARY_HEADER + f"turn {i}: user asked for item {i}\n"), + ("assistant", f"Summary {i}: ...")], + mtime + i, + ) + + def real_sessions(self) -> None: + self.session("real-1", [ + ("user", "Fix the failing date parser test in module B."), + ("assistant", "Fixed."), + ("user", "Thanks, now add a regression test."), + ("assistant", "Added."), + ], 1_000_000) + self.session("real-2", [ + ("user", "Add a --dry-run flag to the export command."), + ("assistant", "Added."), + ], 1_000_100) + + +class TestMachineGeneratedRepeats(unittest.TestCase): + def setUp(self): + self._env = mock.patch.dict(os.environ, {}, clear=False) + self._env.start() + os.environ.pop("SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS", None) + + def tearDown(self): + self._env.stop() + + def test_summary_sessions_do_not_fill_the_session_cap(self): + # The report's repro: three hourly summaries are newer than two real + # sessions, so with limit=3 the summaries used to take every slot. + with tempfile.TemporaryDirectory() as tmp: + fx = _HarvestFixture(tmp) + fx.real_sessions() + fx.summary_sessions(3) + + capped = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project, limit=3) + uncapped = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project) + + self.assertEqual(sorted(d.session_id for d in capped), + ["real-1", "real-2"]) + self.assertEqual(sorted(d.session_id for d in uncapped), + ["real-1", "real-2"]) + + def test_the_mined_intents_are_the_real_ones(self): + with tempfile.TemporaryDirectory() as tmp: + fx = _HarvestFixture(tmp) + fx.real_sessions() + fx.summary_sessions(3) + capped = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project, limit=3) + mined = heuristic_mine(capped) + + self.assertTrue(mined) + self.assertEqual(sorted({s for t in mined for s in t.source_sessions}), + ["real-1", "real-2"]) + for task in mined: + self.assertFalse(task.intent.startswith(SUMMARY_HEADER[:40])) + + def test_short_repeated_one_shot_prompts_are_kept(self): + # The false-positive guard: a real one-shot prompt is usually short + # and repeats *entirely*, so it never reaches the prefix rule. + with tempfile.TemporaryDirectory() as tmp: + fx = _HarvestFixture(tmp) + for i in range(4): + fx.session(f"ask-{i}", + [("user", "Run the tests and fix what is broken."), + ("assistant", "Done.")], + 1_000_000 + i) + kept = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project) + + self.assertEqual(len(kept), 4) + + def test_multi_turn_session_sharing_the_prefix_is_kept(self): + # Only single-turn sessions can be a re-run template; a follow-up + # user turn makes the session interactive. + with tempfile.TemporaryDirectory() as tmp: + fx = _HarvestFixture(tmp) + fx.summary_sessions(3) + fx.session("interactive", [ + ("user", SUMMARY_HEADER + "written by hand this time\n"), + ("assistant", "ok"), + ("user", "no, summarise the other file too"), + ("assistant", "done"), + ], 2_000_100) + kept = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project) + + self.assertEqual([d.session_id for d in kept], ["interactive"]) + + def test_threshold_follows_the_env_var(self): + with tempfile.TemporaryDirectory() as tmp: + fx = _HarvestFixture(tmp) + fx.summary_sessions(2) + + with mock.patch.dict(os.environ, + {"SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS": "2"}): + dropped = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project) + with mock.patch.dict(os.environ, + {"SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS": "4"}): + kept = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project) + with mock.patch.dict(os.environ, + {"SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS": "0"}): + disabled = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project) + + self.assertEqual(dropped, []) + self.assertEqual(len(kept), 2) + self.assertEqual(len(disabled), 2) + + def test_non_integer_threshold_falls_back_to_the_default(self): + with tempfile.TemporaryDirectory() as tmp: + fx = _HarvestFixture(tmp) + fx.summary_sessions(3) + with mock.patch.dict(os.environ, + {"SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS": "many"}): + kept = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project) + self.assertEqual(kept, []) + + def test_prefix_is_normalized_and_length_gated(self): + from skillopt_sleep.harvest import ( + _REPEAT_PREFIX_CHARS, + _machine_generated_repeats, + _repeat_prefix, + ) + + def digest(session_id, prompt, n_user=1): + return SessionDigest(session_id=session_id, project="/p", + user_prompts=[prompt], n_user_turns=n_user) + + long_a = digest("a", SUMMARY_HEADER.upper() + "payload one") + long_b = digest("b", SUMMARY_HEADER.lower() + "payload two") + short = digest("c", "same one-liner", n_user=1) + two_turn = digest("d", SUMMARY_HEADER + "payload", n_user=2) + + self.assertEqual(_repeat_prefix(digest("short", "x" * ( + _REPEAT_PREFIX_CHARS - 1))), "") + self.assertEqual(_repeat_prefix(short), "") + self.assertEqual(_repeat_prefix(two_turn), "") + # Same text, different case and wrapping -> one prefix. + self.assertEqual(_repeat_prefix(long_a), _repeat_prefix(long_b)) + self.assertEqual( + _machine_generated_repeats([long_a, long_b], 2), {"a", "b"}) + self.assertEqual( + _machine_generated_repeats([long_a, long_b, short, two_turn], 2), + {"a", "b"}) + + def test_stats_report_what_was_filtered(self): + with tempfile.TemporaryDirectory() as tmp: + fx = _HarvestFixture(tmp) + fx.real_sessions() + fx.summary_sessions(3) + stats: dict = {} + capped = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project, limit=3, stats=stats) + + self.assertEqual(stats, { + "candidates": 5, + "machine_generated_repeats": 3, + "skipped_by_cap": 0, + "sessions": 2, + }) + self.assertEqual(len(capped), stats["sessions"]) + + def test_stats_count_sessions_past_the_cap(self): + with tempfile.TemporaryDirectory() as tmp: + fx = _HarvestFixture(tmp) + fx.real_sessions() + fx.summary_sessions(3) + stats: dict = {} + harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project, limit=1, stats=stats) + + self.assertEqual(stats["machine_generated_repeats"], 3) + self.assertEqual(stats["skipped_by_cap"], 1) + self.assertEqual(stats["sessions"], 1) + + def test_harvest_note_is_empty_when_nothing_was_filtered(self): + from skillopt_sleep.harvest import format_harvest_note + + self.assertEqual(format_harvest_note({}), "") + self.assertEqual(format_harvest_note(None), "") + self.assertEqual( + format_harvest_note({"machine_generated_repeats": 0, + "skipped_by_cap": 0}), "") + self.assertEqual( + format_harvest_note({"machine_generated_repeats": 3, + "skipped_by_cap": 0}), + " (skipped: 3 machine-generated)") + self.assertEqual( + format_harvest_note({"machine_generated_repeats": 3, + "skipped_by_cap": 2}), + " (skipped: 3 machine-generated, 2 past the session cap)") + + +class TestHarvestForConfigStats(unittest.TestCase): + def test_claude_source_reports_stats_through_the_router(self): + import argparse + + from skillopt_sleep.__main__ import _add_common, _cfg_from_args + from skillopt_sleep.harvest_sources import harvest_for_config + + with tempfile.TemporaryDirectory() as tmp: + fx = _HarvestFixture(tmp) + fx.real_sessions() + fx.summary_sessions(3) + parser = argparse.ArgumentParser() + _add_common(parser) + args = parser.parse_args([ + "--project", fx.project, + "--source", "claude", + "--claude-home", tmp, + ]) + cfg = _cfg_from_args(args) + stats: dict = {} + digests = harvest_for_config(cfg, limit=3, stats=stats) + + self.assertEqual(sorted(d.session_id for d in digests), + ["real-1", "real-2"]) + self.assertEqual(stats["machine_generated_repeats"], 3) + + +if __name__ == "__main__": + unittest.main() From 4af3531a6294368edc602bf264345dfeda5975f7 Mon Sep 17 00:00:00 2001 From: Ansu <230804140223@stu.nepu.edu.cn> Date: Mon, 19 Oct 2026 13:01:33 +0800 Subject: [PATCH 2/2] fix(sleep): make the repeat filter opt-in and keep the cap bounded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two corrections from review, both about the default path rather than the mechanism: * Recurrence is not evidence of who wrote a prompt. Three human code-review requests that share one long checklist prefix and differ in the task payload are structurally identical to a hook re-running itself: one single-turn session each, >120 shared leading characters, a different tail. Dropping those loses real user tasks, which is worse than harvesting a hook's summary sessions, so the filter is now off unless a project opts in with SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS (3 documented as the usual value; 0 disables an exported value again). The docstring and both READMEs say why, and name the counter-example. * With the filter off, harvest keeps its single bounded pass: it digests newest-first and stops at `limit`, so `--max-sessions 1` reads one transcript instead of the whole eligible window. The full-window scan now happens only when the filter is opted into, which is the mode that needs it. Counters no longer claim a producer: `repeated_template_sessions` / "repeated-template sessions" replaces `machine_generated_repeats` / "machine-generated", in the stats dict and in the progress note. Measured on the pre-change code with the new tests: 9 failed, 6 passed — including the shared-checklist case (3 requests dropped) and the bounded-cap case (40 transcripts read for `limit=1`). After: 15 passed. Full suite: 1,846 passed, 11 skipped, 368 subtests. --- docs/sleep/README.md | 29 ++-- plugins/claude-code/README.md | 17 ++- skillopt_sleep/harvest.py | 100 ++++++++++---- tests/test_harvest_machine_repeats.py | 191 +++++++++++++++++++++----- 4 files changed, 259 insertions(+), 78 deletions(-) diff --git a/docs/sleep/README.md b/docs/sleep/README.md index 1cc56bfb0..3ed9d4568 100644 --- a/docs/sleep/README.md +++ b/docs/sleep/README.md @@ -213,7 +213,7 @@ the night: | `SKILLOPT_SLEEP_AGENT_MARKERS` | Comma-separated markers. A session whose **first** user prompt contains one is dropped entire — for prompts written by another tool's sub-agent (memory observers, critic agents, hook-driven runs). | | `SKILLOPT_SLEEP_NEG_FEEDBACK` | Extra comma-separated phrases, in any language, that count as you disapproving of the previous answer. | | `SKILLOPT_SLEEP_POS_FEEDBACK` | The same, for approval. | -| `SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS` | Sessions a prompt prefix must recur in before the repeats are treated as a hook re-running itself (default `3`; `0` turns the filter off). | +| `SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS` | Sessions a prompt prefix must recur in before the repeats are treated as one template (default `0`: off; `3` is the usual opt-in, `0` turns a configured value back off). | The first three cover a tool whose prompt template you can quote, and the filter they feed is exact: it checks the session's first user prompt against @@ -226,22 +226,31 @@ they exist for: export SKILLOPT_SLEEP_AGENT_MARKERS="Below is a conversation log from a coding session" ``` -`SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS` covers the templates you cannot quote. A -single-turn session that shares its first 120 characters of prompt with -`N` others in the same harvest window is a hook re-running one job; the filter -drops those repeats *before* `--max-sessions` is applied, so they cannot -consume the cap the way they did in #319. A session with a follow-up user turn -is never matched (it is interactive by definition), and neither is a prompt -shorter than the prefix — a real one-shot request repeats entirely rather than -sharing a long boilerplate header. +`SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS` covers the templates you cannot quote, but +recurrence is not proof of who wrote a prompt, so it is **off by default**. A +single-turn session that shares its first 120 characters of prompt with `N` +others in the same harvest window is dropped *before* `--max-sessions` is +applied, so the repeats cannot consume the cap the way they did in #319 — but +only once you opt in with a value for `N` (3 is the usual choice). The reason +for the default: three people asking for a code review against one long shared +checklist are structurally identical to a hook re-running itself — one turn, one +long common prefix, a different task after it — and losing a real task is worse +than harvesting a hook's summary sessions. Set the variable for a project whose +recurring sessions you have recognized. A session with a follow-up user turn is +never matched (it is interactive by definition), and neither is a prompt shorter +than the prefix — a real one-shot request repeats entirely rather than sharing a +long boilerplate header. `harvest` and the progress output of `run` / `dry-run` report what was dropped, so a night that looks empty is explainable: ```text -[sleep] 120 sessions -> 40 tasks (skipped: 297 machine-generated, 2 past the session cap) +[sleep] 120 sessions -> 40 tasks (skipped: 297 repeated-template sessions, 2 past the session cap) ``` +With the filter off (the default) nothing is dropped by it, `--max-sessions` +still bounds how many transcripts are read, and the note reports only the cap. + ### VS Code GitHub Copilot Chat Use `--source copilot` to harvest local VS Code GitHub Copilot Chat sessions. diff --git a/plugins/claude-code/README.md b/plugins/claude-code/README.md index 02b15cdc6..f32324171 100644 --- a/plugins/claude-code/README.md +++ b/plugins/claude-code/README.md @@ -117,17 +117,20 @@ app they carry `entrypoint: "claude-desktop"` like interactive ones — so neith export SKILLOPT_SLEEP_AGENT_MARKERS="Below is a conversation log from a coding session" ``` -A template you cannot quote is covered without configuration: a single-turn +A template you cannot quote is covered only after you opt in: a single-turn session sharing its first 120 characters of prompt with three others in the same -harvest window is a hook re-running one job, and it is dropped before the -session cap is applied. Tune that with `SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS` -(default `3`, `0` disables). A session with a follow-up user turn is never -matched, and neither is a prompt shorter than the prefix — your own one-shot -requests repeat entirely rather than sharing a long boilerplate header. +harvest window is dropped before the session cap is applied once +`SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS` is set (default `0`, off; `3` is the usual +choice). Recurrence is not proof of who wrote a prompt — three human requests +against one long shared checklist look the same — so the default keeps every +session and `--max-sessions` still bounds how many transcripts are read. A +session with a follow-up user turn is never matched, and neither is a prompt +shorter than the prefix — your own one-shot requests repeat entirely rather than +sharing a long boilerplate header. `/skillopt-sleep harvest` and the progress line of a `run` / `dry-run` report what was dropped, so a thin night is explainable: -`harvest done: sessions=120 (skipped: 297 machine-generated, 2 past the session cap)`. +`harvest done: sessions=120 (skipped: 297 repeated-template sessions, 2 past the session cap)`. Full reference: [the sleep guide](../../docs/sleep/README.md#harvest-filters-environment). ### Handoff mode (session answers the model calls) diff --git a/skillopt_sleep/harvest.py b/skillopt_sleep/harvest.py index be4af85c2..9132d54d7 100644 --- a/skillopt_sleep/harvest.py +++ b/skillopt_sleep/harvest.py @@ -255,20 +255,34 @@ def _is_headless_replay(digest: "SessionDigest") -> bool: # same harvest window. Real one-shot prompts repeat verbatim too, so a # candidate must additionally be long enough that the shared prefix is # boilerplate — a machine template carries the run's own payload past it. +# +# Recurrence alone cannot tell the two apart, though. Three people (or the same +# person three times) asking for a code review against one long shared +# checklist is structurally identical: one single-turn session each, a long +# common prefix, a different task after it. Calling those "machine-generated" +# would be a guess about who produced them, and dropping user tasks is worse +# than harvesting a hook's own summary sessions — those are noise, but a lost +# task is a lost night. So the filter is **off by default** and enabled per +# project with ``SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS``; what remains is a +# mitigation for a hook whose runs the user has recognized, not a claim the +# harvester can make on its own. _REPEAT_PREFIX_CHARS = 120 # Sessions that must share a prompt prefix before it counts as a template. -_DEFAULT_REPEAT_MIN_SESSIONS = 3 +# 0 disables the filter, and is the default (see above). +_DEFAULT_REPEAT_MIN_SESSIONS = 0 def _repeat_min_sessions() -> int: """How many sessions a prompt prefix must recur in (Issue #319). ``SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS`` overrides the default of - ``_DEFAULT_REPEAT_MIN_SESSIONS``; ``0`` disables the filter, which is the - escape hatch for a project whose own one-shot prompts are long and - identical. Read per call so a long-running process — and the tests — can - change it without reloading the module. + ``_DEFAULT_REPEAT_MIN_SESSIONS`` (0 — off). A project that recognizes a + hook's hourly summary template opts in by setting it to the number of runs + that shape its window (3 by default in the documentation); ``0`` keeps the + filter off even when an earlier shell exported a larger value. Read per + call so a long-running process — and the tests — can change it without + reloading the module. """ raw = os.environ.get("SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS", "").strip() if not raw: @@ -301,10 +315,15 @@ def _repeat_prefix(digest: "SessionDigest") -> str: return " ".join(prompt[:_REPEAT_PREFIX_CHARS].lower().split()) -def _machine_generated_repeats( +def _repeated_template_sessions( digests: List["SessionDigest"], min_sessions: int ) -> set: - """Session ids whose prompt prefix recurs across `min_sessions` sessions.""" + """Session ids whose prompt prefix recurs across `min_sessions` sessions. + + The name says what was measured — a repeated prompt template — because the + producer is not something this function can know. Off unless a project opts + in via ``SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS``. + """ if min_sessions <= 0: return set() groups: Dict[str, List[str]] = {} @@ -328,9 +347,9 @@ def format_harvest_note(stats: Optional[Dict[str, int]]) -> str: if not stats: return "" parts: List[str] = [] - repeats = int(stats.get("machine_generated_repeats", 0) or 0) + repeats = int(stats.get("repeated_template_sessions", 0) or 0) if repeats: - parts.append(f"{repeats} machine-generated") + parts.append(f"{repeats} repeated-template sessions") over_cap = int(stats.get("skipped_by_cap", 0) or 0) if over_cap: parts.append(f"{over_cap} past the session cap") @@ -501,15 +520,19 @@ def harvest( limit : int cap number of digests (0 = no cap) stats : dict|None filled in place with what was filtered out: ``candidates`` (in scope and in window), - ``machine_generated_repeats`` (dropped as a - re-run template, see `_machine_generated_repeats`), + ``repeated_template_sessions`` (dropped as a + re-run template, see + `_repeated_template_sessions`), ``skipped_by_cap`` (in scope, but past `limit`), and ``sessions`` (returned). Callers use `format_harvest_note` to report them. - The window is scanned in full before `limit` is applied: a repeated - template can only be recognized by comparing it with the rest of the - window, and it must not be counted against the cap it used to fill. + With the repeat filter off (the default, see `_repeat_min_sessions`) this + is a single bounded pass: transcripts are digested newest-first and the walk + stops at `limit`. With the filter opted into, the window is scanned in full + before `limit` is applied, because a repeated template can only be + recognized by comparing it with the rest of the window and it must not be + counted against the cap it used to fill. """ digests: List[SessionDigest] = [] if not os.path.isdir(transcripts_dir): @@ -528,34 +551,57 @@ def harvest( paths.sort(key=_safe_mtime, reverse=True) candidates: List[SessionDigest] = [] - for p in paths: - d = digest_transcript(p) - if d is None: - continue + repeat_min = _repeat_min_sessions() + + def _eligible(d: SessionDigest) -> bool: if _is_headless_replay(d): - continue # Issue #62: skip engine's own headless replay sessions + return False # Issue #62: skip engine's own headless replay sessions if _is_agent_session(d): - continue # skip other tools' sub-agent transcripts (claude-mem etc.) + return False # skip other tools' sub-agent transcripts (claude-mem etc.) if not _project_matches(d.project or "", scope, invoked_project): - continue + return False if since_iso and d.ended_at and d.ended_at < since_iso: # Note: files are sorted by mtime but we compare the embedded # ended_at timestamp — mtime can diverge (copy/touch), so we # cannot break here; we must continue to check all files. + return False + return True + + for p in paths: + d = digest_transcript(p) + if d is None or not _eligible(d): continue - candidates.append(d) + if repeat_min <= 0: + # The repeat filter is off (the default): keep the single bounded + # pass older releases ran, so `limit=N` digests N transcripts + # rather than the whole eligible window. + digests.append(d) + if limit and len(digests) >= limit: + break + else: + candidates.append(d) + + if repeat_min <= 0: + if stats is not None: + stats.update( + candidates=len(digests), + repeated_template_sessions=0, + skipped_by_cap=0, + sessions=len(digests), + ) + return digests - # Issue #319: a local hook's single-turn summary sessions arrive hourly. - # They are transcript noise, not the user's tasks, so drop them before the - # cap rather than letting them consume it. - repeated = _machine_generated_repeats(candidates, _repeat_min_sessions()) + # Issue #319 (opt-in): a local hook's single-turn summary sessions arrive + # hourly. They are transcript noise, not the user's tasks, so drop them + # before the cap rather than letting them consume it. + repeated = _repeated_template_sessions(candidates, repeat_min) kept = [d for d in candidates if d.session_id not in repeated] digests = kept[:limit] if limit else kept if stats is not None: stats.update( candidates=len(candidates), - machine_generated_repeats=len(repeated), + repeated_template_sessions=len(repeated), skipped_by_cap=len(kept) - len(digests), sessions=len(digests), ) diff --git a/tests/test_harvest_machine_repeats.py b/tests/test_harvest_machine_repeats.py index b09b7c3af..e8d29ae68 100644 --- a/tests/test_harvest_machine_repeats.py +++ b/tests/test_harvest_machine_repeats.py @@ -8,6 +8,11 @@ `--max-sessions`, they filled the cap and pushed real sessions out of the night entirely. +The filter is opt-in (`SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS`), because recurrence +cannot distinguish a hook's template from several human requests that share a +long checklist. The default must therefore keep every session, and the cap must +stay bounded when the filter is off. + Pure-stdlib (unittest), deterministic, no API key, no third-party deps. """ from __future__ import annotations @@ -18,6 +23,7 @@ import unittest from unittest import mock +from skillopt_sleep import harvest as harvest_module from skillopt_sleep.harvest import harvest from skillopt_sleep.mine import heuristic_mine from skillopt_sleep.types import SessionDigest @@ -29,6 +35,23 @@ "- decisions\n- open items\n\nLOG:\n" ) +# The shape from the review, which must never be dropped by default: three +# human requests for a code review that share one long checklist prefix and +# differ only in the task payload. +REVIEW_HEADER = ( + "Please review the following pull request against our checklist and report " + "every violation you find.\n\n## Checklist\n- error handling\n- logging\n" + "- tests\n- naming\n- docs\n- performance\n- backwards compatibility\n\n" + "## Pull request\n" +) + + +def _opt_in(min_sessions: int = 3): + """Enable the repeat filter for one call (it is off by default).""" + return mock.patch.dict( + os.environ, {"SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS": str(min_sessions)} + ) + class _HarvestFixture: """Writes Claude Code shaped transcripts under a temporary projects dir.""" @@ -77,6 +100,83 @@ def real_sessions(self) -> None: ], 1_000_100) +class TestDefaultIsSafe(unittest.TestCase): + """The two defaults the review asked for.""" + + def setUp(self): + self._env = mock.patch.dict(os.environ, {}, clear=False) + self._env.start() + os.environ.pop("SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS", None) + + def tearDown(self): + self._env.stop() + + def test_shared_checklist_requests_are_not_dropped_by_default(self): + # The review's counter-example: three genuine human code-review + # requests with one long shared prefix and three different payloads. + with tempfile.TemporaryDirectory() as tmp: + fx = _HarvestFixture(tmp) + for i in range(3): + fx.session(f"review-{i}", + [("user", REVIEW_HEADER + f"src/module_{i}.py\n"), + ("assistant", "Findings ...")], + 1_000_000 + i) + kept = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project) + + self.assertEqual(sorted(d.session_id for d in kept), + ["review-0", "review-1", "review-2"]) + + def test_default_cap_still_bounds_how_many_transcripts_are_digested(self): + # With the filter off, `limit=1` must read one transcript, not the + # whole eligible window: the pre-filter code stopped at the cap. + with tempfile.TemporaryDirectory() as tmp: + fx = _HarvestFixture(tmp) + for i in range(40): + fx.session(f"s-{i}", + [("user", f"task number {i}: change file {i}.py"), + ("assistant", "ok")], + 1_000_000 + i) + calls: list = [] + real = harvest_module.digest_transcript + + def counting(path): + calls.append(path) + return real(path) + + with mock.patch.object(harvest_module, "digest_transcript", counting): + got = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project, limit=1) + + self.assertEqual(len(got), 1) + self.assertEqual(len(calls), 1) + + def test_opt_in_scans_the_window_so_the_cap_can_be_refilled(self): + with tempfile.TemporaryDirectory() as tmp: + fx = _HarvestFixture(tmp) + for i in range(40): + fx.session(f"s-{i}", + [("user", f"task number {i}: change file {i}.py"), + ("assistant", "ok")], + 1_000_000 + i) + calls: list = [] + real = harvest_module.digest_transcript + + def counting(path): + calls.append(path) + return real(path) + + with mock.patch.object(harvest_module, "digest_transcript", counting): + with _opt_in(): + got = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project, limit=1) + + self.assertEqual(len(got), 1) + # Opting in trades the bounded read for repeat detection; this pins + # that the trade is deliberate rather than an accident. + self.assertEqual(len(calls), 40) + + class TestMachineGeneratedRepeats(unittest.TestCase): def setUp(self): self._env = mock.patch.dict(os.environ, {}, clear=False) @@ -94,10 +194,11 @@ def test_summary_sessions_do_not_fill_the_session_cap(self): fx.real_sessions() fx.summary_sessions(3) - capped = harvest(fx.transcripts, scope="invoked", - invoked_project=fx.project, limit=3) - uncapped = harvest(fx.transcripts, scope="invoked", - invoked_project=fx.project) + with _opt_in(): + capped = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project, limit=3) + uncapped = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project) self.assertEqual(sorted(d.session_id for d in capped), ["real-1", "real-2"]) @@ -109,8 +210,9 @@ def test_the_mined_intents_are_the_real_ones(self): fx = _HarvestFixture(tmp) fx.real_sessions() fx.summary_sessions(3) - capped = harvest(fx.transcripts, scope="invoked", - invoked_project=fx.project, limit=3) + with _opt_in(): + capped = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project, limit=3) mined = heuristic_mine(capped) self.assertTrue(mined) @@ -129,8 +231,9 @@ def test_short_repeated_one_shot_prompts_are_kept(self): [("user", "Run the tests and fix what is broken."), ("assistant", "Done.")], 1_000_000 + i) - kept = harvest(fx.transcripts, scope="invoked", - invoked_project=fx.project) + with _opt_in(): + kept = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project) self.assertEqual(len(kept), 4) @@ -146,8 +249,9 @@ def test_multi_turn_session_sharing_the_prefix_is_kept(self): ("user", "no, summarise the other file too"), ("assistant", "done"), ], 2_000_100) - kept = harvest(fx.transcripts, scope="invoked", - invoked_project=fx.project) + with _opt_in(): + kept = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project) self.assertEqual([d.session_id for d in kept], ["interactive"]) @@ -156,24 +260,24 @@ def test_threshold_follows_the_env_var(self): fx = _HarvestFixture(tmp) fx.summary_sessions(2) - with mock.patch.dict(os.environ, - {"SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS": "2"}): + with _opt_in(2): dropped = harvest(fx.transcripts, scope="invoked", invoked_project=fx.project) - with mock.patch.dict(os.environ, - {"SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS": "4"}): + with _opt_in(4): kept = harvest(fx.transcripts, scope="invoked", invoked_project=fx.project) - with mock.patch.dict(os.environ, - {"SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS": "0"}): + with _opt_in(0): disabled = harvest(fx.transcripts, scope="invoked", invoked_project=fx.project) + default = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project) self.assertEqual(dropped, []) self.assertEqual(len(kept), 2) self.assertEqual(len(disabled), 2) + self.assertEqual(len(default), 2) - def test_non_integer_threshold_falls_back_to_the_default(self): + def test_non_integer_threshold_falls_back_to_the_safe_default(self): with tempfile.TemporaryDirectory() as tmp: fx = _HarvestFixture(tmp) fx.summary_sessions(3) @@ -181,13 +285,14 @@ def test_non_integer_threshold_falls_back_to_the_default(self): {"SKILLOPT_SLEEP_REPEAT_MIN_SESSIONS": "many"}): kept = harvest(fx.transcripts, scope="invoked", invoked_project=fx.project) - self.assertEqual(kept, []) + # Default is off, so a typo must not start dropping sessions. + self.assertEqual(len(kept), 3) def test_prefix_is_normalized_and_length_gated(self): from skillopt_sleep.harvest import ( _REPEAT_PREFIX_CHARS, - _machine_generated_repeats, _repeat_prefix, + _repeated_template_sessions, ) def digest(session_id, prompt, n_user=1): @@ -206,10 +311,12 @@ def digest(session_id, prompt, n_user=1): # Same text, different case and wrapping -> one prefix. self.assertEqual(_repeat_prefix(long_a), _repeat_prefix(long_b)) self.assertEqual( - _machine_generated_repeats([long_a, long_b], 2), {"a", "b"}) + _repeated_template_sessions([long_a, long_b], 2), {"a", "b"}) self.assertEqual( - _machine_generated_repeats([long_a, long_b, short, two_turn], 2), + _repeated_template_sessions([long_a, long_b, short, two_turn], 2), {"a", "b"}) + # 0 is "off", not "every session repeats". + self.assertEqual(_repeated_template_sessions([long_a, long_b], 0), set()) def test_stats_report_what_was_filtered(self): with tempfile.TemporaryDirectory() as tmp: @@ -217,27 +324,42 @@ def test_stats_report_what_was_filtered(self): fx.real_sessions() fx.summary_sessions(3) stats: dict = {} - capped = harvest(fx.transcripts, scope="invoked", - invoked_project=fx.project, limit=3, stats=stats) + with _opt_in(): + capped = harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project, limit=3, + stats=stats) self.assertEqual(stats, { "candidates": 5, - "machine_generated_repeats": 3, + "repeated_template_sessions": 3, "skipped_by_cap": 0, "sessions": 2, }) self.assertEqual(len(capped), stats["sessions"]) + def test_stats_are_empty_when_the_filter_is_off(self): + with tempfile.TemporaryDirectory() as tmp: + fx = _HarvestFixture(tmp) + fx.real_sessions() + stats: dict = {} + harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project, stats=stats) + + self.assertEqual(stats["repeated_template_sessions"], 0) + self.assertEqual(stats["skipped_by_cap"], 0) + self.assertEqual(stats["sessions"], 2) + def test_stats_count_sessions_past_the_cap(self): with tempfile.TemporaryDirectory() as tmp: fx = _HarvestFixture(tmp) fx.real_sessions() fx.summary_sessions(3) stats: dict = {} - harvest(fx.transcripts, scope="invoked", - invoked_project=fx.project, limit=1, stats=stats) + with _opt_in(): + harvest(fx.transcripts, scope="invoked", + invoked_project=fx.project, limit=1, stats=stats) - self.assertEqual(stats["machine_generated_repeats"], 3) + self.assertEqual(stats["repeated_template_sessions"], 3) self.assertEqual(stats["skipped_by_cap"], 1) self.assertEqual(stats["sessions"], 1) @@ -247,16 +369,16 @@ def test_harvest_note_is_empty_when_nothing_was_filtered(self): self.assertEqual(format_harvest_note({}), "") self.assertEqual(format_harvest_note(None), "") self.assertEqual( - format_harvest_note({"machine_generated_repeats": 0, + format_harvest_note({"repeated_template_sessions": 0, "skipped_by_cap": 0}), "") self.assertEqual( - format_harvest_note({"machine_generated_repeats": 3, + format_harvest_note({"repeated_template_sessions": 3, "skipped_by_cap": 0}), - " (skipped: 3 machine-generated)") + " (skipped: 3 repeated-template sessions)") self.assertEqual( - format_harvest_note({"machine_generated_repeats": 3, + format_harvest_note({"repeated_template_sessions": 3, "skipped_by_cap": 2}), - " (skipped: 3 machine-generated, 2 past the session cap)") + " (skipped: 3 repeated-template sessions, 2 past the session cap)") class TestHarvestForConfigStats(unittest.TestCase): @@ -279,11 +401,12 @@ def test_claude_source_reports_stats_through_the_router(self): ]) cfg = _cfg_from_args(args) stats: dict = {} - digests = harvest_for_config(cfg, limit=3, stats=stats) + with _opt_in(): + digests = harvest_for_config(cfg, limit=3, stats=stats) self.assertEqual(sorted(d.session_id for d in digests), ["real-1", "real-2"]) - self.assertEqual(stats["machine_generated_repeats"], 3) + self.assertEqual(stats["repeated_template_sessions"], 3) if __name__ == "__main__":