From 5ffd035c2190949e3cf2cd4d72886dd3c252c050 Mon Sep 17 00:00:00 2001 From: kuhyx Date: Sun, 26 Jul 2026 19:29:25 +0200 Subject: [PATCH] session-autopsy: measure turn efficiency, the dominant token cost MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cache read is charged per API round-trip on the whole conversation so far, so a long session's cost is driven by TURN COUNT, not message length. One measured session spent 267M cache-read against 856k output — a ratio of 290:1. The report priced tokens but never counted the turns spending them. Across the 263 stored transcripts this now shows 33,508 tool turns at an 8% batching rate, of which 4,437 (13%) decided nothing: polling a counter, running a linter, checking whether a push landed. At 606k average context re-read per turn that is ~2.7B cache-read tokens on turns a script could have taken. - turns.py classifies a command as poll / lint_test / vcs_check / substantive, deliberately conservative: state-changing commands (git commit, push) count as substantive even though they are scriptable, so the reported waste is a floor. A turn is only mechanical when EVERY command in it is. - Batching is grouped by requestId, not by transcript line: one API response can span several lines, and counting per line reports every turn as single-tool (0% batching where the real rate is 12%). - records.py gains TurnEfficiency (schema v3, additive — old records read as zeros) with derived rates written into to_dict so readers need not recompute. - report.py gains a "Turn efficiency" section pricing each mechanical kind. - _TurnTracker owns both the totals and the open turn, after pylint flagged _Accumulator over its attribute budget; _Accumulator.turns is now a property, so every existing caller is unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- ...on-autopsy-turn-efficiency-2026-07-26.json | 17 +++ ...on-autopsy-turn-efficiency-2026-07-26.json | 42 +++++++ python_pkg/session_autopsy/parse.py | 95 ++++++++++++++- python_pkg/session_autopsy/records.py | 66 ++++++++++- python_pkg/session_autopsy/report.py | 49 +++++++- .../session_autopsy/tests/test_parse.py | 49 ++++++++ .../session_autopsy/tests/test_report.py | 28 ++++- .../session_autopsy/tests/test_turns.py | 71 ++++++++++++ python_pkg/session_autopsy/turns.py | 109 ++++++++++++++++++ 9 files changed, 521 insertions(+), 5 deletions(-) create mode 100644 docs/superpowers/contracts/session-autopsy-turn-efficiency-2026-07-26.json create mode 100644 docs/superpowers/evidence/session-autopsy-turn-efficiency-2026-07-26.json create mode 100644 python_pkg/session_autopsy/tests/test_turns.py create mode 100644 python_pkg/session_autopsy/turns.py diff --git a/docs/superpowers/contracts/session-autopsy-turn-efficiency-2026-07-26.json b/docs/superpowers/contracts/session-autopsy-turn-efficiency-2026-07-26.json new file mode 100644 index 00000000..b9ab2ef3 --- /dev/null +++ b/docs/superpowers/contracts/session-autopsy-turn-efficiency-2026-07-26.json @@ -0,0 +1,17 @@ +{ + "title": "Measure turn efficiency in session-autopsy", + "objective": "Session-autopsy priced tokens but never counted the API round-trips spending them. Cache read is charged per round-trip on the whole conversation so far, so a long session's cost is driven by turn count rather than message length — one measured session spent 267M cache-read against 856k output. Success is a report section that counts tool turns, their batching rate, and the subset that decided nothing (polling, lint/test, read-only VCS checks), priced in cache-read tokens.", + "acceptance_criteria": [ + "A command classifier labels Bash commands poll / lint_test / vcs_check / substantive, and a turn is only mechanical when every command in it is", + "Batching is grouped by requestId, because one API response can span several transcript lines and per-line counting reports every turn as single-tool", + "SessionRecord carries TurnEfficiency; the schema bump is additive so pre-v3 records deserialize as zeros rather than failing", + "REPORT.md gains a Turn efficiency section pricing each mechanical kind at the measured average context re-read", + "100% branch coverage across the package, ruff + pylint + mypy clean, full pre-commit gate passes with no --no-verify" + ], + "out_of_scope": [ + "Changing how any existing detector or candidate scores", + "Acting on the findings (scripting the polling loops) — this contract only measures", + "Classifying non-Bash tools; a turn that only ran Edit/Read counts as substantive" + ], + "verifier": "pre-commit run --files ; pytest --cov=python_pkg.session_autopsy (100% gate); python -m python_pkg.session_autopsy scan --force && report, checked against the real 263-transcript store" +} diff --git a/docs/superpowers/evidence/session-autopsy-turn-efficiency-2026-07-26.json b/docs/superpowers/evidence/session-autopsy-turn-efficiency-2026-07-26.json new file mode 100644 index 00000000..29fc8608 --- /dev/null +++ b/docs/superpowers/evidence/session-autopsy-turn-efficiency-2026-07-26.json @@ -0,0 +1,42 @@ +{ + "intent": "Show, per session, how many API round-trips were spent on work that decided nothing, so the waste can be scripted away instead of guessed at.", + "scope": [ + "python_pkg/session_autopsy/turns.py (new classifier)", + "python_pkg/session_autopsy/{parse,records,report}.py", + "python_pkg/session_autopsy/tests/{test_turns,test_parse,test_report}.py", + "Non-goal: acting on the findings, or changing existing detector scoring" + ], + "changes": [ + "turns.py classifies a Bash command as poll / lint_test / vcs_check / substantive. Deliberately conservative: state-changing commands (git commit, git push) are substantive even though they are scriptable, and a turn is mechanical only when every command in it is \u2014 so the reported waste is a floor, not a guess.", + "parse.py groups tool calls by requestId rather than by transcript line. One API response can span several lines, and per-line counting reported 0% batching where the true rate is 12%.", + "records.py adds TurnEfficiency (schema v3, additive: pre-v3 records read as zeros) with derived batching/mechanical rates written into to_dict so consumers need not recompute them.", + "report.py adds a Turn efficiency section pricing each mechanical kind at the measured average context re-read per turn.", + "Turn state and totals were folded into a _TurnTracker class after pylint flagged _Accumulator at 15/12 instance attributes. The budget allowed exactly one new attribute, which forced the right shape: the open turn exists only to be folded into the totals, so they belong in one object. _Accumulator.turns is now a property, keeping every existing caller and test unchanged." + ], + "verification": [ + { + "command": "pytest python_pkg/session_autopsy/tests --cov=python_pkg.session_autopsy", + "result": "pass", + "evidence": "112 passed; every module in the package at 100% statement and branch coverage, including the new turns.py (25 stmts / 12 branches)." + }, + { + "command": "ruff check && ruff format --check && pylint && mypy", + "result": "pass", + "evidence": "ruff: All checks passed, formatted. pylint 10.00/10. mypy clean. 112 tests, every module at 100% statement and branch coverage." + }, + { + "command": "python -m python_pkg.session_autopsy scan --force && report", + "result": "pass", + "evidence": "263 transcripts analyzed in 1.8s. Report: 33,508 tool turns carrying 37,411 calls, batched 2,595 (8%), avg context re-read 606k/turn; mechanical 4,437 turns (13%) ~= 2.7B cache-read tokens (poll 891, lint/test 2,044, vcs 1,502)." + } + ], + "risks": [ + "The classifier is regex-based, so an unusual command shape can be misfiled. Mitigated by the conservative rule (mixed turns count as substantive), which biases the number down rather than up.", + "Grouping by requestId depends on that field being present; lines without one fall back to the message id, and turns with neither collapse together. Only affects batching precision, not the mechanical counts.", + "Schema v3 records are read by older code as unknown keys, which record_from_dict already ignores; no migration is needed." + ], + "rollback": [ + "git revert the commit; TurnEfficiency is additive so stored v3 records remain readable by the reverted code, which ignores the extra key.", + "Re-run scan --force && report and confirm REPORT.md renders without the Turn efficiency section." + ] +} diff --git a/python_pkg/session_autopsy/parse.py b/python_pkg/session_autopsy/parse.py index 8c8cac5a..8b3fd67b 100644 --- a/python_pkg/session_autopsy/parse.py +++ b/python_pkg/session_autopsy/parse.py @@ -7,6 +7,7 @@ parse result outlives the accumulator fields it updates. from __future__ import annotations from collections import Counter +from dataclasses import dataclass, field from datetime import datetime, timezone import json import re @@ -19,8 +20,10 @@ from python_pkg.session_autopsy.records import ( SessionRecord, SkillInvocation, TokenTotals, + TurnEfficiency, ) from python_pkg.session_autopsy.signatures import command_signature, normalize_signature +from python_pkg.session_autopsy.turns import LINT_TEST, POLL, VCS_CHECK, classify_turn if TYPE_CHECKING: from pathlib import Path @@ -83,6 +86,69 @@ _GENERIC_ERROR_SIGS = frozenset( ) +@dataclass +class _OpenTurn: + """The API turn currently being accumulated, keyed by requestId.""" + + request_id: str | None = None + tools: int = 0 + commands: list[str] = field(default_factory=list) + + +class _TurnTracker: + """Running turn-efficiency totals plus the turn currently open. + + The two belong together: the open turn only exists to be folded into the + totals, and keeping them as one object is also what keeps ``_Accumulator`` + inside its instance-attribute budget. + """ + + def __init__(self) -> None: + """Start with empty totals and no open turn.""" + self.totals = TurnEfficiency() + self._open = _OpenTurn() + + def begin(self, request_id: object) -> None: + """Open a new turn if this line belongs to a different API response. + + Args: + request_id: The line's ``requestId`` (or message-id fallback). + """ + if request_id != self._open.request_id: + self.flush() + self._open.request_id = request_id if isinstance(request_id, str) else None + + def add_tool(self) -> None: + """Count one tool_use block in the open turn.""" + self._open.tools += 1 + + def add_command(self, command: str) -> None: + """Record one Bash command issued in the open turn. + + Args: + command: The raw Bash tool input. + """ + self._open.commands.append(command) + + def flush(self) -> None: + """Fold the open turn into the totals and start a fresh one.""" + open_turn = self._open + self._open = _OpenTurn() + if open_turn.tools <= 0: + return + self.totals.api_turns += 1 + self.totals.tool_calls += open_turn.tools + if open_turn.tools > 1: + self.totals.batched_turns += 1 + label = classify_turn(open_turn.commands) + if label == POLL: + self.totals.poll_turns += 1 + elif label == LINT_TEST: + self.totals.lint_test_turns += 1 + elif label == VCS_CHECK: + self.totals.vcs_check_turns += 1 + + class _Accumulator: """Mutable per-session state fed one transcript line at a time.""" @@ -99,6 +165,15 @@ class _Accumulator: self.skill_invocations: list[SkillInvocation] = [] self._current_run: list[str] = [] self._current_skill: SkillInvocation | None = None + # One API response can span several transcript lines, so batching has to be + # grouped by requestId — counting per line would report every turn as + # single-tool and hide the real (much lower) batching rate. + self._turns = _TurnTracker() + + @property + def turns(self) -> TurnEfficiency: + """Turn-efficiency totals accumulated so far.""" + return self._turns.totals def feed_line(self, raw_line: str) -> None: """Parse and account one transcript line. @@ -120,9 +195,10 @@ class _Accumulator: self._dispatch(obj) def finish(self) -> None: - """Flush any still-open Bash run and skill span.""" + """Flush any still-open Bash run, skill span and API turn.""" self._flush_run() self._flush_skill() + self._turns.flush() def _dispatch(self, obj: dict[str, object]) -> None: """Route a parsed line by its ``type``. @@ -180,6 +256,7 @@ class _Accumulator: return self._note_metadata(obj) self.counts.assistant_msgs += 1 + self._turns.begin(obj.get("requestId") or _message_id(message)) usage = message.get("usage") if isinstance(usage, dict): self.tokens.add_usage(usage) @@ -217,6 +294,7 @@ class _Accumulator: text_chars += len(text.strip()) elif block_type == "tool_use": had_tool_use = True + self._turns.add_tool() text_chars += _generated_chars(block) self._on_tool_use(block) return (text_chars, had_tool_use) @@ -250,6 +328,7 @@ class _Accumulator: command = tool_input.get("command") if not isinstance(command, str): return + self._turns.add_command(command) first_line = normalize_signature(command) if first_line: self.bash_first_lines[first_line] += 1 @@ -447,6 +526,7 @@ def parse_session(transcript_path: Path) -> SessionRecord: meta=acc.meta, counts=acc.counts, tokens=acc.tokens, + turns=acc.turns, obs=Observations( tool_histogram=dict(acc.tool_histogram), bash_first_lines=_top(acc.bash_first_lines, MAX_BASH_FIRST_LINES), @@ -456,3 +536,16 @@ def parse_session(transcript_path: Path) -> SessionRecord: typed_prompt_signatures=_top(acc.typed_prompt_signatures, MAX_PROMPT_SIGS), ), ) + + +def _message_id(message: dict[str, object]) -> str | None: + """Fallback turn key for lines that carry no ``requestId``. + + Args: + message: The ``message`` object of an assistant line. + + Returns: + The message id when present, else None. + """ + value = message.get("id") + return value if isinstance(value, str) else None diff --git a/python_pkg/session_autopsy/records.py b/python_pkg/session_autopsy/records.py index 65222104..4767e668 100644 --- a/python_pkg/session_autopsy/records.py +++ b/python_pkg/session_autopsy/records.py @@ -4,7 +4,7 @@ from __future__ import annotations from dataclasses import asdict, dataclass, field -SCHEMA_VERSION = 2 +SCHEMA_VERSION = 3 @dataclass @@ -28,6 +28,39 @@ class TokenTotals: self.cache_write += _as_int(usage.get("cache_creation_input_tokens")) +@dataclass +class TurnEfficiency: + """API round-trips taken, and how many of them a script could have taken. + + Cache-read tokens dominate a long session (one measured session: 235M read + against 800k output), and cache read is charged per round-trip on the whole + conversation so far. So the cost lever is the NUMBER of turns, not the length + of any message — and turns where the model decided nothing are pure waste. + """ + + api_turns: int = 0 + tool_calls: int = 0 + batched_turns: int = 0 + poll_turns: int = 0 + lint_test_turns: int = 0 + vcs_check_turns: int = 0 + + @property + def mechanical_turns(self) -> int: + """Turns whose every action was observation a script could have done.""" + return self.poll_turns + self.lint_test_turns + self.vcs_check_turns + + @property + def batching_rate(self) -> float: + """Share of tool-carrying turns that issued more than one call (0..1).""" + return self.batched_turns / self.api_turns if self.api_turns else 0.0 + + @property + def mechanical_rate(self) -> float: + """Share of tool-carrying turns that were purely mechanical (0..1).""" + return self.mechanical_turns / self.api_turns if self.api_turns else 0.0 + + @dataclass class SessionMeta: """Where and when the session happened.""" @@ -90,6 +123,7 @@ class SessionRecord: meta: SessionMeta = field(default_factory=SessionMeta) counts: ActivityCounts = field(default_factory=ActivityCounts) tokens: TokenTotals = field(default_factory=TokenTotals) + turns: TurnEfficiency = field(default_factory=TurnEfficiency) obs: Observations = field(default_factory=Observations) def to_dict(self) -> dict[str, object]: @@ -98,7 +132,15 @@ class SessionRecord: Returns: A plain dict with nested dataclasses expanded. """ - return asdict(self) + data = asdict(self) + # asdict() only walks fields, so the derived rates are added by hand — + # readers of sessions.jsonl should not have to recompute them. + data["turns"] |= { + "mechanical_turns": self.turns.mechanical_turns, + "batching_rate": round(self.turns.batching_rate, 4), + "mechanical_rate": round(self.turns.mechanical_rate, 4), + } + return data def _meta_from_dict(data: object) -> SessionMeta: @@ -156,6 +198,25 @@ def _tokens_from_dict(data: object) -> TokenTotals: ) +def _turns_from_dict(data: object) -> TurnEfficiency: + """Rebuild turn-efficiency counts from their serialized form. + + Args: + data: The ``turns`` value of a stored record. + + Returns: + The reconstructed counts (zeros on junk or on pre-v3 records). + """ + return TurnEfficiency( + api_turns=_as_int(_dict_get(data, "api_turns")), + tool_calls=_as_int(_dict_get(data, "tool_calls")), + batched_turns=_as_int(_dict_get(data, "batched_turns")), + poll_turns=_as_int(_dict_get(data, "poll_turns")), + lint_test_turns=_as_int(_dict_get(data, "lint_test_turns")), + vcs_check_turns=_as_int(_dict_get(data, "vcs_check_turns")), + ) + + def _obs_from_dict(data: object) -> Observations: """Rebuild the observations block from its serialized form. @@ -211,6 +272,7 @@ def record_from_dict(data: dict[str, object]) -> SessionRecord: meta=_meta_from_dict(data.get("meta")), counts=_counts_from_dict(data.get("counts")), tokens=_tokens_from_dict(data.get("tokens")), + turns=_turns_from_dict(data.get("turns")), obs=_obs_from_dict(data.get("obs")), ) diff --git a/python_pkg/session_autopsy/report.py b/python_pkg/session_autopsy/report.py index 2a6ea8c0..c869bc17 100644 --- a/python_pkg/session_autopsy/report.py +++ b/python_pkg/session_autopsy/report.py @@ -254,7 +254,11 @@ def render_report( The Markdown document. """ handled = handled_ids(compiled) - parts = [_header(records, now), _candidate_table(result.candidates, handled)] + parts = [ + _header(records, now), + _turn_efficiency(records), + _candidate_table(result.candidates, handled), + ] parts.extend( _candidate_detail(candidate) for candidate in result.candidates[:DETAIL_TOP] ) @@ -293,6 +297,49 @@ def _header(records: list[SessionRecord], now: datetime) -> str: ) +def _turn_efficiency(records: list[SessionRecord]) -> str: + """Render the turn-efficiency block. + + Cache read is charged per API round-trip on the whole conversation so far, so + turns the model did not need to take are the single largest recoverable cost — + far larger than message length. This block prices them. + + Args: + records: All session records. + + Returns: + The turn-efficiency section. + """ + api = sum(r.turns.api_turns for r in records) + if not api: + return "## Turn efficiency\n\nno tool turns recorded yet\n" + batched = sum(r.turns.batched_turns for r in records) + poll = sum(r.turns.poll_turns for r in records) + lint = sum(r.turns.lint_test_turns for r in records) + vcs = sum(r.turns.vcs_check_turns for r in records) + calls = sum(r.turns.tool_calls for r in records) + cache_read = sum(r.tokens.cache_read for r in records) + mechanical = poll + lint + vcs + per_turn = cache_read // api if api else 0 + return ( + "## Turn efficiency\n\n" + f"{api} tool turns carrying {calls} calls | " + f"batched: {batched} ({batched / api:.0%}) | " + f"avg context re-read: {fmt_tokens(per_turn)}/turn\n\n" + "| mechanical turn kind | turns | est. cache-read |\n" + "| --- | ---: | ---: |\n" + f"| polling (progress, liveness, clocks) | {poll} " + f"| {fmt_tokens(poll * per_turn)} |\n" + f"| lint / test (exit-code adjudicated) | {lint} " + f"| {fmt_tokens(lint * per_turn)} |\n" + f"| read-only VCS checks | {vcs} | {fmt_tokens(vcs * per_turn)} |\n" + f"| **total** | **{mechanical}** ({mechanical / api:.0%}) | " + f"**{fmt_tokens(mechanical * per_turn)}** |\n\n" + "These turns decided nothing: a script could have taken them and reported " + "once. Batching independent calls cuts the rest.\n" + ) + + def _action_cell(cand: Candidate, handled: set[str]) -> str: """The table's action column for one candidate. diff --git a/python_pkg/session_autopsy/tests/test_parse.py b/python_pkg/session_autopsy/tests/test_parse.py index 9ebb40cb..6ea7350b 100644 --- a/python_pkg/session_autopsy/tests/test_parse.py +++ b/python_pkg/session_autopsy/tests/test_parse.py @@ -305,3 +305,52 @@ def test_parse_session_without_subagents(tmp_path: Path) -> None: rec = parse_session(main) assert rec.counts.subagent_count == 0 assert subagent_files(main) == [] + + +def test_turn_efficiency_groups_by_request_id() -> None: + """One API response can span several transcript lines. + + Counting per line would report every turn as single-tool and hide the real + batching rate, so batching is grouped by requestId. + """ + acc = _Accumulator() + first = assistant_line(bash_block("wc -l < /tmp/scrape.log")) + first["requestId"] = "req-1" + second = assistant_line(bash_block("tail -3 /tmp/scrape.log")) + second["requestId"] = "req-1" # same API response, two tool calls + third = assistant_line(bash_block("pytest -q")) + third["requestId"] = "req-2" + _feed(acc, first, second, third) + acc.finish() + + assert acc.turns.api_turns == 2 + assert acc.turns.tool_calls == 3 + assert acc.turns.batched_turns == 1 # req-1 carried two calls + assert acc.turns.batching_rate == 0.5 + assert acc.turns.poll_turns == 1 # req-1 was pure log watching + assert acc.turns.lint_test_turns == 1 # req-2 was the test gate + assert acc.turns.mechanical_turns == 2 + + +def test_turn_efficiency_ignores_turns_without_tools() -> None: + acc = _Accumulator() + prose = assistant_line({"type": "text", "text": "just talking"}) + prose["requestId"] = "req-9" + _feed(acc, prose) + acc.finish() + assert acc.turns.api_turns == 0 + assert acc.turns.batching_rate == 0.0 + assert acc.turns.mechanical_rate == 0.0 + + +def test_turn_efficiency_counts_vcs_checks_and_real_work_apart() -> None: + acc = _Accumulator() + check = assistant_line(bash_block("git ls-remote origin refs/heads/main")) + check["requestId"] = "req-a" + work = assistant_line(bash_block("./run.sh --profile kcd index")) + work["requestId"] = "req-b" + _feed(acc, check, work) + acc.finish() + assert acc.turns.vcs_check_turns == 1 + assert acc.turns.mechanical_turns == 1 # the index build is real work + assert acc.turns.api_turns == 2 diff --git a/python_pkg/session_autopsy/tests/test_report.py b/python_pkg/session_autopsy/tests/test_report.py index a9d28f56..a12bfb2e 100644 --- a/python_pkg/session_autopsy/tests/test_report.py +++ b/python_pkg/session_autopsy/tests/test_report.py @@ -8,7 +8,12 @@ from typing import TYPE_CHECKING from python_pkg.session_autopsy import report from python_pkg.session_autopsy.detectors import Candidate, analyze -from python_pkg.session_autopsy.records import ActivityCounts, Observations +from python_pkg.session_autopsy.records import ( + ActivityCounts, + Observations, + TokenTotals, + TurnEfficiency, +) from python_pkg.session_autopsy.tests.conftest import ( invocation, record, @@ -212,3 +217,24 @@ def test_state_file_is_valid_json(tmp_path: Path) -> None: payload = json.loads((tmp_path / report.STATE_FILE).read_text(encoding="utf-8")) assert payload["candidate_ids"] == ["x"] assert payload["last_generated"].startswith("2026-07-24") + + +def test_turn_efficiency_section_prices_mechanical_turns() -> None: + """The point of the section: put a token number on turns nobody needed.""" + rec = record("s1", tokens=TokenTotals(cache_read=100_000_000)) + rec.turns = TurnEfficiency( + api_turns=100, + tool_calls=120, + batched_turns=12, + poll_turns=20, + lint_test_turns=10, + vcs_check_turns=5, + ) + text = report._turn_efficiency([rec]) + assert "100 tool turns carrying 120 calls" in text + assert "batched: 12 (12%)" in text + assert "**35** (35%)" in text # 20 + 10 + 5 mechanical turns + + +def test_turn_efficiency_section_without_tool_turns() -> None: + assert "no tool turns recorded yet" in report._turn_efficiency([record("s1")]) diff --git a/python_pkg/session_autopsy/tests/test_turns.py b/python_pkg/session_autopsy/tests/test_turns.py new file mode 100644 index 00000000..ca654be3 --- /dev/null +++ b/python_pkg/session_autopsy/tests/test_turns.py @@ -0,0 +1,71 @@ +"""Tests for classifying turns a script could have taken instead of the model.""" + +from __future__ import annotations + +import pytest + +from python_pkg.session_autopsy.turns import ( + LINT_TEST, + POLL, + SUBSTANTIVE, + VCS_CHECK, + classify_command, + classify_turn, +) + + +@pytest.mark.parametrize( + ("command", "expected"), + [ + # Waiting on something. + ("kill -0 262023 2>/dev/null && echo ALIVE", POLL), + ("pgrep -af claude | head", POLL), + ("ps -eo pid,pgid,stat,args | grep python", POLL), + ("wc -l < /tmp/scrape.log", POLL), + ("tail -5 /tmp/scrape.log", POLL), + ("nvidia-smi --query-gpu=utilization.gpu --format=csv", POLL), + ("sleep 30", POLL), + ("until grep -q DONE log; do sleep 5; done", POLL), + ("date '+%H:%M'", POLL), + # Adjudicated by an exit code. + ("pytest -q --cov=wikikb", LINT_TEST), + (".venv/bin/python -m ruff check wikikb", LINT_TEST), + ("pre-commit run --all-files", LINT_TEST), + ("shellcheck scripts/safepush.sh", LINT_TEST), + # Read-only VCS inspection. + ("git status --short", VCS_CHECK), + ("git ls-remote origin refs/heads/main", VCS_CHECK), + ("git rev-parse HEAD", VCS_CHECK), + ("gh api repos/kuhyx/wiki-kb --jq .visibility", VCS_CHECK), + # Real work: anything that changes state, fetches, or builds. + ("./run.sh --profile kcd index", SUBSTANTIVE), + ("git commit -q -m 'thing'", SUBSTANTIVE), + ("git push origin main", SUBSTANTIVE), + ("curl -sS https://example.com/api.php", SUBSTANTIVE), + ("", SUBSTANTIVE), + (" ", SUBSTANTIVE), + ], +) +def test_classify_command(command: str, expected: str) -> None: + assert classify_command(command) == expected + + +def test_commit_and_push_are_substantive_even_though_scriptable() -> None: + # They change state, so a turn carrying them was not pure observation. Counting + # them as waste would overstate the finding, and an overstated report is worse + # than none. + assert classify_command("git add -A && git commit -m x") == SUBSTANTIVE + + +def test_classify_turn_requires_every_command_to_be_mechanical() -> None: + assert classify_turn(["git status", "git rev-parse HEAD"]) == VCS_CHECK + assert classify_turn(["pytest -q"]) == LINT_TEST + # Mixed kinds: still mechanical individually, but the turn is not one thing. + assert classify_turn(["git status", "pytest -q"]) == SUBSTANTIVE + # One real action makes the round-trip necessary regardless of what else it did. + assert classify_turn(["wc -l < log", "./run.sh index"]) == SUBSTANTIVE + + +def test_classify_turn_with_no_bash_is_substantive() -> None: + # A turn that ran Edit/Write/Read rather than Bash is not a polling turn. + assert classify_turn([]) == SUBSTANTIVE diff --git a/python_pkg/session_autopsy/turns.py b/python_pkg/session_autopsy/turns.py new file mode 100644 index 00000000..956ef1c6 --- /dev/null +++ b/python_pkg/session_autopsy/turns.py @@ -0,0 +1,109 @@ +"""Classify assistant turns by whether a model was needed to take them. + +Cache-read tokens dominate the cost of a long session — one measured session spent +235M cache-read against 800k output, a ratio of roughly 290:1. Cache read is paid per +API round-trip on the whole conversation so far, so the lever is the NUMBER OF TURNS, +not the length of any message. A short turn taken late is as expensive as a long one. + +That makes it worth counting the turns where the model decided nothing: polling a +counter, running a linter, checking whether a push landed. Those are pass/fail +mechanisms that code can adjudicate, which is exactly what belongs in a script rather +than in a round-trip (see ``rules/token-spend.instructions.md``). + +Classification is deliberately conservative: a command is only called mechanical when +its whole purpose is observation. Anything that edits, fetches, builds or decides +counts as substantive, so the reported waste is a floor rather than a guess. +""" + +from __future__ import annotations + +import re + +POLL = "poll" +LINT_TEST = "lint_test" +VCS_CHECK = "vcs_check" +SUBSTANTIVE = "substantive" + +MECHANICAL = frozenset({POLL, LINT_TEST, VCS_CHECK}) + +# Ordered: the first pattern to match wins, so put the narrow ones first. +_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = ( + # Waiting on something: process liveness, log tails, progress counters, clocks. + ( + POLL, + re.compile( + r"""(?x) + \bkill\s+-0\b # is that pid still alive + | \bpgrep\b | \bpidof\b + | \bps\s+-[eo] # ps -eo / ps -o : process listing + | \bnvidia-smi\b + | \bwc\s+-l\s*< # counting lines in a log + | \btail\s+(-[nf]?\d*\s+)?\S*\.log + | ^\s*sleep\s + | \buntil\s+.*\bdo\b.*\bdone\b + | \bdate\s+['\"]?\+ + """ + ), + ), + # Adjudicated by an exit code, never by a model. + ( + LINT_TEST, + re.compile( + r"\b(pytest|ruff|mypy|shellcheck|pre-commit\s+run|npm\s+test|cargo\s+test)\b" + ), + ), + # Read-only VCS inspection. `git commit`/`push`/`add` are excluded on purpose: + # they change state, so they are substantive even though they are scriptable. + ( + VCS_CHECK, + re.compile( + r"""(?x) + \bgit\s+(status|log|diff|ls-remote|rev-parse|remote\s+-v|branch|show)\b + | \bgh\s+(api|repo\s+view|pr\s+view)\b + """ + ), + ), +) + + +def classify_command(command: str) -> str: + """Classify one Bash command. + + Args: + command: The raw Bash tool input (may be multi-line). + + Returns: + One of :data:`POLL`, :data:`LINT_TEST`, :data:`VCS_CHECK` or + :data:`SUBSTANTIVE`. + """ + text = command.strip() + if not text: + return SUBSTANTIVE + for label, pattern in _PATTERNS: + if pattern.search(text): + return label + return SUBSTANTIVE + + +def classify_turn(commands: list[str]) -> str: + """Classify a whole turn from the Bash commands it issued. + + A turn only counts as mechanical when *every* command in it is — one real piece + of work makes the round-trip necessary regardless of what else it carried, and + over-counting waste would make the report untrustworthy. + + Args: + commands: Every Bash command issued in this turn, in order. + + Returns: + The shared mechanical label, or :data:`SUBSTANTIVE` if the turn mixed kinds, + carried real work, or ran no Bash at all. + """ + if not commands: + return SUBSTANTIVE + labels = {classify_command(c) for c in commands} + if len(labels) == 1: + only = labels.pop() + if only in MECHANICAL: + return only + return SUBSTANTIVE