diff --git a/docs/superpowers/contracts/token-audit-labels-and-bash-metric-2026-10-10.json b/docs/superpowers/contracts/token-audit-labels-and-bash-metric-2026-10-10.json new file mode 100644 index 00000000..8bdcb5c1 --- /dev/null +++ b/docs/superpowers/contracts/token-audit-labels-and-bash-metric-2026-10-10.json @@ -0,0 +1,15 @@ +{ + "title": "token_audit: label every subagent; Bash metric per day + big-result share", + "objective": "The weekly audit must attribute every subagent dollar to a task label (no '?'/'unknown' rows) and must not report a Bash-output regression that is only a shrinking call count. Success: the 2026-10-04..10-10 window renders with no '?'/'unknown' subagent rows, and the snapshot compares bash_tokens_per_day and bash_big_result_share instead of bash_tokens_per_call.", + "acceptance_criteria": [ + "Subagent files continued in a post-/clear session borrow the spawning session's agent-.meta.json; otherwise the first prompt line labels them; never '?'", + "extra_metrics emits bash_tokens_per_day and bash_big_result_share (results > 2000 tokens); COMPARED uses them", + "The Large-Bash lever keys on bash_big_result_share > 0.25", + "token_audit tests pass at 100% branch coverage" + ], + "out_of_scope": [ + "De-duplicating the 8 API calls that appear in both the original and continuation subagent file", + "A PreToolUse guard against whole-file cat/sed (needs settings.json, user's call)" + ], + "verifier": "python3 -m pytest python_pkg/token_audit --cov=python_pkg/token_audit --cov-branch; python3 -m python_pkg.token_audit --since <10-04 14:13 UTC> --no-write --recommend" +} diff --git a/docs/superpowers/evidence/token-audit-subagent-labels-2026-10-10.json b/docs/superpowers/evidence/token-audit-subagent-labels-2026-10-10.json new file mode 100644 index 00000000..fd972689 --- /dev/null +++ b/docs/superpowers/evidence/token-audit-subagent-labels-2026-10-10.json @@ -0,0 +1,31 @@ +{ + "intent": "Every subagent in the token audit carries a task label and type; no '?' / 'unknown' rows in the subagent tables.", + "scope": [ + "python_pkg/token_audit/discover.py, python_pkg/token_audit/parse.py and their tests", + "Non-goal: no hook or settings.json change; every Agent launch already records a description" + ], + "changes": [ + "agent_meta borrows a missing description/type from the same agent-.meta.json in another session dir (background agents that outlive /clear continue in the new session's subagents/ with no meta)", + "load_session falls back to the first user prompt line (60 chars), then '(no label) agent-', instead of '?'" + ], + "verification": [ + { + "command": "python3 -m pytest -q python_pkg/token_audit/tests --cov=python_pkg.token_audit --cov-branch", + "result": "pass", + "evidence": "200 passed, 100.00% branch coverage" + }, + { + "command": "python3 -m python_pkg.token_audit --since 1791123199.748 --until 1791627950.749 --no-write", + "result": "pass", + "evidence": "Same window as the 2026-10-10 snapshot: 'unknown' ($35.23) and '?' ($35.93) rows gone; all 14 affected files labelled from the spawning session's meta" + } + ], + "risks": [ + "Agent ids colliding across sessions would borrow the wrong label (ids are 17 random hex chars; none seen across 479 files)", + "One extra glob over projects/*/*/subagents per unlabelled subagent file" + ], + "rollback": [ + "git revert the commit", + "Re-run the audit with --no-write and confirm the old '?' row returns" + ] +} diff --git a/python_pkg/token_audit/__main__.py b/python_pkg/token_audit/__main__.py index 382f50e5..dbe2c605 100644 --- a/python_pkg/token_audit/__main__.py +++ b/python_pkg/token_audit/__main__.py @@ -137,7 +137,9 @@ def main(argv: list[str] | None = None) -> int: priced = breakdown.build(sessions, pricing.load()) snap["v"] = review.SNAPSHOT_VERSION snap["effective_window"] = {"since": priced.earliest, "until": priced.latest} - snap["metrics"] = breakdown.metrics(priced) | review.extra_metrics(totals, axes) + snap["metrics"] = breakdown.metrics(priced) | review.extra_metrics( + totals, axes, priced.days + ) previous = history.last_snapshot(args.out) markdown = report.render(totals, axes, window, sessions, snap, None) markdown += "\n".join( diff --git a/python_pkg/token_audit/attribute.py b/python_pkg/token_audit/attribute.py index 2ccbe590..295f3b03 100644 --- a/python_pkg/token_audit/attribute.py +++ b/python_pkg/token_audit/attribute.py @@ -90,13 +90,26 @@ class Batching: return self.calls / self.messages if self.messages else 0.0 +BIG_RESULT = 2000 + + +@dataclass +class ToolPayloads: + """Tool-result tokens and call counts per tool name.""" + + tokens: Counter[str] = field(default_factory=Counter) + calls: Counter[str] = field(default_factory=Counter) + # Tokens from results over BIG_RESULT: the part a range/head would have + # cut (whole-file cat/sed dumps). + big_tokens: Counter[str] = field(default_factory=Counter) + + @dataclass class Axes: """Every ranking the report needs, in one pass over the sessions.""" project: Counter[str] = field(default_factory=Counter) - tool_tokens: Counter[str] = field(default_factory=Counter) - tool_calls: Counter[str] = field(default_factory=Counter) + tools: ToolPayloads = field(default_factory=ToolPayloads) mcp_calls: Counter[str] = field(default_factory=Counter) skills: Counter[str] = field(default_factory=Counter) models: Counter[str] = field(default_factory=Counter) @@ -155,8 +168,10 @@ def _add_batching(axes: Axes, session: Session) -> None: def _add_tools(axes: Axes, session: Session) -> None: """Attribute tool-result payloads to tools, MCP servers and skills.""" for call in session.tools: - axes.tool_tokens[call.name] += call.result_tokens - axes.tool_calls[call.name] += 1 + axes.tools.tokens[call.name] += call.result_tokens + axes.tools.calls[call.name] += 1 + if call.result_tokens > BIG_RESULT: + axes.tools.big_tokens[call.name] += call.result_tokens server = call.mcp_server if server is not None: axes.mcp_calls[server] += 1 diff --git a/python_pkg/token_audit/discover.py b/python_pkg/token_audit/discover.py index eb103c48..01c52ad9 100644 --- a/python_pkg/token_audit/discover.py +++ b/python_pkg/token_audit/discover.py @@ -7,11 +7,18 @@ Two layouts hold billed turns: ``Agent`` call, with a sibling ``agent-.meta.json`` naming the agentType and description. Before 2026-10-04 the audit globbed only the first layout and reported subagents at 0.0% while they were 35% of real spend. + +A background agent outlives ``/clear``: the rest of its transcript lands in the +*new* session's ``subagents/`` under the same ``agent-`` name, starting +mid-conversation and with no ``.meta.json`` (or one without a description). +:func:`agent_meta` borrows the label from the spawning session's copy; on +2026-10-10 that was all 14 unlabelled files, 5.2% of a week's spend. """ from __future__ import annotations import json +import re from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -19,6 +26,10 @@ if TYPE_CHECKING: PATTERNS: tuple[str, ...] = ("*/*.jsonl", "*/*/subagents/agent-*.jsonl") +# Harness-injected ``...`` blocks (````, +# ````): identical across agents, so never a useful label. +WRAPPER = re.compile(r"<([\w-]+)[^>]*>.*?", re.DOTALL) + def find_transcripts(root: Path, since: float) -> list[Path]: """Return transcripts that may hold records after ``since``, newest first. @@ -59,9 +70,58 @@ def first_cwd(path: Path) -> str | None: return None -def agent_meta(path: Path) -> dict[str, Any]: - """Return a subagent transcript's sibling ``.meta.json``, or ``{}``.""" - meta = path.with_suffix(".meta.json") +def _meta_file(meta: Path) -> dict[str, Any]: + """Read one ``.meta.json``, ``{}`` when missing or unreadable.""" if not meta.exists(): return {} return _record(meta.read_text(encoding="utf-8", errors="replace")) or {} + + +def agent_meta(path: Path) -> dict[str, Any]: + """Return a subagent transcript's metadata, or ``{}``. + + The sibling ``.meta.json`` wins; any field it lacks is filled from the same + agent's meta in another session directory (see the module docstring). + """ + own = _meta_file(path.with_suffix(".meta.json")) + if own.get("description"): + return own + projects = path.parents[3] + for other in sorted(projects.glob(f"*/*/subagents/{path.stem}.meta.json")): + borrowed = _meta_file(other) + if borrowed.get("description"): + return {**borrowed, **{k: v for k, v in own.items() if v}} + return own + + +def _prompt_text(record: dict[str, Any]) -> str | None: + """The typed text of a ``user`` record, ignoring tool results.""" + if record.get("type") != "user": + return None + message = record.get("message") + content = message.get("content") if isinstance(message, dict) else None + if isinstance(content, str): + return content + if isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + return str(block.get("text") or "") + return None + + +def first_prompt(path: Path, limit: int = 60) -> str: + """First non-blank line of a transcript's first user prompt, truncated. + + The label of last resort for a subagent nobody described: better than + ``?`` because it names the work, and an empty result means none exists. + """ + with path.open(encoding="utf-8", errors="replace") as handle: + for line in handle: + record = _record(line) + text = _prompt_text(record) if record is not None else None + text = WRAPPER.sub("", text or "") + lines = [part.strip() for part in text.splitlines()] + first = next((part for part in lines if part), "") + if first: + return first if len(first) <= limit else first[: limit - 1] + "…" + return "" diff --git a/python_pkg/token_audit/parse.py b/python_pkg/token_audit/parse.py index e768297a..86e1c9d9 100644 --- a/python_pkg/token_audit/parse.py +++ b/python_pkg/token_audit/parse.py @@ -182,7 +182,8 @@ def load_session(path: Path, window: records.Window | None = None) -> Session: A subagent transcript (``/subagents/agent-*.jsonl``) is tagged with its parent session id and the ``agentType``/``description`` the - spawning ``Agent`` call recorded in the sibling ``.meta.json``. + spawning ``Agent`` call recorded in the sibling ``.meta.json``. With no + description anywhere, the first prompt line stands in, then the agent id. """ session = Session( session_id=path.stem, path=str(path), cwd=discover.first_cwd(path) @@ -191,7 +192,11 @@ def load_session(path: Path, window: records.Window | None = None) -> Session: meta = discover.agent_meta(path) session.parent_id = path.parent.parent.name session.agent_type = str(meta.get("agentType") or "unknown") - session.description = str(meta.get("description") or "") + session.description = str( + meta.get("description") + or discover.first_prompt(path) + or f"(no label) {path.stem}" + ) for kind, event in iter_events(path, window): if kind == "turn" and isinstance(event, Turn): session.turns.append(event) diff --git a/python_pkg/token_audit/recommend.py b/python_pkg/token_audit/recommend.py index 7f7bb482..f3135cff 100644 --- a/python_pkg/token_audit/recommend.py +++ b/python_pkg/token_audit/recommend.py @@ -19,7 +19,7 @@ SUBAGENT_OPUS_SHARE = 0.5 OVER_BUDGET_SHARE = 0.2 HOME_CWD_SHARE = 0.2 BATCHING_MIN = 1.15 -BASH_TOKENS_PER_CALL = 250 +BASH_BIG_SHARE = 0.25 PREFIX_GROWTH = 0.05 IMAGE_SHARE = 0.03 @@ -108,15 +108,16 @@ def _mechanics(m: dict[str, Any], prev: dict[str, Any] | None) -> list[Lever]: "a whole read-only chain and report once", ), ) - bash = _num(m, "bash_tokens_per_call") - if bash > BASH_TOKENS_PER_CALL: + big = _num(m, "bash_big_result_share") + if big > BASH_BIG_SHARE: out.append( Lever( "agent mechanics", "Large Bash results", - f"{bash:.0f} result tokens per Bash call", - "compress command output (RTK if its A/B won) and keep " - "| head/| tail discipline", + f"{big:.0%} of Bash result tokens come from results over 2k " + f"({_num(m, 'bash_tokens_per_day'):,.0f} tokens/day)", + "read files with ranges (sed -n 'a,bp', grep -n), never cat a " + "whole file; cap every pipeline stage, not just the last", ), ) if prev is not None: diff --git a/python_pkg/token_audit/report.py b/python_pkg/token_audit/report.py index 96c7836f..4f1c8a07 100644 --- a/python_pkg/token_audit/report.py +++ b/python_pkg/token_audit/report.py @@ -157,8 +157,8 @@ def render( lines += ["", "## Tool result payloads (estimated tokens returned)", ""] lines += _table( [ - (name, f"{tok:,} over {axes.tool_calls[name]:,} calls") - for name, tok in axes.tool_tokens.most_common(TOP_N) + (name, f"{tok:,} over {axes.tools.calls[name]:,} calls") + for name, tok in axes.tools.tokens.most_common(TOP_N) ], ("tool", "result tokens"), ) diff --git a/python_pkg/token_audit/review.py b/python_pkg/token_audit/review.py index 0f63186e..1ec0e179 100644 --- a/python_pkg/token_audit/review.py +++ b/python_pkg/token_audit/review.py @@ -27,22 +27,32 @@ COMPARED: tuple[str, ...] = ( "over_budget_usd_share", "home_cwd_usd_share", "batching_calls_per_msg", - "bash_tokens_per_call", + # Per-day + big-result share, not per-call: a per-call mean rose 41% on + # 2026-10-10 with Bash tokens/day flat, only because cheap calls vanished. + "bash_tokens_per_day", + "bash_big_result_share", "standing_tokens_est", ) -def extra_metrics(totals: Totals, axes: Axes) -> dict[str, float]: +def extra_metrics(totals: Totals, axes: Axes, days: float = 0.0) -> dict[str, float]: """Metrics that come from the token axes rather than from dollars.""" - bash_calls = axes.tool_calls.get("Bash", 0) + bash_calls = axes.tools.calls.get("Bash", 0) + bash_tokens = axes.tools.tokens.get("Bash", 0) standing = sum(s.est_tokens for s in surfaces.collect()) return { "batching_calls_per_msg": round(axes.batching.per_message, 3), "bash_tokens_per_call": ( - round(axes.tool_tokens.get("Bash", 0) / bash_calls, 1) + round(axes.tools.tokens.get("Bash", 0) / bash_calls, 1) if bash_calls else 0.0 ), + "bash_tokens_per_day": round(bash_tokens / days, 1) if days else 0.0, + "bash_big_result_share": ( + round(axes.tools.big_tokens["Bash"] / bash_tokens, 4) + if bash_tokens + else 0.0 + ), "image_usd_share": ( round(axes.image_cost / totals.weighted, 4) if totals.weighted else 0.0 ), diff --git a/python_pkg/token_audit/tests/test_attribute.py b/python_pkg/token_audit/tests/test_attribute.py index db6166b7..c596b1f9 100644 --- a/python_pkg/token_audit/tests/test_attribute.py +++ b/python_pkg/token_audit/tests/test_attribute.py @@ -85,8 +85,8 @@ def test_tool_mcp_and_skill_axes() -> None: ToolCall(name="Skill", result_tokens=1, skill="finish"), ] _, axes = attribute.build([_session(tools=tools)]) - assert axes.tool_tokens["Read"] == 15 - assert axes.tool_calls["Read"] == 2 + assert axes.tools.tokens["Read"] == 15 + assert axes.tools.calls["Read"] == 2 assert axes.mcp_calls["aseprite"] == 1 assert axes.skills["finish"] == 1 assert axes.images.count == 1 diff --git a/python_pkg/token_audit/tests/test_discover.py b/python_pkg/token_audit/tests/test_discover.py index f9d7c5eb..1976cf22 100644 --- a/python_pkg/token_audit/tests/test_discover.py +++ b/python_pkg/token_audit/tests/test_discover.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import os from pathlib import Path @@ -49,3 +50,75 @@ def test_agent_meta_empty_when_missing_or_bad(tmp_path: Path) -> None: assert discover.agent_meta(path) == {} _touch(tmp_path / "agent-1.meta.json", 1, "garbage") assert discover.agent_meta(path) == {} + + +def _meta(path: Path, meta: dict[str, object] | str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + text = meta if isinstance(meta, str) else json.dumps(meta) + path.write_text(text, encoding="utf-8") + + +def test_agent_meta_prefers_own_description(tmp_path: Path) -> None: + own = tmp_path / "p" / "s" / "subagents" / "agent-1.meta.json" + _meta(own, {"agentType": "Plan", "description": "mine"}) + _meta( + tmp_path / "p" / "o" / "subagents" / "agent-1.meta.json", {"description": "x"} + ) + path = own.parent / "agent-1.jsonl" + assert discover.agent_meta(path) == {"agentType": "Plan", "description": "mine"} + + +def test_agent_meta_borrows_missing_fields(tmp_path: Path) -> None: + sub = tmp_path / "p" / "new" / "subagents" + _meta(sub / "agent-2.meta.json", {"agentType": "fork", "spawnDepth": 0}) + _meta(tmp_path / "p" / "bad" / "subagents" / "agent-2.meta.json", "not json") + _meta(tmp_path / "p" / "nod" / "subagents" / "agent-2.meta.json", {"x": 1}) + _meta( + tmp_path / "q" / "old" / "subagents" / "agent-2.meta.json", + {"agentType": "general-purpose", "description": "spawned"}, + ) + assert discover.agent_meta(sub / "agent-2.jsonl") == { + "agentType": "fork", + "description": "spawned", + } + + +def test_agent_meta_empty_when_nothing_found(tmp_path: Path) -> None: + path = tmp_path / "p" / "s" / "subagents" / "agent-3.jsonl" + path.parent.mkdir(parents=True) + assert discover.agent_meta(path) == {} + + +def test_first_prompt_skips_non_prompts_and_truncates(tmp_path: Path) -> None: + rows = [ + "junk", + json.dumps({"type": "assistant", "message": {"content": "no"}}), + json.dumps({"type": "user", "message": "flat"}), + json.dumps({"type": "user", "message": {"content": 7}}), + json.dumps( + {"type": "user", "message": {"content": ["s", {"type": "tool_result"}]}} + ), + json.dumps({"type": "user", "message": {"content": " \n"}}), + json.dumps( + { + "type": "user", + "message": {"content": [{"type": "text", "text": "x" * 80}]}, + } + ), + ] + path = _touch(tmp_path / "t.jsonl", 1, "\n".join(rows)) + assert discover.first_prompt(path) == "x" * 59 + "…" + assert discover.first_prompt(path, limit=100) == "x" * 80 + + +def test_first_prompt_text_block_without_text(tmp_path: Path) -> None: + row = {"type": "user", "message": {"content": [{"type": "text"}]}} + path = _touch(tmp_path / "t.jsonl", 1, json.dumps(row)) + assert discover.first_prompt(path) == "" + + +def test_first_prompt_strips_harness_wrappers(tmp_path: Path) -> None: + text = "\nYou are a fork x\n\nFix CI" + row = {"type": "user", "message": {"content": text}} + path = _touch(tmp_path / "t.jsonl", 1, json.dumps(row)) + assert discover.first_prompt(path) == "Fix CI" diff --git a/python_pkg/token_audit/tests/test_parse_subagent.py b/python_pkg/token_audit/tests/test_parse_subagent.py index 3d1491c1..43c4102d 100644 --- a/python_pkg/token_audit/tests/test_parse_subagent.py +++ b/python_pkg/token_audit/tests/test_parse_subagent.py @@ -54,7 +54,36 @@ def test_load_session_subagent_without_meta_defaults(tmp_path: Path) -> None: path.write_text("{}", encoding="utf-8") session = parse.load_session(path) assert session.agent_type == "unknown" - assert session.description == "" + assert session.description == "(no label) agent-b" + + +def test_load_session_borrows_label_across_clear(tmp_path: Path) -> None: + """An agent that outlived ``/clear`` keeps its spawning session's label.""" + old = tmp_path / "proj" / "old" / "subagents" + new = tmp_path / "proj" / "new" / "subagents" + old.mkdir(parents=True) + new.mkdir(parents=True) + (old / "agent-c.meta.json").write_text( + json.dumps({"agentType": "Explore", "description": "map the repo"}), + encoding="utf-8", + ) + path = new / "agent-c.jsonl" + path.write_text("{}", encoding="utf-8") + session = parse.load_session(path) + assert session.agent_type == "Explore" + assert session.description == "map the repo" + + +def test_load_session_falls_back_to_first_prompt(tmp_path: Path) -> None: + sub = tmp_path / "proj" / "sess-3" / "subagents" + sub.mkdir(parents=True) + path = sub / "agent-d.jsonl" + prompt: dict[str, object] = { + "type": "user", + "message": {"content": "\nAudit the hooks\nmore"}, + } + path.write_text(_line(prompt), encoding="utf-8") + assert parse.load_session(path).description == "Audit the hooks" def test_window_filters_records_by_timestamp(tmp_path: Path) -> None: diff --git a/python_pkg/token_audit/tests/test_recommend.py b/python_pkg/token_audit/tests/test_recommend.py index 6a5a55cc..d9f89603 100644 --- a/python_pkg/token_audit/tests/test_recommend.py +++ b/python_pkg/token_audit/tests/test_recommend.py @@ -15,7 +15,8 @@ def _fire_all() -> dict[str, Any]: "over_budget_usd_share": 0.5, "home_cwd_usd_share": 0.4, "batching_calls_per_msg": 1.05, - "bash_tokens_per_call": 900, + "bash_big_result_share": 0.4, + "bash_tokens_per_day": 900_000, "standing_tokens_est": 120, "image_usd_share": 0.1, } diff --git a/python_pkg/token_audit/tests/test_review.py b/python_pkg/token_audit/tests/test_review.py index a58cc1c6..6aad5d58 100644 --- a/python_pkg/token_audit/tests/test_review.py +++ b/python_pkg/token_audit/tests/test_review.py @@ -35,6 +35,8 @@ def test_extra_metrics_without_bash_or_spend() -> None: totals, axes = _axes(bash=False, spend=False) m = review.extra_metrics(totals, axes) assert m["bash_tokens_per_call"] == 0.0 + assert m["bash_tokens_per_day"] == 0.0 + assert m["bash_big_result_share"] == 0.0 assert m["image_usd_share"] == 0.0 @@ -113,3 +115,13 @@ def test_render_with_levers_ignores_old_or_missing_previous( for prev in (None, {"v": 1, "metrics": {"standing_tokens_est": 100}}): text = "\n".join(review.render(snap, prev, levers=True)) assert "Always-loaded context grew" not in text + + +def test_extra_metrics_bash_per_day_and_big_share() -> None: + usage = {"output_tokens": 10} + turn = Turn(usage=usage, context=0, model="claude-opus-5", is_sidechain=False) + tools = [ToolCall("Bash", 3000), ToolCall("Bash", 1000), ToolCall("Read", 5000)] + totals, axes = attribute.build([Session("s", "s.jsonl", turns=[turn], tools=tools)]) + m = review.extra_metrics(totals, axes, days=2.0) + assert m["bash_tokens_per_day"] == 2000.0 + assert m["bash_big_result_share"] == 0.75