diff --git a/docs/superpowers/contracts/token-dashboard-core-2026-10-18.json b/docs/superpowers/contracts/token-dashboard-core-2026-10-18.json new file mode 100644 index 00000000..78877c17 --- /dev/null +++ b/docs/superpowers/contracts/token-dashboard-core-2026-10-18.json @@ -0,0 +1,15 @@ +{ + "title": "token_dashboard: durable token store + ingester (dashboard core)", + "objective": "Ingest Claude Code transcripts and aux sources into a durable, migrated SQLite store that the control-panel /tokens page reads, with an incremental pass under 60 s and 0 LLM tokens.", + "acceptance_criteria": [ + "python_pkg/token_dashboard and python_pkg/token_audit stay at 100% line and branch coverage; every file <= 250 lines", + "python3 -m python_pkg.token_dashboard.ingest --budget 40 finishes in under 60 s and is resumable", + "schema.connect migrates a v1/v2 store forward without deleting rows", + "calls/marks/turn_detail rows are replaced, never duplicated, when a transcript is re-walked" + ], + "out_of_scope": [ + "Remaining /tokens panel groups A-Q", + "Deleting or rebuilding ~/data/token-dashboard/tokens.sqlite" + ], + "verifier": "pytest --cov=python_pkg/token_dashboard --cov=python_pkg/token_audit --cov-branch; the ingest command; pre-commit hooks on the staged files" +} diff --git a/docs/superpowers/evidence/token-dashboard-core-2026-10-18.json b/docs/superpowers/evidence/token-dashboard-core-2026-10-18.json new file mode 100644 index 00000000..a60979aa --- /dev/null +++ b/docs/superpowers/evidence/token-dashboard-core-2026-10-18.json @@ -0,0 +1,40 @@ +{ + "intent": "A durable SQLite token store (~/data/token-dashboard/tokens.sqlite) that outlives Claude Code's transcript cleanup and feeds the control-panel /tokens cost dashboard, with 0 LLM tokens spent.", + "scope": [ + "python_pkg/token_dashboard/ (new): walk, rows, store, schema v3 with ordered additive migrations, finalize, carry, efficiency, ab, levers, context_files, aux, commits, survival, blocks, ingest; detail layer: classify, detail, detail_rows, detail_store, marks, hook_join (per-call, per-mark and per-turn rows with creation + carry cost); ab_rows (per-assignment A/B outcomes)", + "python_pkg/token_audit: persist structured lever candidates (Lever.metric join key) and expose the record primitives the walker shares", + "Non-goal: the remaining dashboard panel groups A-Q (tracked in ~/src/control-panel/TODO-token-dashboard-panels.md)" + ], + "changes": [ + "Resumable incremental ingest: walks changed transcripts until its --budget is spent, commits, exits pending if any remain", + "Schema migrations never drop the store; rows of vanished transcripts are kept", + "token_audit --recommend stores the lever candidates as data in the snapshot (snap['levers'])", + "New raw tables calls / marks / turn_detail (no schema version bump: CREATE IF NOT EXISTS) with key, cwd, model, ts, day, is_sub so the dashboard's filter clause applies; carry.Tracker settles creation (next call's cache-write multiplier x input price) and carry_calls too", + "ab.rebuild also stores per-assignment rows in ab_rows so the A/B panel can recount arms per cwd/model/scope" + ], + "verification": [ + { + "command": "python3 -m pytest -q python_pkg/token_dashboard python_pkg/token_audit --cov=python_pkg/token_dashboard --cov=python_pkg/token_audit --cov-branch", + "result": "pass", + "evidence": "token_dashboard + token_audit: 100.00% line+branch coverage (pytest --cov, --cov-branch)" + }, + { + "command": "python3 -m python_pkg.token_dashboard.ingest --budget 40", + "result": "pass", + "evidence": "token_dashboard: parsed 7 files, refreshed 2 sessions in 0.7s; exit 0" + }, + { + "command": "sqlite3 tokens.sqlite 'DELETE FROM files' && python3 -m python_pkg.token_dashboard.ingest --budget 40", + "result": "pass", + "evidence": "full re-walk of 1616 transcripts in one 32.5 s pass; 114,599 calls, 103,713 turn_detail rows; store kept (backup tokens.v3-2026-10-18.sqlite)" + } + ], + "risks": [ + "A schema change without a MIGRATIONS entry would leave old stores without the new column", + "Transcript format drift in Claude Code could make the walker skip new record types" + ], + "rollback": [ + "git revert this commit; the store file is left untouched and stays readable by the previous code", + "Validate: control-panel /api/tokens/overview still answers from the existing store" + ] +} diff --git a/python_pkg/token_audit/__main__.py b/python_pkg/token_audit/__main__.py index dbe2c605..39bb447f 100644 --- a/python_pkg/token_audit/__main__.py +++ b/python_pkg/token_audit/__main__.py @@ -9,6 +9,7 @@ is worse than no number, because it gets acted on. from __future__ import annotations import argparse +from dataclasses import asdict import json from pathlib import Path import sys @@ -141,13 +142,18 @@ def main(argv: list[str] | None = None) -> int: totals, axes, priced.days ) previous = history.last_snapshot(args.out) + found = review.candidates(snap, previous) if args.recommend else None + if found is not None: + # Persisted so the dashboard reads proposed levers as data instead of + # re-parsing this run's markdown table. + snap["levers"] = [asdict(lever) for lever in found] markdown = report.render(totals, axes, window, sessions, snap, None) markdown += "\n".join( [ "", *breakdown.render(priced), "", - *review.render(snap, previous, levers=args.recommend), + *review.render(snap, previous, levers=args.recommend, found=found), "", ], ) diff --git a/python_pkg/token_audit/parse.py b/python_pkg/token_audit/parse.py index 86e1c9d9..0d43f2d4 100644 --- a/python_pkg/token_audit/parse.py +++ b/python_pkg/token_audit/parse.py @@ -203,3 +203,10 @@ def load_session(path: Path, window: records.Window | None = None) -> Session: elif isinstance(event, ToolCall): session.tools.append(event) return session + + +# Public names for the record primitives, shared with token_dashboard's walker +# (which needs prompt boundaries and timestamps that iter_events drops). +load_line = _load_line +content_blocks = _blocks +pair_result = _pair_result diff --git a/python_pkg/token_audit/recommend.py b/python_pkg/token_audit/recommend.py index f3135cff..621617e0 100644 --- a/python_pkg/token_audit/recommend.py +++ b/python_pkg/token_audit/recommend.py @@ -9,7 +9,7 @@ an ``if`` statement and lives here. from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Any # Rule thresholds. Each is the point past which the lever is worth a look, @@ -33,6 +33,22 @@ class Lever: evidence: str action: str est_usd_per_day: float | None = None + # The snapshot metric the lever is meant to move: the join key to ledger + # lines, whose free-text titles never match these. + metric: str = "" + + +# Snapshot metric each rule's lever targets, by title (titles are fixed). +TITLE_METRIC: dict[str, str] = { + "Opus runs most subagent work": "subagent_opus_usd_share", + "Calls over the context budget": "over_budget_usd_share", + "Sessions started in ~": "home_cwd_usd_share", + "Independent tool calls not batched": "batching_calls_per_msg", + "Large Bash results": "bash_big_result_share", + "Always-loaded context grew": "standing_tokens_est", + "Images held in context": "image_usd_share", + "Unused MCP servers / skills": "standing_tokens_est", +} def _num(metrics: dict[str, Any], key: str) -> float: @@ -173,7 +189,8 @@ def candidates( + _mechanics(metrics, previous) + _dead(dead_mcp, dead_skills) ) - return sorted(levers, key=lambda lv: -(lv.est_usd_per_day or 0.0)) + tagged = [replace(lv, metric=TITLE_METRIC.get(lv.title, "")) for lv in levers] + return sorted(tagged, key=lambda lv: -(lv.est_usd_per_day or 0.0)) def render(levers: list[Lever]) -> list[str]: diff --git a/python_pkg/token_audit/review.py b/python_pkg/token_audit/review.py index 1ec0e179..a3ceb536 100644 --- a/python_pkg/token_audit/review.py +++ b/python_pkg/token_audit/review.py @@ -101,25 +101,37 @@ def _ledger(snap: dict[str, Any]) -> list[str]: return out +def candidates( + snap: dict[str, Any], prev: dict[str, Any] | None +) -> list[recommend.Lever]: + """Every rule-based lever that fires for this snapshot.""" + dead_mcp = [u.name for u in unused.mcp_usage() if u.dead] + dead_skills = [u.name for u in unused.skill_usage() if u.dead] + prev_metrics = ( + prev.get("metrics") if prev and prev.get("v", 1) >= SNAPSHOT_VERSION else None + ) + return recommend.candidates(snap["metrics"], prev_metrics, dead_mcp, dead_skills) + + def render( - snap: dict[str, Any], prev: dict[str, Any] | None, *, levers: bool + snap: dict[str, Any], + prev: dict[str, Any] | None, + *, + levers: bool, + found: list[recommend.Lever] | None = None, ) -> list[str]: - """Markdown for comparison, ledger and (optionally) candidate levers.""" + """Markdown for comparison, ledger and (optionally) candidate levers. + + ``found`` is the already-computed candidate list when the caller also + persists it in the snapshot, so the unused-surface scan runs once. + """ lines = ["## Since the previous analysis", ""] lines += _comparison(snap["metrics"], prev) lines += ["", "## What helped (ledger)", ""] lines += _ledger(snap) if levers: - dead_mcp = [u.name for u in unused.mcp_usage() if u.dead] - dead_skills = [u.name for u in unused.skill_usage() if u.dead] - prev_metrics = ( - prev.get("metrics") - if prev and prev.get("v", 1) >= SNAPSHOT_VERSION - else None - ) - found = recommend.candidates( - snap["metrics"], prev_metrics, dead_mcp, dead_skills - ) + if found is None: + found = candidates(snap, prev) lines += ["", "## Candidate levers (rule-based)", ""] lines += recommend.render(found) return lines diff --git a/python_pkg/token_audit/tests/test_cli.py b/python_pkg/token_audit/tests/test_cli.py index b61a30e9..770502d6 100644 --- a/python_pkg/token_audit/tests/test_cli.py +++ b/python_pkg/token_audit/tests/test_cli.py @@ -205,3 +205,28 @@ def test_rerun_starts_where_previous_window_ended(tmp_path: Path) -> None: assert cli.main(args) == 0 # The first run's window end is now the default start: nothing newer. assert cli.main(args) == cli.EXIT_NO_DATA + + +def test_recommend_persists_levers_as_structured_snapshot_data( + tmp_path: Path, +) -> None: + root = tmp_path / "projects" + root.mkdir() + _transcript(root, turns=3) + out = tmp_path / "out" + assert cli.main(["--root", str(root), "--out", str(out), "--recommend"]) == 0 + (snap_path,) = (out / "history").glob("*.json") + snap = json.loads(snap_path.read_text(encoding="utf-8")) + assert isinstance(snap["levers"], list) + keys = {"title", "metric", "evidence", "action"} + assert all(keys <= set(lever) for lever in snap["levers"]) + + +def test_without_recommend_no_levers_key(tmp_path: Path) -> None: + root = tmp_path / "projects" + root.mkdir() + _transcript(root, turns=3) + out = tmp_path / "out" + assert cli.main(["--root", str(root), "--out", str(out)]) == 0 + (snap_path,) = (out / "history").glob("*.json") + assert "levers" not in json.loads(snap_path.read_text(encoding="utf-8")) diff --git a/python_pkg/token_audit/tests/test_recommend.py b/python_pkg/token_audit/tests/test_recommend.py index d9f89603..df0ba856 100644 --- a/python_pkg/token_audit/tests/test_recommend.py +++ b/python_pkg/token_audit/tests/test_recommend.py @@ -88,3 +88,21 @@ def test_render_table_formats_estimates() -> None: assert lines[0].startswith("| category") assert "| $1.50 |" in lines[2] assert "| - |" in lines[3] + + +def test_every_rule_title_maps_to_a_snapshot_metric() -> None: + metrics = { + "subagent_usd_share": 0.9, + "subagent_opus_usd_share": 0.9, + "over_budget_usd_share": 0.9, + "home_cwd_usd_share": 0.9, + "batching_calls_per_msg": 1.0, + "bash_big_result_share": 0.9, + "image_usd_share": 0.9, + "standing_tokens_est": 200, + "usd_total": 10, + "usd_per_day": 1, + } + found = recommend.candidates(metrics, {"standing_tokens_est": 100}, ["m"], []) + assert len(found) == len(recommend.TITLE_METRIC) + assert all(lv.metric == recommend.TITLE_METRIC[lv.title] for lv in found) diff --git a/python_pkg/token_dashboard/__init__.py b/python_pkg/token_dashboard/__init__.py new file mode 100644 index 00000000..74014bf8 --- /dev/null +++ b/python_pkg/token_dashboard/__init__.py @@ -0,0 +1,5 @@ +"""Durable, zero-token store of Claude Code spend for the control-panel /tokens page. + +``ingest`` walks transcripts (reusing token_audit's parser, dedup and prices) +into SQLite under ``~/data/token-dashboard/``; the dashboard only reads it. +""" diff --git a/python_pkg/token_dashboard/ab.py b/python_pkg/token_dashboard/ab.py new file mode 100644 index 00000000..14588f60 --- /dev/null +++ b/python_pkg/token_dashboard/ab.py @@ -0,0 +1,243 @@ +"""A/B test of judgment subagents: Sonnet arm vs Opus arm, 0 tokens. + +Assignments come from ``~/.claude/logs/agent-ab.jsonl`` (one line per +dispatched review/debug/design/plan agent: ``tool_use_id``, ``arm``, ...). +Each is joined on ``tool_use_id`` to the ``Agent`` call the ingester stored, +and judged on outcomes visible in the transcript after the agent returned: + +* **redo** -- the parent dispatched another Agent in the same session whose + description shares >= :data:`REDO_JACCARD` of its word set; +* **follow-up** -- the parent sent that agent a ``SendMessage``; +* **error** -- the call errored or the agent was aborted; +* **correction** (noisy) -- the next user prompt opens with a correction + phrase (:data:`CORRECTION`); +* the parent session's end tag. + +Rates get 95% Wilson intervals and per-arm differences Newcombe's hybrid +interval. The banner turns RED only when, with >= :data:`MIN_N` per arm, the +Sonnet arm's redo or correction rate is higher and the two Wilson intervals do +not overlap. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +import math +from pathlib import Path +import re +from typing import TYPE_CHECKING, Any + +from python_pkg.token_dashboard import ab_rows + +if TYPE_CHECKING: + import sqlite3 + +LOG = Path.home() / ".claude" / "logs" / "agent-ab.jsonl" +Z = 1.96 +MIN_N = 20 +REDO_JACCARD = 0.6 +ERRORS = frozenset({"error", "aborted", "killed", "failed", "cancelled"}) +CORRECTION = re.compile( + r"^\W*no\b|\bwrong\b|\bthat'?s not\b|\bthat is not\b|\bincorrect\b" + r"|\brevert\b|\byou missed\b", + re.IGNORECASE, +) +WORD = re.compile(r"[a-z0-9]+") +RATES = ("redo", "followup", "error", "correction") + + +def wilson(k: int, n: int, z: float = Z) -> tuple[float, float]: + """95% Wilson score interval for ``k`` successes in ``n`` trials.""" + if n == 0: + return 0.0, 1.0 + p = k / n + centre = p + z * z / (2 * n) + spread = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) + denom = 1 + z * z / n + return max(0.0, (centre - spread) / denom), min(1.0, (centre + spread) / denom) + + +def diff_ci(k1: int, n1: int, k2: int, n2: int) -> tuple[float, float, float]: + """``p1 - p2`` with Newcombe's hybrid-score 95% interval.""" + p1, p2 = (k1 / n1 if n1 else 0.0), (k2 / n2 if n2 else 0.0) + l1, u1 = wilson(k1, n1) + l2, u2 = wilson(k2, n2) + diff = p1 - p2 + low = diff - math.sqrt((p1 - l1) ** 2 + (u2 - p2) ** 2) + high = diff + math.sqrt((u1 - p1) ** 2 + (p2 - l2) ** 2) + return diff, low, high + + +def jaccard(a: str, b: str) -> float: + """Word-set Jaccard similarity of two descriptions.""" + left, right = set(WORD.findall(a.lower())), set(WORD.findall(b.lower())) + union = left | right + return len(left & right) / len(union) if union else 0.0 + + +def is_correction(text: str) -> bool: + """Whether a user prompt reads as correcting the previous answer.""" + return CORRECTION.search(text) is not None + + +@dataclass(frozen=True) +class ArmCounts: + """What the banner needs from one arm.""" + + n: int + redo: int + correction: int + + +def banner(sonnet: ArmCounts, opus: ArmCounts) -> dict[str, str]: + """RED / GREEN / GREY verdict for the top of every /tokens view.""" + n = min(sonnet.n, opus.n) + if n < MIN_N: + return {"level": "grey", "text": f"A/B judgment agents: collecting (n={n})"} + for name in ("redo", "correction"): + ks, ko = getattr(sonnet, name), getattr(opus, name) + low_s, _ = wilson(ks, sonnet.n) + _, high_o = wilson(ko, opus.n) + if ks / sonnet.n > ko / opus.n and low_s > high_o: + return { + "level": "red", + "text": f"QUALITY DROP: Sonnet {name} rate {ks}/{sonnet.n} vs Opus " + f"{ko}/{opus.n} (95% CIs do not overlap)", + } + return { + "level": "green", + "text": f"A/B judgment agents: no detectable drop (n={n})", + } + + +def _assignments(log: Path) -> list[dict[str, Any]]: + """Well-formed assignment lines.""" + if not log.is_file(): + return [] + rows = [] + for line in log.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + row = json.loads(line) + except ValueError: + continue + if isinstance(row, dict) and row.get("tool_use_id") and row.get("arm"): + rows.append(row) + return rows + + +def _outcome(conn: sqlite3.Connection, tool_use_id: str) -> dict[str, Any] | None: + """The joined call: cost, duration and the four flags, or ``None``.""" + call = conn.execute( + "SELECT key, agent_id, ts, description, result_ts, status FROM agent_calls" + " WHERE tool_use_id = ?", + (tool_use_id,), + ).fetchone() + if call is None: + return None + key, agent_id, ts, desc, result_ts, status = call + main = key.split("/", 1)[0] + sub = conn.execute( + "SELECT usd, wall_s, last_ts FROM subagent_link WHERE sub_key = ?", + (f"{main}/agent-{agent_id}",), + ).fetchone() or (0.0, (result_ts or ts) - ts, result_ts or ts) + returned = max(result_ts or ts, sub[2] or 0) + later = conn.execute( + "SELECT description FROM agent_calls WHERE key = ? AND ts > ?", + (key, returned), + ).fetchall() + followup = conn.execute( + "SELECT COUNT(*) FROM sent_messages WHERE key = ? AND to_agent = ? AND ts > ?", + (key, agent_id, returned), + ).fetchone()[0] + nxt = conn.execute( + "SELECT text FROM prompts WHERE key = ? AND ts > ? ORDER BY ts LIMIT 1", + (main, returned), + ).fetchone() + tag = conn.execute( + "SELECT final_tag FROM session_summary WHERE key = ?", (main,) + ).fetchone() + return { + "key": key, + "ts": ts, + "usd": sub[0] or 0.0, + "duration_s": sub[1] or 0.0, + "redo": any(jaccard(desc, d) >= REDO_JACCARD for (d,) in later), + "followup": followup > 0, + "error": (status or "").lower() in ERRORS, + "correction": nxt is not None and is_correction(nxt[0]), + "tag": (tag[0] if tag else None) or "untagged", + } + + +def _mean_ci(values: list[float]) -> dict[str, float]: + """Mean with a normal-approximation 95% interval.""" + n = len(values) + if n == 0: + return {"mean": 0.0, "low": 0.0, "high": 0.0} + mean = sum(values) / n + var = sum((v - mean) ** 2 for v in values) / (n - 1) if n > 1 else 0.0 + half = Z * math.sqrt(var / n) + return {"mean": mean, "low": mean - half, "high": mean + half} + + +def _arm(rows: list[dict[str, Any]]) -> dict[str, Any]: + """Per-arm summary.""" + n = len(rows) + out: dict[str, Any] = {"n": n} + for name in RATES: + k = sum(1 for r in rows if r[name]) + low, high = wilson(k, n) + out[name] = {"k": k, "rate": k / n if n else None, "low": low, "high": high} + out["usd"] = _mean_ci([r["usd"] for r in rows]) + out["duration_s"] = _mean_ci([r["duration_s"] for r in rows]) + tags: dict[str, int] = {} + for r in rows: + tags[r["tag"]] = tags.get(r["tag"], 0) + 1 + out["tags"] = tags + return out + + +def rebuild(conn: sqlite3.Connection, log: Path | None = None) -> dict[str, Any]: + """Join every assignment, summarise both arms, store in ``meta['ab']``.""" + joined: dict[str, list[dict[str, Any]]] = {"sonnet": [], "opus": []} + kept: list[tuple[str, str, dict[str, Any]]] = [] + unjoined = 0 + for row in _assignments(log or LOG): + found = _outcome(conn, str(row["tool_use_id"])) + if found is None or row["arm"] not in joined: + unjoined += 1 + continue + joined[row["arm"]].append(found) + kept.append((row["arm"], str(row["tool_use_id"]), found)) + ab_rows.store(conn, kept) + arms = {name: _arm(rows) for name, rows in joined.items()} + diffs = { + name: dict( + zip( + ("diff", "low", "high"), + diff_ci( + arms["sonnet"][name]["k"], + arms["sonnet"]["n"], + arms["opus"][name]["k"], + arms["opus"]["n"], + ), + strict=True, + ) + ) + for name in RATES + } + counts = { + name: ArmCounts( + arms[name]["n"], arms[name]["redo"]["k"], arms[name]["correction"]["k"] + ) + for name in joined + } + payload = { + "arms": arms, + "diffs": diffs, + "unjoined": unjoined, + "banner": banner(counts["sonnet"], counts["opus"]), + } + conn.execute("INSERT OR REPLACE INTO meta VALUES ('ab', ?)", (json.dumps(payload),)) + return payload diff --git a/python_pkg/token_dashboard/ab_rows.py b/python_pkg/token_dashboard/ab_rows.py new file mode 100644 index 00000000..452260c5 --- /dev/null +++ b/python_pkg/token_dashboard/ab_rows.py @@ -0,0 +1,56 @@ +"""Per-assignment A/B outcomes, so the dashboard can filter the A/B panel. + +:func:`ab.rebuild` summarises all assignments into ``meta['ab']`` (the +all-time verdict). This module also keeps one row per joined assignment, with +the parent call's session, cwd, model and scope, so a cwd / model / scope +filter can recount the arms. Derived: replaced wholesale on every rebuild. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + import sqlite3 + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS ab_rows( + tool_use_id TEXT PRIMARY KEY, arm TEXT, key TEXT, main_key TEXT, is_sub INT, + cwd TEXT, model TEXT, ts REAL, usd REAL, duration_s REAL, redo INT, + followup INT, error INT, correction INT, tag TEXT); +""" + + +def store( + conn: sqlite3.Connection, joined: list[tuple[str, str, dict[str, Any]]] +) -> None: + """Replace every row with ``(arm, tool_use_id, outcome)`` triples.""" + conn.executescript(SCHEMA) + conn.execute("DELETE FROM ab_rows") + for arm, tool_use_id, found in joined: + key = str(found["key"]) + where = conn.execute( + "SELECT cwd, model FROM turns WHERE key = ? AND ts <= ?" + " ORDER BY ts DESC LIMIT 1", + (key, found["ts"]), + ).fetchone() or (None, None) + conn.execute( + "INSERT OR REPLACE INTO ab_rows VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + tool_use_id, + arm, + key, + key.split("/", 1)[0], + int("/" in key), + where[0], + where[1], + found["ts"], + found["usd"], + found["duration_s"], + int(found["redo"]), + int(found["followup"]), + int(found["error"]), + int(found["correction"]), + found["tag"], + ), + ) diff --git a/python_pkg/token_dashboard/active.py b/python_pkg/token_dashboard/active.py new file mode 100644 index 00000000..a9ff996e --- /dev/null +++ b/python_pkg/token_dashboard/active.py @@ -0,0 +1,56 @@ +"""Active time: the sum of gaps between consecutive records, ignoring idle gaps. + +A gap longer than :data:`IDLE_GAP` contributes **zero** (not the cap): it means +nobody and nothing was working. Storing each file as maximal runs of records +spaced at most ``IDLE_GAP`` apart is lossless for that rule -- the active time +of several files' merged timeline equals the measure of the union of their runs +with gaps of up to ``IDLE_GAP`` between runs bridged. That is what lets a main +session and the subagents it spawned, ingested in different runs, combine +exactly afterwards. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + +IDLE_GAP = 300.0 +Span = tuple[float, float] + + +def intervals(stamps: Iterable[float], gap: float = IDLE_GAP) -> list[Span]: + """Collapse sorted-or-not timestamps into maximal runs.""" + out: list[Span] = [] + for stamp in sorted(stamps): + if out and stamp - out[-1][1] <= gap: + out[-1] = (out[-1][0], stamp) + else: + out.append((stamp, stamp)) + return out + + +def measure( + spans: Iterable[Span], + low: float = float("-inf"), + high: float = float("inf"), + gap: float = IDLE_GAP, +) -> float: + """Active seconds of the union of runs, clipped to ``[low, high)``.""" + merged = intervals_union(spans, gap) + total = 0.0 + for start, end in merged: + total += max(0.0, min(end, high) - max(start, low)) + return total + + +def intervals_union(spans: Iterable[Span], gap: float = IDLE_GAP) -> list[Span]: + """Union of runs, bridging gaps no longer than ``gap``.""" + out: list[Span] = [] + for start, end in sorted(spans): + if out and start - out[-1][1] <= gap: + out[-1] = (out[-1][0], max(out[-1][1], end)) + else: + out.append((start, end)) + return out diff --git a/python_pkg/token_dashboard/aux.py b/python_pkg/token_dashboard/aux.py new file mode 100644 index 00000000..0cb992da --- /dev/null +++ b/python_pkg/token_dashboard/aux.py @@ -0,0 +1,200 @@ +"""Non-transcript sources: snapshots, levers, autopsy aggregates, activity logs. + +Each loader is idempotent (it replaces its own rows) and cheap, so every +complete ingest pass reruns all of them. + +The session-autopsy cache (``~/.claude/autopsy/sessions.jsonl``) is the only +source of per-session numbers for transcripts already deleted. It is +**aggregate only**: tokens are summed per transcript record, not per API +message (about 2x), there is no model split and no subagents. It is therefore +scaled by ``api_turns / assistant_msgs``, priced as Opus 5.5, and then +calibrated against sessions present in both sources (``meta`` key +``autopsy_calibration``). It is never mixed into the per-session tables. +""" + +from __future__ import annotations + +from collections import Counter +from datetime import datetime +import json +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from python_pkg.token_audit import pricing +from python_pkg.token_dashboard import levers, store + +if TYPE_CHECKING: + from collections.abc import Iterator + import sqlite3 + +CLAUDE = Path.home() / ".claude" +AUTOPSY = CLAUDE / "autopsy" / "sessions.jsonl" +PROMPT_HISTORY = CLAUDE / "history.jsonl" +TIME_CAP = CLAUDE / "logs" / "time-cap.jsonl" +_MS = 1000.0 +_MS_EPOCH = 1e11 # anything larger is milliseconds, not seconds + + +def _jsonl(path: Path) -> Iterator[dict[str, Any]]: + """Well-formed object lines of a JSONL file (none if it is missing).""" + if not path.is_file(): + return + with path.open(encoding="utf-8", errors="replace") as handle: + for line in handle: + try: + row = json.loads(line) + except ValueError: + continue + if isinstance(row, dict): + yield row + + +def _stamp(raw: object) -> float | None: + """ISO string or epoch (s or ms) to unix seconds.""" + if isinstance(raw, (int, float)): + return raw / _MS if raw > _MS_EPOCH else float(raw) + if isinstance(raw, str): + try: + return datetime.fromisoformat(raw).timestamp() + except ValueError: + return None + return None + + +def snapshots(conn: sqlite3.Connection, report: Path) -> None: + """Copy every history snapshot's window and metrics.""" + for path in sorted((report / "history").glob("*.json")): + try: + snap = json.loads(path.read_text(encoding="utf-8")) + except ValueError: + continue + window = snap.get("window", {}) + conn.execute( + "INSERT OR REPLACE INTO snapshots VALUES (?,?,?,?,?)", + ( + path.stem, + snap.get("generated"), + window.get("since"), + window.get("until"), + json.dumps(snap.get("metrics", {})), + ), + ) + + +def _nearest(generated: list[tuple[float, str]], stamp: float) -> str: + """The snapshot day generated closest to ``stamp``.""" + return min(generated, key=lambda g: abs(g[0] - stamp))[1] + + +def lever_rows(conn: sqlite3.Connection, report: Path) -> None: + """Rebuild the ``levers`` table from every source.""" + generated = [ + (_stamp(g) or 0.0, name[:10]) + for name, g in conn.execute("SELECT name, generated FROM snapshots") + ] + tables: dict[str, str] = {} + found = conn.execute("SELECT ts, markdown FROM lever_tables ORDER BY ts") + weekly = report / "WEEKLY.md" + extra = [] + if weekly.is_file(): + extra = [(weekly.stat().st_mtime, weekly.read_text(encoding="utf-8"))] + for ts, markdown in [*found, *extra]: + # A table belongs to the snapshot generated closest to it. + if generated: + tables.setdefault(_nearest(generated, ts), markdown) + conn.execute("DELETE FROM levers") + conn.executemany( + "INSERT INTO levers VALUES (?,?,?,?,?,?,?,?,?,?,?)", + [tuple(row.values()) for row in levers.build(report, tables)], + ) + + +def autopsy(conn: sqlite3.Connection) -> None: + """Load autopsy aggregates and calibrate their dollar estimate.""" + price = pricing.load().for_model("claude-opus-5-5") + for row in _jsonl(AUTOPSY): + meta, tok, turns = ( + row.get("meta", {}), + row.get("tokens", {}), + row.get("turns", {}), + ) + started = _stamp(meta.get("started_at")) + records = row.get("counts", {}).get("assistant_msgs") or 0 + api = turns.get("api_turns") or records + if started is None or not records: + continue + scale = api / records + raw = ( + tok.get("input", 0) * price.input + + tok.get("cache_write", 0) * price.input * pricing.WRITE_1H + + tok.get("cache_read", 0) * price.cache_read + + tok.get("output", 0) * price.output + ) / pricing.PER_MILLION + conn.execute( + "INSERT OR REPLACE INTO aggregate_sessions VALUES" + " (?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + row.get("session_id"), + "autopsy", + store.day_of(started), + started, + meta.get("cwd"), + meta.get("title"), + tok.get("input", 0), + tok.get("cache_write", 0), + tok.get("cache_read", 0), + tok.get("output", 0), + api, + records, + raw * scale, + ), + ) + est, real = conn.execute( + "SELECT SUM(a.usd_est), SUM(s.usd_main) FROM aggregate_sessions a" + " JOIN session_summary s ON s.key = a.session_id" + ).fetchone() + factor = (real / est) if est and real else 1.0 + conn.execute( + "INSERT OR REPLACE INTO meta VALUES ('autopsy_calibration', ?)", (str(factor),) + ) + + +def activity_logs(conn: sqlite3.Connection, report: Path) -> None: + """Prompt counts per day, time-cap events, task-size misses and estimates.""" + days: Counter[str] = Counter() + for row in _jsonl(PROMPT_HISTORY): + stamp = _stamp(row.get("timestamp")) + if stamp is not None: + days[store.day_of(stamp)] += 1 + # Upserts, never a wipe: if history.jsonl is trimmed, the days it lost stay. + conn.executemany( + "INSERT INTO prompt_activity VALUES (?,?) ON CONFLICT(day) DO UPDATE" + " SET prompts = MAX(prompts, excluded.prompts)", + days.items(), + ) + sources = ( + ("time-cap", TIME_CAP, "event"), + ("task-size-miss", report / "task-size" / "misses.jsonl", "kind"), + ) + for name, path, kind_key in sources: + conn.executemany( + "INSERT OR IGNORE INTO events VALUES (?,?,?,?)", + [ + ( + name, + _stamp(r.get("ts")), + str(r.get(kind_key)), + str(r.get("command") or r.get("first_prompt") or "")[:200], + ) + for r in _jsonl(path) + ], + ) + for row in _jsonl(report / "task-size" / "verdicts.jsonl"): + verdict = row.get("verdict") + calls = verdict.get("est_calls") if isinstance(verdict, dict) else None + if row.get("status") == "ok" and isinstance(calls, int): + # The first sized prompt of a session is its pre-work estimate. + conn.execute( + "INSERT OR IGNORE INTO estimates VALUES (?,?)", + (row.get("session"), calls), + ) diff --git a/python_pkg/token_dashboard/blocks.py b/python_pkg/token_dashboard/blocks.py new file mode 100644 index 00000000..9fcf1f8e --- /dev/null +++ b/python_pkg/token_dashboard/blocks.py @@ -0,0 +1,104 @@ +"""Pure readers for individual transcript records and content blocks. + +Split from :mod:`walk` so the record-level rules (what counts as a typed +prompt, how a status tag or a commit hash is recognised) are testable without a +file and the walker stays under the file-length cap. +""" + +from __future__ import annotations + +import os +from pathlib import Path +import re +from typing import Any + +from python_pkg.token_audit.discover import WRAPPER + +PROMPT_CHARS = 120 +LEVER_HEADING = "Candidate levers (rule-based)" +# kuhy's convention: every final reply ends on one of these status tags. +TAG = re.compile(r"^\W*(DONE|RUNNING|QUESTION|NEEDS ACTION)\W*:", re.MULTILINE) +COMMAND = re.compile(r"\s*(\S+?)\s*") +# `git commit` prints `[branch sha] subject`, also through finish_auto.sh. +COMMIT_LINE = re.compile( + r"^\[[^\]\s]+(?: \(root-commit\))? ([0-9a-f]{7,40})\]", re.MULTILINE +) +REPO_HINT = re.compile(r"(?:git\s+-C|\bcd)\s+(\"[^\"]+\"|'[^']+'|[^\s;&|]+)") +_SKIP_PREFIXES = ("[Request interrupted", "Caveat:") + + +def text_of(content: object) -> str: + """Flatten a message/tool-result ``content`` into plain text.""" + if isinstance(content, str): + return content + if not isinstance(content, list): + return "" + parts = [ + str(block.get("text") or "") + for block in content + if isinstance(block, dict) and block.get("type") == "text" + ] + return "\n".join(parts) + + +def prompt_text(record: dict[str, Any]) -> str | None: + """The typed text of a real user prompt, or ``None`` for anything else. + + Excludes tool results, ``isMeta`` / compact-summary records, interrupts, + and harness-injected ```` blocks. A slash command keeps its name. + """ + if record.get("type") != "user" or record.get("isMeta"): + return None + if record.get("isCompactSummary") or record.get("isVisibleInTranscriptOnly"): + return None + message = record.get("message") + content = message.get("content") if isinstance(message, dict) else None + if isinstance(content, list) and any( + isinstance(b, dict) and b.get("type") == "tool_result" for b in content + ): + return None + raw = text_of(content) + command = COMMAND.search(raw) + text = WRAPPER.sub("", raw).strip() + if command is not None and not text: + text = command.group(1) + if not text or text.startswith(_SKIP_PREFIXES): + return None + return " ".join(text.split())[:PROMPT_CHARS] + + +def final_tag(text: str) -> str | None: + """The last status tag in a reply, or ``None`` when it carries none.""" + found = TAG.findall(text) + return found[-1] if found else None + + +def commit_shas(text: str) -> list[str]: + """Commit hashes printed by ``git commit`` in a tool result.""" + return COMMIT_LINE.findall(text) + + +def repo_hint(command: str, cwd: str | None) -> str: + """The repository a commit command ran in: ``-C``/``cd`` target, else cwd.""" + match = REPO_HINT.search(command) + target = match.group(1).strip("\"'") if match else (cwd or "") + if not target: + return "" + path = Path(target).expanduser() + if cwd and not path.is_absolute(): + path = Path(cwd) / path + return os.path.normpath(path) + + +def lever_table(text: str) -> str | None: + """The markdown table following the candidate-levers heading, if any.""" + at = text.find(LEVER_HEADING) + if at < 0: + return None + rows: list[str] = [] + for line in text[at:].splitlines()[1:]: + if line.startswith("|"): + rows.append(line) + elif rows: + break + return "\n".join(rows) if rows else None diff --git a/python_pkg/token_dashboard/carry.py b/python_pkg/token_dashboard/carry.py new file mode 100644 index 00000000..2459836f --- /dev/null +++ b/python_pkg/token_dashboard/carry.py @@ -0,0 +1,79 @@ +"""Carry cost: what a tool result costs by staying in context afterwards. + +A result of N tokens is cache-read again on every later API call of the same +file until a compaction drops it (or the session ends), so its real price is +N x (cache-read price of each later call) -- usually far more than the call +that fetched it. A compaction shows up as the context shrinking below +``COMPACTION_DROP_RATIO`` of the previous call's, the same rule token_audit's +image-cost model uses. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Protocol + +from python_pkg.token_audit.model import COMPACTION_DROP_RATIO +from python_pkg.token_audit.pricing import PER_MILLION, WRITE_5M + +if TYPE_CHECKING: + from python_pkg.token_audit.pricing import PriceBook + + +class _Row(Protocol): + """Anything with creation / carry accumulators.""" + + create_usd: float + carry_usd: float + carry_calls: int + + +class Tracker: + """Collects calls and results in order, settles carry at the end.""" + + def __init__(self) -> None: + """Start with no calls and no results.""" + self.calls: list[tuple[str, int]] = [] + self.writes: list[float] = [] + self.results: list[tuple[int, int, _Row]] = [] + + def turn(self, model: str, context: int, write: float = WRITE_5M) -> None: + """Note one API call, in order, with its cache-write price multiplier.""" + self.calls.append((model, context)) + self.writes.append(write) + + def result(self, row: _Row, tokens: int) -> None: + """Note a result arriving before call number ``len(self.calls)``.""" + self.results.append((len(self.calls), tokens, row)) + + def spans(self) -> list[tuple[int, int, int, _Row]]: + """``(first, end, tokens, row)``: the calls ``first..end-1`` carry it.""" + # cut[i]: index of the first call at or after i that starts a new + # context (after a compaction), else len(calls). + size = len(self.calls) + cut = [size] * (size + 1) + for index in range(size - 1, -1, -1): + dropped = index > 0 and self.calls[index][1] < ( + COMPACTION_DROP_RATIO * self.calls[index - 1][1] + ) + cut[index] = index if dropped else cut[index + 1] + return [ + (start, cut[start + 1], tokens, row) + for start, tokens, row in self.results + if start < size + ] + + def settle(self, book: PriceBook) -> None: + """Add each result's creation and carry to its row. + + Creation: the first call after the result writes it to cache, at that + call's write multiplier of input price. Carry: every call from there + to the next compaction re-reads it at the cache-read price. + """ + prefix = [0.0] + for model, _ in self.calls: + prefix.append(prefix[-1] + book.for_model(model).cache_read) + for start, end, tokens, row in self.spans(): + price = book.for_model(self.calls[start][0]).input + row.create_usd += tokens * self.writes[start] * price / PER_MILLION + row.carry_usd += tokens * (prefix[end] - prefix[start]) / PER_MILLION + row.carry_calls += end - start diff --git a/python_pkg/token_dashboard/check_window.py b/python_pkg/token_dashboard/check_window.py new file mode 100644 index 00000000..6170374b --- /dev/null +++ b/python_pkg/token_dashboard/check_window.py @@ -0,0 +1,78 @@ +"""Reconcile the store against a token_audit snapshot: within 1% or exit 1. + +The snapshot's ``usd_per_day`` is ``usd_total`` over the span between the +earliest and latest priced record inside its window (not the window length), +so the same definition is applied to the store's turns here. + + python3 -m python_pkg.token_dashboard.check_window # newest + python3 -m python_pkg.token_dashboard.check_window --snapshot 2026-10-17 +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys +from typing import TYPE_CHECKING + +from python_pkg.token_audit import history +from python_pkg.token_dashboard import schema + +if TYPE_CHECKING: + import sqlite3 + +TOLERANCE = 0.01 +DAY_S = 86400.0 +MIN_SPAN_S = 60.0 + + +def measure(conn: sqlite3.Connection, since: float, until: float) -> dict[str, float]: + """``usd_total``, ``usd_per_day`` and ``calls`` from the store's turns.""" + usd, calls, first, last = conn.execute( + "SELECT SUM(usd), COUNT(*), MIN(ts), MAX(ts) FROM turns" + " WHERE ts BETWEEN ? AND ?", + (since, until), + ).fetchone() + usd = usd or 0.0 + days = max((last or 0) - (first or 0), MIN_SPAN_S) / DAY_S + return {"usd_total": usd, "usd_per_day": usd / days, "calls": float(calls)} + + +def main(argv: list[str] | None = None) -> int: + """Compare and print one line per metric; 0 when every one is within 1%.""" + parser = argparse.ArgumentParser(prog="token_dashboard.check_window") + parser.add_argument("--db", type=Path, default=schema.DEFAULT_DB) + parser.add_argument("--report-dir", type=Path, default=history.REPORT_DIR) + parser.add_argument("--snapshot", default=None, help="history file stem") + args = parser.parse_args(argv) + folder = args.report_dir / history.HISTORY_NAME + path = ( + folder / f"{args.snapshot}.json" + if args.snapshot + else max(folder.glob("*.json"), default=None) + ) + if path is None or not path.is_file() or not args.db.is_file(): + sys.stderr.write(f"missing snapshot or store ({path}, {args.db})\n") + return 1 + snap = json.loads(path.read_text(encoding="utf-8")) + window = snap["window"] + conn = schema.connect(args.db, readonly=True) + try: + got = measure(conn, float(window["since"]), float(window["until"])) + finally: + conn.close() + ok = True + for key, value in got.items(): + want = float(snap["metrics"][key]) + drift = abs(value - want) / want if want else 0.0 + ok &= drift <= TOLERANCE + sys.stdout.write( + f"{key}: store {value:,.2f} vs snapshot {want:,.2f}" + f" ({drift:.2%}) {'OK' if drift <= TOLERANCE else 'FAIL'}\n" + ) + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python_pkg/token_dashboard/classify.py b/python_pkg/token_dashboard/classify.py new file mode 100644 index 00000000..8878b1ac --- /dev/null +++ b/python_pkg/token_dashboard/classify.py @@ -0,0 +1,216 @@ +"""Name what a tool call was, for the carry-cost and tool panels. + +``family`` is the bucket a panel groups by (``git status``, ``mcp:``, +``Read:memory``); ``target`` is the one thing it touched (the command, path, +host or query), capped so the store stays small; ``signature`` identifies +identical repeated calls (loops) without storing the full input. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from urllib.parse import urlsplit + +from python_pkg.token_audit.model import IMAGE_SUFFIXES + +TARGET_CHARS = 160 +SIG_CHARS = 16 +EMPTY = "(empty)" + +# Words that only wrap the real command (and the flags/numbers they take). +_WRAPPERS = frozenset( + { + "sudo", + "env", + "nice", + "ionice", + "timeout", + "time", + "nohup", + "exec", + "command", + "capped.sh", + "stdbuf", + "xargs", + } +) +# Tools whose first non-flag argument is what the call really did. +_SUBCOMMANDS = frozenset( + { + "git", + "pnpm", + "npm", + "npx", + "flutter", + "dart", + "cargo", + "go", + "docker", + "systemctl", + "gh", + "pre-commit", + "uv", + "pip", + "yay", + "pacman", + "adb", + "make", + "fvm", + "journalctl", + } +) +_TAKES_VALUE = frozenset({"-C", "-c", "-n", "--dir", "--prefix"}) +_SHELLS = frozenset({"bash", "sh", "zsh"}) +_ASSIGN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=") +_SEGMENTS = re.compile(r"\s*(?:&&|\|\||;|\||\n)\s*") +_NUMBERISH = re.compile(r"^\d+[smhd]?$") +_ERRORS = ( + ("Exit code", "exit_code"), + ("", "tool_error"), + ("doesn't want to proceed", "rejected"), + ("Permission for this action was denied", "denied"), + ("hook error", "hook"), + ("TIME CAP", "time_cap"), + ("[Request interrupted", "interrupted"), +) + + +def _base(word: str) -> str: + """Last path component of a command word.""" + return word.rsplit("/", 1)[-1] + + +def _command_words(words: list[str]) -> list[str]: + """Drop leading assignments, wrappers and the flags/values they take.""" + index = 0 + while index < len(words): + word = words[index] + if not ( + _ASSIGN.match(word) + or _base(word) in _WRAPPERS + or word.startswith("-") + or _NUMBERISH.match(word) + ): + break + index += 1 + return words[index:] + + +def _first_argument(words: list[str]) -> str: + """The first non-flag word, skipping flags that take a value.""" + skip = False + for word in words: + if skip: + skip = False + elif word in _TAKES_VALUE: + skip = True + elif not word.startswith(("-", "<", ">", "2>")): + return word + return "" + + +def _name_command(words: list[str]) -> str: + """``git status``, ``python3 -m pytest``, ``bash x.sh`` or the bare name.""" + head, rest = _base(words[0]), words[1:] + if head.startswith("python"): + if "-m" in rest: + after = rest[rest.index("-m") + 1 :] + return f"{head} -m {after[0]}" if after else f"{head} -m" + script = _first_argument(rest) + return f"{head} {_base(script)}" if script else head + if head in _SHELLS and "-c" in rest: + return f"{head} -c" + if head in _SHELLS or head in _SUBCOMMANDS: + arg = _first_argument(rest) + return f"{head} {_base(arg) if head in _SHELLS else arg}".strip() + return head + + +def bash_family(command: str) -> str: + """The command a Bash call really ran, ignoring wrappers and ``cd``.""" + last = "" + for segment in _SEGMENTS.split(command.strip()): + words = _command_words(segment.split()) + if not words: + continue + last = _name_command(words) + if last != "cd": + return last + return last or EMPTY + + +_PATH_KINDS = ( + ("memory", ("/.claude/", "/memory/"), ("/.claude/", "/memories/")), + ("rules", ("/.claude/rules",), ("rules-ondemand",)), + ("skill", ("/skills/",)), + ("claude", ("/.claude/",)), +) + + +def _path_kind(path: str) -> str: + """What a file path is, as far as standing context goes.""" + name = _base(path) + if name.startswith("CLAUDE") and name.endswith(".md"): + return "claude-md" + for kind, *needle_sets in _PATH_KINDS: + if any(all(n in path for n in needles) for needles in needle_sets): + return kind + lowered = path.lower() + return "image" if lowered.endswith(tuple(IMAGE_SUFFIXES)) else "file" + + +def _text(tool_input: dict[str, object], key: str) -> str: + """One string input field, empty when absent.""" + value = tool_input.get(key) + return value if isinstance(value, str) else "" + + +def family(name: str, tool_input: dict[str, object]) -> str: + """The bucket a call is grouped under.""" + if name == "Bash": + return bash_family(_text(tool_input, "command")) + if name.startswith("mcp__"): + return f"mcp:{name.split('__')[1]}" + if name == "Skill": + return "skill" + if name in {"Read", "Write", "Edit", "MultiEdit", "NotebookEdit"}: + return f"{name}:{_path_kind(_text(tool_input, 'file_path'))}" + return name + + +def target(name: str, tool_input: dict[str, object]) -> str: + """The one thing a call touched, whitespace-collapsed and capped.""" + if name == "Bash": + found = _text(tool_input, "command") + elif name.startswith("mcp__"): + found = name.split("__", 2)[-1] + elif name == "Skill": + found = _text(tool_input, "skill") + elif name == "WebFetch": + found = urlsplit(_text(tool_input, "url")).netloc + elif name in {"WebSearch", "Grep", "Glob"}: + found = _text(tool_input, "query") or _text(tool_input, "pattern") + elif name in {"Agent", "Task"}: + kind = _text(tool_input, "subagent_type") or "general-purpose" + worktree = _text(tool_input, "isolation") == "worktree" + found = f"{kind} [worktree]" if worktree else kind + else: + found = _text(tool_input, "file_path") + return " ".join(found.split())[:TARGET_CHARS] + + +def signature(name: str, tool_input: dict[str, object]) -> str: + """A short stable hash of the full call, for spotting identical repeats.""" + raw = json.dumps([name, tool_input], sort_keys=True, default=str) + return hashlib.sha1(raw.encode(), usedforsecurity=False).hexdigest()[:SIG_CHARS] + + +def error_kind(text: str) -> str: + """Why a tool call failed, from its result text.""" + head = text[:400] + for needle, kind in _ERRORS: + if needle in head: + return kind + return "error" diff --git a/python_pkg/token_dashboard/commits.py b/python_pkg/token_dashboard/commits.py new file mode 100644 index 00000000..13d0ecdd --- /dev/null +++ b/python_pkg/token_dashboard/commits.py @@ -0,0 +1,105 @@ +"""Find the commits each session made, from git rather than from tool output. + +Measured 2026-10-17: of 601 recent commit commands only 35 printed a parseable +``[branch sha]`` line -- most run through ``finish_auto.sh`` with a tailed or +backgrounded output. So every commit command is stored with its time window +and repository, and the hashes come from ``git log`` instead: + +* a commit whose hash a command printed belongs to that command's session; +* otherwise a commit carrying a ``Co-Authored-By: Claude`` trailer belongs to + the commit command in the same repository whose window covers its commit + time (from :data:`LEAD_S` before the call to :data:`TAIL_S` after its + result -- a backgrounded finish_auto.sh commits after the call returns), + the nearest one when several do. + +Rows are only ever added, so a repository deleted later keeps its history. +""" + +from __future__ import annotations + +from collections import defaultdict +import re +from typing import TYPE_CHECKING + +from python_pkg.token_dashboard import gitcmd + +if TYPE_CHECKING: + import sqlite3 + +LEAD_S = 60.0 +TAIL_S = 1800.0 +TRAILER = re.compile(r"^Co-Authored-By:\s*Claude", re.IGNORECASE | re.MULTILINE) +_FIELD, _RECORD = "\x1f", "\x1e" +_LOG_FIELDS = 3 + + +def _git(repo: str, *args: str) -> str | None: + """Run git in ``repo``; ``None`` on any failure.""" + return gitcmd.run(repo, *args) + + +def _toplevel(hint: str, cache: dict[str, str | None]) -> str | None: + """The work tree containing ``hint``, cached per hint.""" + if hint not in cache: + out = _git(hint, "rev-parse", "--show-toplevel") if hint else None + cache[hint] = out.strip() if out else None + return cache[hint] + + +def _log(repo: str, since: float) -> list[tuple[str, float, bool]]: + """``(sha, commit time, has Claude trailer)`` on HEAD since ``since``.""" + out = _git( + repo, + "log", + "HEAD", + f"--since=@{int(since)}", + f"--format=%H{_FIELD}%ct{_FIELD}%B{_RECORD}", + ) + rows = [] + for record in (out or "").split(_RECORD): + parts = record.strip("\n").split(_FIELD, 2) + if len(parts) == _LOG_FIELDS: + rows.append((parts[0], float(parts[1]), bool(TRAILER.search(parts[2])))) + return rows + + +Command = tuple[str, int, float, float, str] # key, prompt, start, end, sha + + +def _match( + sha: str, ts: float, cmds: list[Command], *, trailer: bool +) -> Command | None: + """The command that made this commit, if any.""" + for cmd in cmds: + if cmd[4] and sha.startswith(cmd[4]): + return cmd + if not trailer: + return None + covering = [c for c in cmds if c[2] - LEAD_S <= ts <= max(c[2], c[3]) + TAIL_S] + if not covering: + return None + return min(covering, key=lambda c: abs(ts - c[2])) + + +def discover(conn: sqlite3.Connection) -> int: + """Attribute git commits to sessions; returns how many rows were added.""" + cache: dict[str, str | None] = {} + by_repo: dict[str, list[Command]] = defaultdict(list) + for key, prompt, start, end, sha, hint in conn.execute( + "SELECT key, prompt_idx, ts, end_ts, sha, repo FROM commits" + ): + top = _toplevel(hint, cache) + if top is not None: + by_repo[top].append((key, prompt, start, end, sha)) + added = 0 + for repo, cmds in by_repo.items(): + since = min(c[2] for c in cmds) - LEAD_S + for sha, ts, trailer in _log(repo, since): + cmd = _match(sha, ts, cmds, trailer=trailer) + if cmd is None: + continue + added += conn.execute( + "INSERT OR IGNORE INTO session_commits VALUES (?,?,?,?,?)", + (sha, repo, ts, cmd[0], cmd[1]), + ).rowcount + return added diff --git a/python_pkg/token_dashboard/context_files.py b/python_pkg/token_dashboard/context_files.py new file mode 100644 index 00000000..b94a9ee7 --- /dev/null +++ b/python_pkg/token_dashboard/context_files.py @@ -0,0 +1,162 @@ +"""Which files enter Claude's context, and how big each has been over time. + +:func:`find` lists them per file (token_audit's :mod:`surfaces` sums them per +surface, which cannot say which rule grew). Only the list and the size history +are stored -- the dashboard reads each file live when asked, so it never shows +a stale copy. + +Size history comes from two places, both cheap: ``~/.claude``'s own git +history (one ``git log --raw`` plus one ``git cat-file --batch-check`` for +every blob, incremental from the last commit seen) and the weekly prune's +``prune//sizes*.json`` snapshots. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +from pathlib import Path +from typing import TYPE_CHECKING + +from python_pkg.token_audit import pricing +from python_pkg.token_audit.surfaces import CLAUDE_DIR +from python_pkg.token_dashboard import gitcmd + +if TYPE_CHECKING: + import sqlite3 + +SRC_DIR = Path.home() / "src" +MEMORY_LIMIT = 24_400 # MEMORY.md is only read up to ~24.4 KB +CLAUDE_MD_LIMIT = 40_000 # Claude Code warns about a CLAUDE.md over 40k chars +DESCRIPTION_LIMIT = 1024 # a skill description's maximum length +_BATCH_FIELDS = 3 # cat-file --batch-check: + + +@dataclass(frozen=True) +class ContextFile: + """One file and how it reaches the context.""" + + path: str + kind: str + standing: str # always | in repo | description + limit: int | None + + +def find(claude: Path | None = None, src: Path | None = None) -> list[ContextFile]: + """Every context file on disk (on-demand backups excluded).""" + claude = claude or CLAUDE_DIR + src = src or SRC_DIR + out = [ + ContextFile( + str(claude / "CLAUDE.md"), "global CLAUDE.md", "always", CLAUDE_MD_LIMIT + ) + ] + groups = ( + ("memories", "*.md", "memories (@-imported)", "always", None), + ("rules", "**/*.md", "rules (loaded recursively)", "always", None), + ("skills", "*/SKILL.md", "skill", "description", DESCRIPTION_LIMIT), + ("agents", "*.md", "agent definition", "description", DESCRIPTION_LIMIT), + ("output-styles", "*.md", "output style", "always", None), + ) + for folder, pattern, kind, standing, limit in groups: + out += [ + ContextFile(str(p), kind, standing, limit) + for p in sorted((claude / folder).glob(pattern)) + if p.is_file() + ] + memory = claude / "projects" / "-home-kuhy" / "memory" / "MEMORY.md" + out.append(ContextFile(str(memory), "auto-memory index", "always", MEMORY_LIMIT)) + for name in ("CLAUDE.md", "AGENTS.md"): + out += [ + ContextFile(str(p), f"repo {name}", "in repo", CLAUDE_MD_LIMIT) + for p in sorted(src.glob(f"*/{name}")) + ] + return [f for f in out if Path(f.path).is_file()] + + +def _git(repo: Path, *args: str, stdin: str | None = None) -> str: + """Run git, returning stdout ('' on failure).""" + return gitcmd.run(repo, *args, stdin=stdin) or "" + + +def _git_sizes(claude: Path, since: str) -> tuple[list[tuple[str, float, str]], str]: + """``(relpath, commit time, blob)`` for commits after ``since``; new head.""" + head = _git(claude, "rev-parse", "HEAD").strip() + if not head or head == since: + return [], since + span = f"{since}..HEAD" if since else "HEAD" + out = _git(claude, "log", "--raw", "--no-abbrev", "--format=@%ct", span) + rows: list[tuple[str, float, str]] = [] + stamp = 0.0 + for line in out.splitlines(): + if line.startswith("@"): + stamp = float(line[1:]) + elif line.startswith(":") and line.endswith(".md"): + meta, _, rel = line.partition("\t") + blob = meta.split()[3] + rows.append((rel, stamp, blob)) + return rows, head + + +def refresh( + conn: sqlite3.Connection, claude: Path | None = None, src: Path | None = None +) -> None: + """Re-list context files and append any new size-history points.""" + claude = claude or CLAUDE_DIR + files = find(claude, src) + conn.execute("DELETE FROM context_files") + conn.executemany( + "INSERT INTO context_files VALUES (?,?,?,?)", + [(f.path, f.kind, f.standing, f.limit) for f in files], + ) + row = conn.execute("SELECT value FROM meta WHERE key='context_git_head'").fetchone() + rows, head = _git_sizes(claude, row[0] if row else "") + zero = "0" * 40 + blobs = sorted({b for _, _, b in rows if b != zero}) + sizes: dict[str, int] = {} + batch = _git(claude, "cat-file", "--batch-check", stdin="\n".join(blobs) + "\n") + for line in batch.splitlines(): + parts = line.split() + if len(parts) == _BATCH_FIELDS and parts[2].isdigit(): + sizes[parts[0]] = int(parts[2]) + conn.executemany( + "INSERT OR IGNORE INTO context_sizes VALUES (?,?,?,?)", + [(str(claude / rel), ts, sizes.get(b, 0), "git") for rel, ts, b in rows], + ) + conn.execute("INSERT OR REPLACE INTO meta VALUES ('context_git_head', ?)", (head,)) + for path in sorted((claude / "token-report" / "prune").glob("*/sizes*.json")): + try: + data = json.loads(path.read_text(encoding="utf-8")).get("file_bytes", {}) + except ValueError, OSError, AttributeError: + continue + stamp = path.stat().st_mtime + conn.executemany( + "INSERT OR IGNORE INTO context_sizes VALUES (?,?,?,?)", + [ + (str(claude / rel), stamp, size, f"prune {path.parent.name}") + for rel, size in data.items() + if isinstance(size, int) + ], + ) + + +def price_mix(conn: sqlite3.Connection, now: float, days: float = 7.0) -> float: + """Call-weighted cache-read USD/MTok over the last ``days``, into ``meta``. + + A standing file costs its tokens x calls/day x this price: every call + re-reads the whole prefix from cache. + """ + book = pricing.load() + rows = conn.execute( + "SELECT model, COUNT(*) FROM turns WHERE ts > ? GROUP BY model", + (now - days * 86400,), + ).fetchall() + calls = sum(n for _, n in rows) + price = ( + sum(book.for_model(m).cache_read * n for m, n in rows) / calls if calls else 0.0 + ) + conn.execute( + "INSERT OR REPLACE INTO meta VALUES ('cache_read_usd_per_mtok', ?)", + (str(price),), + ) + return price diff --git a/python_pkg/token_dashboard/detail.py b/python_pkg/token_dashboard/detail.py new file mode 100644 index 00000000..4dda523a --- /dev/null +++ b/python_pkg/token_dashboard/detail.py @@ -0,0 +1,207 @@ +"""Collect the detail rows (calls, marks, turn facts) during a file walk. + +The walker feeds every record, tool call, tool result and priced turn here; +:meth:`Detail.finish` settles creation and carry cost on each row. Marks come +from what Claude Code writes next to the conversation (see :mod:`marks`). +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +from python_pkg.token_audit import parse, records +from python_pkg.token_audit.pricing import WRITE_1H, WRITE_5M +from python_pkg.token_dashboard import blocks, carry, classify, hook_join, marks +from python_pkg.token_dashboard.detail_rows import ( + CallRow, + DetailRows, + MarkRow, + TurnDetail, + Where, +) + +if TYPE_CHECKING: + from python_pkg.token_audit.model import Turn + from python_pkg.token_audit.pricing import PriceBook + +CHARS = parse.CHARS_PER_TOKEN + + +def write_multiplier(usage: dict[str, int]) -> float: + """Average cache-write price of one call, as a multiple of input.""" + total = usage.get("cache_creation_input_tokens", 0) + if not total: + return WRITE_5M + one_hour = min(usage.get(records.CACHE_1H, 0), total) + return (one_hour * WRITE_1H + (total - one_hour) * WRITE_5M) / total + + +def _message(rec: dict[str, Any]) -> dict[str, Any]: + """A record's ``message`` dict, empty when absent.""" + message = rec.get("message") + return message if isinstance(message, dict) else {} + + +class _Output: + """Visible output of one API message, accumulated over its records.""" + + def __init__(self, latency: float) -> None: + """Start empty; ``latency`` is the gap before the first record.""" + self.latency = latency + self.text = 0 + self.tool = 0 + self.thinking = False + + def add(self, block: dict[str, Any]) -> None: + """Count one content block.""" + kind = block.get("type") + if kind == "text": + self.text += len(str(block.get("text") or "")) + elif kind == "tool_use": + self.tool += len(json.dumps(block.get("input") or {})) + elif kind in {"thinking", "redacted_thinking"}: + self.thinking = True + + def split(self, output: int) -> tuple[int, int, int]: + """``(text, tool, thinking)`` tokens summing to ``output``. + + Visible blocks are estimated by length; billed output left over is + thinking (whose text Claude Code does not keep). With no thinking + block, or more visible text than output, the visible estimates are + scaled to the billed total instead. + """ + text, tool = self.text // CHARS, self.tool // CHARS + visible = text + tool + if not visible: + return (0, 0, output) if self.thinking else (output, 0, 0) + if visible > output or not self.thinking: + text = round(output * text / visible) + return text, output - text, 0 + return text, tool, output - visible + + +class Detail: + """Detail state for one file.""" + + def __init__(self) -> None: + """Nothing seen yet.""" + self.rows = DetailRows() + self.tracker = carry.Tracker() + self.calls: dict[str, CallRow] = {} + self.outputs: dict[str, _Output] = {} + self.last_ts = 0.0 + self.model = "" + self.hooks = hook_join.Hooks() + + def mark(self, found: marks.Found, ts: float, prompt: int) -> MarkRow: + """Turn a found mark into a row; carry it if it sits in context.""" + row = MarkRow( + Where(prompt, ts, self.model), + found.kind, + found.name, + detail=found.detail, + count=found.count, + tokens=found.tokens, + extra=found.extra, + duration_ms=found.duration_ms, + ) + self.rows.marks.append(row) + if row.tokens and found.kind not in marks.NOT_IN_CONTEXT: + self.tracker.result(row.cost, row.tokens) + self.hooks.track(row, found) + return row + + def record(self, rec: dict[str, Any], ts: float, prompt: int) -> None: + """Account one record (after the walker opened any new prompt).""" + kind = rec.get("type") + if kind == "assistant": + self.assistant(rec, ts, prompt) + elif kind == "user" and blocks.prompt_text(rec) is not None: + self.hooks.prompt_opened() + self.last_ts = ts + for found in marks.of_record(rec): + self.mark(found, ts, prompt) + + def assistant(self, rec: dict[str, Any], ts: float, prompt: int) -> None: + """Error placeholders become marks; real messages add visible output.""" + message = _message(rec) + if rec.get("isApiErrorMessage"): + text = blocks.text_of(message.get("content")) + detail = str(rec.get("apiError") or text)[: classify.TARGET_CHARS] + name = str(rec.get("error") or "unknown") + self.mark(marks.Found("api_error", name, detail=detail), ts, prompt) + return + mid = str(message.get("id") or "") + if mid not in self.outputs: + self.outputs[mid] = _Output(max(0.0, ts - (self.last_ts or ts))) + for block in parse.content_blocks(rec): + self.outputs[mid].add(block) + + def use( + self, rec: dict[str, Any], block: dict[str, Any], ts: float, prompt: int + ) -> None: + """Open a call row for a ``tool_use`` block.""" + message = _message(rec) + raw = block.get("input") + tool_input: dict[str, object] = raw if isinstance(raw, dict) else {} + name = str(block.get("name") or "unknown") + row = CallRow( + Where(prompt, ts, str(message.get("model") or self.model)), + name=name, + family=classify.family(name, tool_input), + target=classify.target(name, tool_input), + sig=classify.signature(name, tool_input), + message_id=str(message.get("id") or ""), + ) + self.calls[str(block.get("id") or "")] = row + self.rows.calls.append(row) + + def result(self, block: dict[str, Any], ts: float) -> None: + """Close the call a ``tool_result`` answers.""" + row = self.calls.pop(str(block.get("tool_use_id") or ""), None) + if row is None: + return + content = block.get("content") + images, text = marks.split_images(content) + row.end_ts = ts + row.images = images + row.result_tokens = images * parse.IMAGE_TOKENS + len(text) // CHARS + if block.get("is_error"): + row.error_kind = classify.error_kind(blocks.text_of(content)) + self.tracker.result(row.cost, row.result_tokens) + + def turn(self, turn: Turn, cost: float, prompt: int) -> None: + """Note a priced API call.""" + usage = turn.usage + self.model = turn.model + self.tracker.turn(turn.model, turn.context, write_multiplier(usage)) + self.hooks.turn(cost, turn.timestamp) + output = self.outputs.pop(turn.message_id, None) or _Output(0.0) + text, tool, think = output.split(usage.get("output_tokens", 0)) + one_hour = usage.get(records.CACHE_1H, 0) + self.rows.turns.append( + TurnDetail( + Where(prompt, turn.timestamp, turn.model), + latency_s=output.latency, + cw_1h=one_hour, + cw_5m=usage.get("cache_creation_input_tokens", 0) - one_hour, + out_text=text, + out_tool=tool, + out_think=think, + tool_calls=turn.tool_calls, + ) + ) + + def finish(self, book: PriceBook) -> DetailRows: + """Settle costs, fill batch sizes and models, merge repeated marks.""" + self.tracker.settle(book) + per_message: dict[str, int] = {} + for call in self.rows.calls: + per_message[call.message_id] = per_message.get(call.message_id, 0) + 1 + for call in self.rows.calls: + call.batch = per_message[call.message_id] + first_model = self.rows.turns[0].where.model if self.rows.turns else "" + self.hooks.resolve() + self.rows.marks = hook_join.merge(self.rows.marks, first_model) + return self.rows diff --git a/python_pkg/token_dashboard/detail_rows.py b/python_pkg/token_dashboard/detail_rows.py new file mode 100644 index 00000000..acf09d71 --- /dev/null +++ b/python_pkg/token_dashboard/detail_rows.py @@ -0,0 +1,91 @@ +"""Row types for the detail tables: one tool call, one context mark, one turn. + +Each call and mark carries a :class:`Cost` (see :mod:`carry`): what putting +the chunk into the prompt cache cost, and what re-reading it on every later +call of the same file cost until a compaction dropped it. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class Cost: + """Creation + carry of one chunk of context, and the calls that carried it.""" + + create_usd: float = 0.0 + carry_usd: float = 0.0 + carry_calls: int = 0 + + +@dataclass(frozen=True) +class Where: + """When something happened: prompt index, timestamp, serving model.""" + + prompt_idx: int + ts: float + model: str + + +@dataclass +class CallRow: + """One tool call, paired with its result (``error_kind`` empty when ok).""" + + where: Where + name: str + family: str + target: str + sig: str + message_id: str + end_ts: float = 0.0 + batch: int = 1 + result_tokens: int = 0 + images: int = 0 + error_kind: str = "" + cost: Cost = field(default_factory=Cost) + + +@dataclass +class MarkRow: + """Something that entered (or touched) context without being a tool call. + + ``kind`` is one of ``hook_run``, ``hook_context``, ``hook_block``, + ``hook_error``, ``hook_msg``, ``attach``, ``compaction``, ``api_error``, + ``interrupt``, ``image``. Rows with the same prompt, kind, name, detail + and model are summed into one, with ``count`` occurrences. + """ + + where: Where + kind: str + name: str + detail: str = "" + count: int = 1 + tokens: int = 0 + extra: int = 0 + duration_ms: int = 0 + retry_usd: float = 0.0 + cost: Cost = field(default_factory=Cost) + + +@dataclass(frozen=True) +class TurnDetail: + """Per-API-call facts the ``turns`` table does not hold.""" + + where: Where + latency_s: float + cw_1h: int + cw_5m: int + out_text: int + out_tool: int + out_think: int + tool_calls: int + + +@dataclass +class DetailRows: + """Everything one file contributes to the detail tables.""" + + calls: list[CallRow] = field(default_factory=list) + marks: list[MarkRow] = field(default_factory=list) + turns: list[TurnDetail] = field(default_factory=list) diff --git a/python_pkg/token_dashboard/detail_store.py b/python_pkg/token_dashboard/detail_store.py new file mode 100644 index 00000000..00d3bb2a --- /dev/null +++ b/python_pkg/token_dashboard/detail_store.py @@ -0,0 +1,110 @@ +"""Write one file's detail rows (calls, marks, turn facts) into the store. + +Every row repeats the owning file's key, main session key, subagent flag and +cwd, plus its own model, timestamp and local day, so the dashboard's shared +filter clause (date range, cwd, model, scope) applies to these tables as-is. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import sqlite3 + + from python_pkg.token_dashboard.detail_rows import Cost, Where + from python_pkg.token_dashboard.rows import FileFacts + +CLEAR = ( + "DELETE FROM calls WHERE key = ?", + "DELETE FROM marks WHERE key = ?", + "DELETE FROM turn_detail WHERE key = ?", +) + +Owner = tuple[str, str, int, str | None] + + +def day_of(stamp: float) -> str: + """Local calendar day of a unix timestamp.""" + return datetime.fromtimestamp(stamp).astimezone().strftime("%Y-%m-%d") + + +def _where(where: Where) -> tuple[str, float, str, int]: + """``model, ts, day, prompt_idx`` columns.""" + return (where.model, where.ts, day_of(where.ts), where.prompt_idx) + + +def _cost(cost: Cost) -> tuple[float, float, int]: + """``create_usd, carry_usd, carry_calls`` columns.""" + return (cost.create_usd, cost.carry_usd, cost.carry_calls) + + +def write(conn: sqlite3.Connection, facts: FileFacts, owner: Owner) -> None: + """Insert the detail rows; :func:`store.write` already cleared old ones.""" + detail = facts.rows.detail + conn.executemany( + "INSERT INTO calls (key, main_key, is_sub, cwd, model, ts, day," + " prompt_idx, end_ts, name, family, target, sig, batch, result_tokens," + " images, is_error, error_kind, create_usd, carry_usd, carry_calls)" + " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + [ + ( + *owner, + *_where(c.where), + c.end_ts, + c.name, + c.family, + c.target, + c.sig, + c.batch, + c.result_tokens, + c.images, + int(bool(c.error_kind)), + c.error_kind, + *_cost(c.cost), + ) + for c in detail.calls + ], + ) + conn.executemany( + "INSERT INTO marks (key, main_key, is_sub, cwd, model, ts, day," + " prompt_idx, kind, name, detail, count, tokens, extra, duration_ms," + " retry_usd, create_usd, carry_usd, carry_calls)" + " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + [ + ( + *owner, + *_where(m.where), + m.kind, + m.name, + m.detail, + m.count, + m.tokens, + m.extra, + m.duration_ms, + m.retry_usd, + *_cost(m.cost), + ) + for m in detail.marks + ], + ) + conn.executemany( + "INSERT INTO turn_detail (key, main_key, is_sub, cwd, model, ts, day," + " prompt_idx, latency_s, cw_1h, cw_5m, out_text, out_tool, out_think," + " tool_calls) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + [ + ( + *owner, + *_where(t.where), + t.latency_s, + t.cw_1h, + t.cw_5m, + t.out_text, + t.out_tool, + t.out_think, + t.tool_calls, + ) + for t in detail.turns + ], + ) diff --git a/python_pkg/token_dashboard/efficiency.py b/python_pkg/token_dashboard/efficiency.py new file mode 100644 index 00000000..5cf770ac --- /dev/null +++ b/python_pkg/token_dashboard/efficiency.py @@ -0,0 +1,153 @@ +"""Three 0-token efficiency trends, weekly. + +1. **USD per DONE session** -- all spend of status-tagged sessions in a week + divided by how many of them ended on ``DONE:``. Failed and parked sessions + count against the done ones, which is the point. Sessions with no tag at all + (before the convention) are counted but kept out of the ratio. +2. **USD per surviving commit** -- the week's total spend divided by commits + made that week that survived :mod:`survival`'s 14-day rule. +3. **USD per estimated size unit** -- spend of sessions the task-size estimator + sized, divided by the sum of its ``est_calls`` for them. +""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +from datetime import datetime, timedelta +import json +from typing import TYPE_CHECKING, Any + +from python_pkg.token_dashboard import survival + +if TYPE_CHECKING: + from collections.abc import Iterable + import sqlite3 + +Week = dict[str, Any] + + +@dataclass(frozen=True) +class SessionCost: + """What one main session cost and how it ended.""" + + key: str + first_ts: float + usd: float + tag: str | None + + +def week_of(stamp: float) -> str: + """Monday (local time) of the week containing ``stamp``.""" + day = datetime.fromtimestamp(stamp).astimezone().date() + return (day - timedelta(days=day.weekday())).isoformat() + + +def _ratio(part: float, whole: float) -> float | None: + """``part / whole``, ``None`` when ``whole`` is zero.""" + return part / whole if whole else None + + +def done_weekly(rows: Iterable[SessionCost]) -> list[Week]: + """USD per DONE session and the DONE share, per week.""" + weeks: dict[str, Week] = defaultdict( + lambda: {"done": 0, "other": 0, "untagged": 0, "usd_tagged": 0.0} + ) + for row in rows: + week = weeks[week_of(row.first_ts)] + if row.tag is None: + week["untagged"] += 1 + continue + week["usd_tagged"] += row.usd + week["done" if row.tag == "DONE" else "other"] += 1 + out = [] + for name in sorted(weeks): + week = weeks[name] + tagged = week["done"] + week["other"] + week |= { + "week": name, + "usd_per_done": _ratio(week["usd_tagged"], week["done"]), + "done_share": _ratio(week["done"], tagged), + } + out.append(week) + return out + + +def commits_weekly( + verdicts: Iterable[tuple[float, str]], week_usd: dict[str, float] +) -> list[Week]: + """USD per surviving commit, with lost / pending / unknown counts.""" + weeks: dict[str, Week] = defaultdict( + lambda: {"survived": 0, "lost": 0, "pending": 0, "unknown": 0} + ) + for stamp, verdict in verdicts: + week = weeks[week_of(stamp)] + if verdict == "survived": + week["survived"] += 1 + elif verdict in survival.FINAL: + week["lost"] += 1 + elif verdict == "pending": + week["pending"] += 1 + else: + week["unknown"] += 1 + out = [] + for name in sorted(weeks): + usd = week_usd.get(name, 0.0) + week = weeks[name] + week |= { + "week": name, + "usd": usd, + "usd_per_survivor": _ratio(usd, week["survived"]), + } + out.append(week) + return out + + +def estimate_weekly(rows: Iterable[SessionCost], est: dict[str, int]) -> list[Week]: + """USD per estimated call, over sessions the estimator sized.""" + weeks: dict[str, Week] = defaultdict(lambda: {"usd": 0.0, "est_calls": 0, "n": 0}) + for row in rows: + if row.key not in est: + continue + week = weeks[week_of(row.first_ts)] + week["usd"] += row.usd + week["est_calls"] += est[row.key] + week["n"] += 1 + return [ + weeks[name] + | { + "week": name, + "usd_per_unit": _ratio(weeks[name]["usd"], weeks[name]["est_calls"]), + } + for name in sorted(weeks) + ] + + +def rebuild(conn: sqlite3.Connection, now: float) -> None: + """Recompute the three weekly series into ``meta['efficiency']``.""" + rows = [ + SessionCost(key, first, usd, tag) + for key, first, usd, tag in conn.execute( + "SELECT key, first_ts, usd, final_tag FROM session_summary" + ) + ] + week_usd: dict[str, float] = defaultdict(float) + for row in rows: + week_usd[week_of(row.first_ts)] += row.usd + judged = conn.execute( + "SELECT c.ts, v.verdict FROM session_commits c" + " LEFT JOIN commit_verdicts v ON v.sha = c.sha" + ).fetchall() + verdicts = [ + (ts, verdict or ("pending" if now - ts < survival.WINDOW_S else "unchecked")) + for ts, verdict in judged + ] + est = dict(conn.execute("SELECT session_id, est_calls FROM estimates")) + payload = { + "done": done_weekly(rows), + "commits": commits_weekly(verdicts, week_usd), + "estimate": estimate_weekly(rows, est), + } + conn.execute( + "INSERT OR REPLACE INTO meta VALUES ('efficiency', ?)", (json.dumps(payload),) + ) diff --git a/python_pkg/token_dashboard/finalize.py b/python_pkg/token_dashboard/finalize.py new file mode 100644 index 00000000..926b9a43 --- /dev/null +++ b/python_pkg/token_dashboard/finalize.py @@ -0,0 +1,217 @@ +"""Derive per-session, per-prompt and per-subagent summaries from raw rows. + +Run for every main session touched by an ingest pass. A subagent file may be +ingested in a different pass than its parent, so everything here is computed +from the store, never from the in-memory walk. + +Linking a subagent to the prompt that spawned it: the ``toolUseId`` in its +``.meta.json`` (or the ``agentId`` the ``Agent`` result reported) names an +``Agent`` call; that call's prompt is the answer. A depth-2 agent's call sits +in its parent agent's file, so the chain is followed upward. An agent with no +resolvable spawn -- one that outlived ``/clear`` -- stays ``prompt_idx NULL`` +("unattributed") rather than being dropped, so session cost always reconciles. +""" + +from __future__ import annotations + +from collections import defaultdict +import json +from typing import TYPE_CHECKING, Any + +from python_pkg.token_dashboard import active + +if TYPE_CHECKING: + import sqlite3 + +_MAX_DEPTH = 8 + + +def _rows(conn: sqlite3.Connection, sql: str, *args: object) -> list[Any]: + """Run a query and return all rows.""" + return conn.execute(sql, args).fetchall() + + +class _Session: + """Everything stored for one main session and its subagents.""" + + def __init__(self, conn: sqlite3.Connection, main: str) -> None: + self.main = main + self.sessions = { + r[0]: r + for r in _rows( + conn, + "SELECT key, cwd, agent_type, description, spawn_tool_use_id," + " agent_model, first_ts, last_ts, final_tag" + " FROM sessions WHERE main_key = ?", + main, + ) + } + self.turns = _rows( + conn, + "SELECT key, prompt_idx, usd, model FROM turns WHERE main_key = ?", + main, + ) + keys = json.dumps(list(self.sessions)) + self.calls = _rows( + conn, + "SELECT key, tool_use_id, agent_id, prompt_idx, description," + " subagent_type FROM agent_calls WHERE key IN" + " (SELECT value FROM json_each(?))", + keys, + ) + self.spans_by_key: dict[str, list[active.Span]] = defaultdict(list) + for key, start, end in _rows( + conn, + "SELECT key, start, end FROM activity WHERE key IN" + " (SELECT value FROM json_each(?))", + keys, + ): + self.spans_by_key[key].append((start, end)) + self.spans = [s for spans in self.spans_by_key.values() for s in spans] + self.prompts = _rows( + conn, + "SELECT idx, ts, text FROM prompts WHERE key = ? ORDER BY idx", + main, + ) + self.usd: dict[str, float] = defaultdict(float) + self.ncalls: dict[str, int] = defaultdict(int) + self.models: dict[str, dict[str, float]] = defaultdict( + lambda: defaultdict(float) + ) + for key, _, usd, model in self.turns: + self.usd[key] += usd + self.ncalls[key] += 1 + self.models[key][model] += usd + + def spawn(self, sub: str) -> tuple[Any, ...] | None: + """The ``Agent`` call that spawned ``sub``, if any is stored.""" + tool_use_id = self.sessions[sub][4] + agent_id = sub.rsplit("/agent-", 1)[-1] + for call in self.calls: + if (tool_use_id and call[1] == tool_use_id) or call[2] == agent_id: + return tuple(call) + return None + + def resolve(self, sub: str) -> tuple[int | None, int, tuple[Any, ...] | None]: + """``(prompt_idx, depth, spawning call)`` for a subagent.""" + first = self.spawn(sub) + call, depth = first, 1 + while call is not None and call[0] != self.main and depth < _MAX_DEPTH: + call = self.spawn(call[0]) if call[0] in self.sessions else None + depth += 1 + return (call[3] if call is not None else None), depth, first + + +def _sub_rows(state: _Session) -> list[tuple[Any, ...]]: + """One ``subagent_link`` row per subagent of the session.""" + out = [] + for key, row in state.sessions.items(): + if key == state.main: + continue + prompt_idx, depth, call = state.resolve(key) + models = state.models.get(key) or {} + model = max(models, key=models.__getitem__) if models else row[5] + spans = state.spans_by_key.get(key, []) + out.append( + ( + key, + state.main, + prompt_idx, + depth, + row[3] or (call[4] if call else "") or key, + row[2] or (call[5] if call else "") or "unknown", + model, + state.usd.get(key, 0.0), + state.ncalls.get(key, 0), + row[6], + row[7], + row[7] - row[6], + active.measure(spans), + ) + ) + return out + + +def refresh(conn: sqlite3.Connection, main: str) -> None: + """Rebuild every summary row for one main session.""" + state = _Session(conn, main) + conn.execute("DELETE FROM subagent_link WHERE main_key = ?", (main,)) + conn.execute("DELETE FROM prompt_summary WHERE key = ?", (main,)) + conn.execute("DELETE FROM session_summary WHERE key = ?", (main,)) + if not state.sessions: + return + subs = _sub_rows(state) + conn.executemany( + "INSERT INTO subagent_link VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", subs + ) + first = min((r[6] for r in state.sessions.values() if r[6]), default=0.0) + last = max(r[7] for r in state.sessions.values()) + _prompt_rows(conn, state, subs, last) + own = state.sessions.get(main) + usd_main = state.usd.get(main, 0.0) + usd_sub = sum(r[7] for r in subs) + merged: dict[str, float] = defaultdict(float) + for models in state.models.values(): + for model, usd in models.items(): + merged[model] += usd + conn.execute( + "INSERT INTO session_summary VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + main, + own[1] if own else next(iter(state.sessions.values()))[1], + state.prompts[0][2] if state.prompts else (subs[0][4] if subs else ""), + first, + last, + last - first, + active.measure(state.spans), + usd_main + usd_sub, + usd_main, + usd_sub, + sum(state.ncalls.values()), + len(state.prompts), + len(subs), + json.dumps(dict(merged)), + own[8] if own else None, + ), + ) + + +def _prompt_rows( + conn: sqlite3.Connection, + state: _Session, + subs: list[tuple[Any, ...]], + last: float, +) -> None: + """One ``prompt_summary`` row per prompt, plus one for pre-prompt turns.""" + own: dict[int, float] = defaultdict(float) + calls: dict[int, int] = defaultdict(int) + for key, idx, usd, _ in state.turns: + if key == state.main: + own[idx] += usd + calls[idx] += 1 + sub_usd: dict[int, float] = defaultdict(float) + for row in subs: + if row[2] is not None: + sub_usd[row[2]] += row[7] + prompts = list(state.prompts) + if own.get(-1) or sub_usd.get(-1): + start = min((s for s, _ in state.spans), default=0.0) + prompts.insert(0, (-1, start, "(before the first prompt)")) + bounds = [ts for _, ts, _ in prompts[1:]] + ([last + 1] if prompts else []) + conn.executemany( + "INSERT INTO prompt_summary VALUES (?,?,?,?,?,?,?,?,?)", + [ + ( + state.main, + idx, + ts, + text, + own.get(idx, 0.0), + sub_usd.get(idx, 0.0), + active.measure(state.spans, ts, end), + min(end, last) - ts, + calls.get(idx, 0), + ) + for (idx, ts, text), end in zip(prompts, bounds, strict=True) + ], + ) diff --git a/python_pkg/token_dashboard/gitcmd.py b/python_pkg/token_dashboard/gitcmd.py new file mode 100644 index 00000000..22f3d9ac --- /dev/null +++ b/python_pkg/token_dashboard/gitcmd.py @@ -0,0 +1,29 @@ +"""One way to run git for the dashboard: absolute binary, bounded, never raises.""" + +from __future__ import annotations + +import shutil +import subprocess +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + +GIT = shutil.which("git") or "/usr/bin/git" +TIMEOUT_S = 30 + + +def run(repo: str | Path, *args: str, stdin: str | None = None) -> str | None: + """``git -C repo args``: stdout, or ``None`` on any failure.""" + try: + done = subprocess.run( + [GIT, "-C", str(repo), *args], + input=stdin, + capture_output=True, + text=True, + check=False, + timeout=TIMEOUT_S, + ) + except OSError, subprocess.TimeoutExpired: + return None + return done.stdout if done.returncode == 0 else None diff --git a/python_pkg/token_dashboard/hook_join.py b/python_pkg/token_dashboard/hook_join.py new file mode 100644 index 00000000..d8b15fbc --- /dev/null +++ b/python_pkg/token_dashboard/hook_join.py @@ -0,0 +1,91 @@ +"""Join hook context to the run that produced it; price forced retries; merge. + +``hook_additional_context`` names no command, so it is matched to a +``hook_success`` run of the same tool-use id and hook name whose stdout +contains the context text. A ``hook_blocking_error`` costs the calls it +forces: a Stop bounce every call until the next prompt, any other block the +one call that reacts to it. +""" + +from __future__ import annotations + +from dataclasses import replace +import json +from typing import TYPE_CHECKING + +from python_pkg.token_dashboard.marks import SNIPPET + +if TYPE_CHECKING: + from python_pkg.token_dashboard.detail_rows import MarkRow + from python_pkg.token_dashboard.marks import Found + +_STOPS = frozenset({"Stop", "SubagentStop"}) + + +class Hooks: + """Pending joins and open retry windows for one file.""" + + def __init__(self) -> None: + """Nothing seen yet.""" + self.runs: dict[tuple[str, str], list[tuple[str, str]]] = {} + self.pending: list[tuple[MarkRow, tuple[str, str], str]] = [] + self.until_prompt: list[MarkRow] = [] + self.next_turn: list[MarkRow] = [] + + def track(self, row: MarkRow, found: Found) -> None: + """Remember what later records or turns need to complete ``row``.""" + if found.kind == "hook_run": + self.runs.setdefault(found.key, []).append((row.detail, found.text)) + elif found.kind == "hook_context": + self.pending.append((row, found.key, found.text)) + elif found.kind == "hook_block": + stop = row.name in _STOPS + (self.until_prompt if stop else self.next_turn).append(row) + + def prompt_opened(self) -> None: + """A new prompt ends every Stop-bounce retry.""" + self.until_prompt.clear() + + def turn(self, cost: float, ts: float) -> None: + """Charge an API call made after a block to that block's retry. + + Turns are released one message late, so a turn released after the + block record may still predate it: the timestamp decides. + """ + for row in [*self.until_prompt, *self.next_turn]: + if ts > row.where.ts: + row.retry_usd += cost + self.next_turn = [row for row in self.next_turn if ts <= row.where.ts] + + def resolve(self) -> None: + """Name the script behind each injected hook context.""" + for row, key, text in self.pending: + snippet = text[:SNIPPET] + escaped = json.dumps(snippet)[1:-1] + for detail, stdout in self.runs.get(key, []): + if snippet and (snippet in stdout or escaped in stdout): + row.detail = detail + break + + +def merge(rows: list[MarkRow], first_model: str) -> list[MarkRow]: + """Sum marks sharing prompt, kind, name, detail and model; oldest first.""" + merged: dict[tuple[int, str, str, str, str], MarkRow] = {} + for row in rows: + row.where = replace(row.where, model=row.where.model or first_model) + where = row.where + key = (where.prompt_idx, row.kind, row.name, row.detail, where.model) + held = merged.get(key) + if held is None: + merged[key] = row + continue + held.where = replace(held.where, ts=min(held.where.ts, where.ts)) + held.count += row.count + held.tokens += row.tokens + held.extra += row.extra + held.duration_ms += row.duration_ms + held.retry_usd += row.retry_usd + held.cost.create_usd += row.cost.create_usd + held.cost.carry_usd += row.cost.carry_usd + held.cost.carry_calls += row.cost.carry_calls + return sorted(merged.values(), key=lambda r: r.where.ts) diff --git a/python_pkg/token_dashboard/ingest.py b/python_pkg/token_dashboard/ingest.py new file mode 100644 index 00000000..50190ad0 --- /dev/null +++ b/python_pkg/token_dashboard/ingest.py @@ -0,0 +1,137 @@ +"""Refresh the token store: changed transcripts first, then every aux source. + +Resumable by construction. Each pass walks changed files until its time budget +is spent (always at least one), commits, and exits :data:`EXIT_PENDING` if any +are left -- so a first backfill of months of transcripts is a handful of +sub-minute runs, and the systemd timer simply picks up where the last ended. + + python3 -m python_pkg.token_dashboard.ingest # one pass + python3 -m python_pkg.token_dashboard.ingest --budget 40 +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from pathlib import Path +import sys +import time +from typing import TYPE_CHECKING + +from python_pkg.token_audit import discover, history +from python_pkg.token_dashboard import ( + ab, + aux, + commits, + context_files, + efficiency, + finalize, + schema, + store, + survival, + walk, +) + +if TYPE_CHECKING: + import sqlite3 + +ROOT = Path.home() / ".claude" / "projects" +DEFAULT_BUDGET = 40.0 +EXIT_PENDING = 75 # EX_TEMPFAIL: run again to continue + + +@dataclass(frozen=True) +class Outcome: + """What one pass did.""" + + parsed: int + finalized: int + complete: bool + + +def _changed( + conn: sqlite3.Connection, root: Path +) -> list[tuple[Path, tuple[int, float]]]: + """Transcripts whose size or mtime differ from what the store last saw.""" + seen = { + path: (size, mtime) + for path, size, mtime in conn.execute("SELECT path, size, mtime FROM files") + } + out = [] + for pattern in discover.PATTERNS: + for path in root.glob(pattern): + stat = path.stat() + now = (stat.st_size, stat.st_mtime) + if seen.get(str(path)) != now: + out.append((path, now)) + # Oldest first: a backfill lands history in order, and a live session + # (newest mtime) is re-read last, closest to the pass's end. + return sorted(out, key=lambda item: item[1][1]) + + +def run( + root: Path = ROOT, + db_path: Path = schema.DEFAULT_DB, + report_dir: Path = history.REPORT_DIR, + budget_s: float = DEFAULT_BUDGET, +) -> Outcome: + """One resumable pass.""" + started = time.monotonic() + deadline = started + budget_s + conn = schema.connect(db_path) + try: + todo = _changed(conn, root) + mains: set[str] = set() + parsed = 0 + for path, stat in todo: + if parsed and time.monotonic() > deadline: + break + facts = walk.walk_file(path, root) + store.write(conn, facts, str(path), stat) + mains.add(facts.parent_session or facts.key) + parsed += 1 + for main_key in sorted(mains): + finalize.refresh(conn, main_key) + conn.commit() + complete = parsed == len(todo) + if complete: + aux.snapshots(conn, report_dir) + aux.lever_rows(conn, report_dir) + aux.autopsy(conn) + aux.activity_logs(conn, report_dir) + commits.discover(conn) + now = time.time() + complete = survival.update(conn, now, deadline, time.monotonic) + efficiency.rebuild(conn, now) + ab.rebuild(conn) + context_files.refresh(conn) + context_files.price_mix(conn, now) + conn.execute( + "INSERT OR REPLACE INTO meta VALUES ('refreshed_at', ?)", (str(now),) + ) + conn.commit() + finally: + conn.close() + return Outcome(parsed, len(mains), complete) + + +def main(argv: list[str] | None = None) -> int: + """CLI: 0 when the store is current, :data:`EXIT_PENDING` when not.""" + parser = argparse.ArgumentParser(prog="token_dashboard.ingest") + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--db", type=Path, default=schema.DEFAULT_DB) + parser.add_argument("--report-dir", type=Path, default=history.REPORT_DIR) + parser.add_argument("--budget", type=float, default=DEFAULT_BUDGET) + args = parser.parse_args(argv) + started = time.monotonic() + outcome = run(args.root, args.db, args.report_dir, args.budget) + sys.stderr.write( + f"token_dashboard: parsed {outcome.parsed} files, refreshed " + f"{outcome.finalized} sessions in {time.monotonic() - started:.1f}s" + f"{'' if outcome.complete else ' (more pending: run again)'}\n" + ) + return 0 if outcome.complete else EXIT_PENDING + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python_pkg/token_dashboard/levers.py b/python_pkg/token_dashboard/levers.py new file mode 100644 index 00000000..dcf309cd --- /dev/null +++ b/python_pkg/token_dashboard/levers.py @@ -0,0 +1,173 @@ +"""Every lever ever applied, proposed or passed over, with its outcome. + +Sources, newest form first: + +* ``history/*.json`` ``levers`` -- structured candidates (snapshots from + 2026-10-18 on; see ``token_audit.review``). +* Markdown "Candidate levers" tables -- for older snapshots. WEEKLY.md is + overwritten every run, so older tables survive only in transcript tool + results; :mod:`walk` keeps those as ``lever_tables`` rows. +* ``ledger.jsonl`` -- applied levers, scored by ``token_audit.history.score``. +* ``prune//PROPOSAL.md`` -- weekly standing-prefix prune proposals. + +Status of a candidate: ``applied`` if a ledger line on the same metric is dated +on or after the snapshot, ``not applied`` if a newer snapshot came and went +without one, else ``proposed``. Titles never match ledger text, so the join is +on the metric key. +""" + +from __future__ import annotations + +import json +import re +from typing import TYPE_CHECKING, Any + +from python_pkg.token_audit import history +from python_pkg.token_audit.recommend import TITLE_METRIC + +if TYPE_CHECKING: + from pathlib import Path + +Row = dict[str, Any] +DATE = re.compile(r"\d{4}-\d{2}-\d{2}") +_COLUMNS = 5 + + +def parse_candidates(markdown: str) -> list[Row]: + """Rows of a ``| category | lever | evidence | est $/day | action |`` table.""" + rows = [] + for line in markdown.splitlines(): + cells = [c.strip() for c in line.strip().strip("|").split("|")] + if len(cells) != _COLUMNS or cells[0] in {"category", ""}: + continue + if set(cells[0]) <= {"-"}: + continue + est = cells[3].lstrip("$").replace(",", "") + rows.append( + { + "category": cells[0], + "title": cells[1], + "evidence": cells[2], + "est_usd_per_day": float(est) + if est.replace(".", "").isdigit() + else None, + "action": cells[4], + "metric": TITLE_METRIC.get(cells[1], ""), + } + ) + return rows + + +def ledger_date(entry: Row) -> str | None: + """A ledger line's date, else one named in its text (prune lines).""" + if isinstance(entry.get("date"), str): + return str(entry["date"]) + found = DATE.search(str(entry.get("lever", ""))) + return found.group(0) if found else None + + +def _snapshots(report: Path) -> list[tuple[str, Row]]: + """``(date, snapshot)`` oldest first.""" + out = [] + for path in sorted((report / "history").glob("*.json")): + try: + snap = json.loads(path.read_text(encoding="utf-8")) + except ValueError: + continue + if isinstance(snap, dict): + out.append((path.stem[:10], snap)) + return out + + +def _applied(report: Path, latest: Row) -> list[Row]: + """Ledger lines, scored against the newest snapshot.""" + entries = history.read_ledger(report) + rows = [] + for entry, verdict in zip(entries, history.score(entries, latest), strict=True): + rows.append( + { + "source": "ledger", + "date": ledger_date(entry), + "lever": verdict.lever, + "metric": verdict.metric, + "baseline": verdict.baseline, + "predicted": verdict.predicted, + "actual": verdict.actual, + "lower_is_better": int(bool(entry.get("lower_is_better", True))), + "status": "applied", + "outcome": verdict.outcome, + "detail": f"{verdict.note}; undo: {entry.get('undo', '-')}", + } + ) + return rows + + +def _candidates( + snaps: list[tuple[str, Row]], tables: dict[str, str], applied: list[Row] +) -> list[Row]: + """Candidate rows per snapshot, structured first, markdown second.""" + rows = [] + for position, (day, snap) in enumerate(snaps): + found = snap.get("levers") + if not isinstance(found, list): + found = parse_candidates(tables.get(day, "")) + newer = position + 1 < len(snaps) + for lever in found: + metric = str(lever.get("metric") or "") + hit = any( + a["metric"] == metric and (a["date"] or "") >= day for a in applied + ) + status = "applied" if hit else ("not applied" if newer else "proposed") + rows.append( + { + "source": f"snapshot {day}", + "date": day, + "lever": lever.get("title"), + "metric": metric, + "baseline": snap.get("metrics", {}).get(metric), + "predicted": None, + "actual": None, + "lower_is_better": 1, + "status": status, + "outcome": None, + "detail": f"{lever.get('evidence')} -> {lever.get('action')}" + f" (est ${lever.get('est_usd_per_day') or '-'}/day)", + } + ) + return rows + + +def _prunes(report: Path, applied: list[Row]) -> list[Row]: + """One row per prune proposal.""" + paths = sorted((report / "prune").glob("*/PROPOSAL.md")) + rows = [] + for position, path in enumerate(paths): + day = path.parent.name + lines = [x for x in path.read_text(encoding="utf-8").splitlines() if x.strip()] + hit = any(f"prune {day}" in str(a["lever"]) for a in applied) + latest = position == len(paths) - 1 + status = "applied" if hit else ("proposed" if latest else "not applied") + rows.append( + { + "source": f"prune {day}", + "date": day, + "lever": lines[0].lstrip("# ") if lines else f"prune {day}", + "metric": "standing_tokens_est", + "baseline": None, + "predicted": None, + "actual": None, + "lower_is_better": 1, + "status": status, + "outcome": None, + "detail": lines[1] if len(lines) > 1 else "", + } + ) + return rows + + +def build(report: Path, tables: dict[str, str]) -> list[Row]: + """Every lever row, applied ones first.""" + snaps = _snapshots(report) + latest = snaps[-1][1] if snaps else {} + applied = _applied(report, latest) + return applied + _candidates(snaps, tables, applied) + _prunes(report, applied) diff --git a/python_pkg/token_dashboard/marks.py b/python_pkg/token_dashboard/marks.py new file mode 100644 index 00000000..b98de66f --- /dev/null +++ b/python_pkg/token_dashboard/marks.py @@ -0,0 +1,177 @@ +"""Read context marks out of the records that are not the conversation. + +Hooks write ``attachment`` records: ``hook_success`` (every run, with its +command and duration; SessionStart / UserPromptSubmit stdout is injected as +context), ``hook_additional_context`` (injected text, without the command -- +joined back to its run by :mod:`hook_join`), ``hook_blocking_error`` (a block +or a Stop-hook bounce) and ``hook_system_message`` (shown to the user only). +Other attachment types in :data:`INJECTED` are rendered into the prompt; +the rest (``prompt_snapshot``, ``environment`` ...) only record what was sent. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +from typing import TYPE_CHECKING, Any + +from python_pkg.token_audit import parse +from python_pkg.token_dashboard import blocks, classify + +if TYPE_CHECKING: + from collections.abc import Iterator + +CHARS = parse.CHARS_PER_TOKEN +# Marks that never sit in the model's context, so carry nothing. +NOT_IN_CONTEXT = frozenset({"hook_msg", "compaction", "api_error", "interrupt"}) +# Hook events whose plain stdout Claude Code adds to the context. +INJECTING = frozenset({"SessionStart", "UserPromptSubmit"}) +INJECTED = frozenset( + { + "nested_memory", + "edited_text_file", + "total_tokens_reminder", + "skill_listing", + "deferred_tools_delta", + "mcp_instructions_delta", + "agent_listing_delta", + "queued_command", + "task_reminder", + "silent_turn_reminder", + "file", + "todo", + "command_permissions", + } +) +_SCRIPTS = (".sh", ".py", ".js", ".ts", ".mjs") +SNIPPET = 40 + + +@dataclass(frozen=True) +class Found: + """One mark read from a record, before it becomes a row. + + ``key`` (tool-use id, hook name) and ``text`` only matter to hook runs + and hook context, which :mod:`hook_join` pairs up. + """ + + kind: str + name: str + detail: str = "" + count: int = 1 + tokens: int = 0 + extra: int = 0 + duration_ms: int = 0 + key: tuple[str, str] = ("", "") + text: str = "" + + +def short_command(command: str) -> str: + """A hook command's script name, or its first 40 characters.""" + words = command.split() + scripts = [w for w in words if w.endswith(_SCRIPTS)] + if scripts: + return scripts[0].rsplit("/", 1)[-1] + return " ".join(words)[:SNIPPET] + + +def _int(value: object) -> int: + """A number that may arrive as a string; 0 when it is neither.""" + try: + return int(float(str(value))) + except ValueError: + return 0 + + +def split_images(content: object) -> tuple[int, str]: + """Image block count and the JSON text of everything else.""" + if not isinstance(content, list): + return 0, "" if content is None else json.dumps(content) + images = [b for b in content if isinstance(b, dict) and b.get("type") == "image"] + rest = [b for b in content if b not in images] + return len(images), json.dumps(rest) + + +def _tokens(text: object) -> int: + """Estimated tokens of a text value.""" + return len(str(text or "")) // CHARS + + +def _hook(att: dict[str, Any]) -> Iterator[Found]: + """Marks of one hook attachment.""" + kind = att.get("type") + name = str(att.get("hookName") or "") + key = (str(att.get("toolUseID") or ""), name) + content = att.get("content") + if kind == "hook_success": + text = str(content or "") + injected = att.get("hookEvent") in INJECTING + injected = injected and not text.lstrip().startswith("{") + yield Found( + "hook_run", + name, + detail=short_command(str(att.get("command") or "")), + duration_ms=_int(att.get("durationMs")), + tokens=_tokens(text) if injected else 0, + key=key, + text=str(att.get("stdout") or ""), + ) + elif kind == "hook_additional_context": + for item in content if isinstance(content, list) else [content]: + text = str(item or "") + yield Found("hook_context", name, tokens=_tokens(text), key=key, text=text) + elif kind == "hook_blocking_error": + raw = att.get("blockingError") + error = raw if isinstance(raw, dict) else {"blockingError": raw} + command = short_command(str(error.get("command") or "")) + yield Found( + "hook_block", name, detail=command, tokens=_tokens(error["blockingError"]) + ) + elif kind == "hook_non_blocking_error": + yield Found("hook_error", name, tokens=_tokens(content)) + elif kind == "hook_system_message": + yield Found("hook_msg", name) + + +def _attachment(att: dict[str, Any]) -> Iterator[Found]: + """Marks of one attachment record.""" + kind = str(att.get("type") or "") + if kind.startswith("hook_"): + yield from _hook(att) + elif kind in INJECTED: + rest = {k: v for k, v in att.items() if k != "type"} + path = str(att.get("path") or "")[: classify.TARGET_CHARS] + yield Found("attach", kind, detail=path, tokens=_tokens(json.dumps(rest))) + + +def _user(rec: dict[str, Any]) -> Iterator[Found]: + """Interruptions and pasted images.""" + message = rec.get("message") + content = message.get("content") if isinstance(message, dict) else None + if blocks.text_of(content).startswith("[Request interrupted"): + yield Found("interrupt", "user") + images, _ = split_images(content) + if images: + yield Found("image", "user", count=images, tokens=images * parse.IMAGE_TOKENS) + + +def of_record(rec: dict[str, Any]) -> list[Found]: + """Every mark one non-assistant record carries.""" + kind = rec.get("type") + att = rec.get("attachment") + if kind == "attachment" and isinstance(att, dict): + return list(_attachment(att)) + if kind == "system" and rec.get("subtype") == "compact_boundary": + meta = rec.get("compactMetadata") + meta = meta if isinstance(meta, dict) else {} + found = Found( + "compaction", + str(meta.get("trigger") or "manual"), + tokens=_int(meta.get("preTokens")), + extra=_int(meta.get("postTokens")), + duration_ms=_int(meta.get("durationMs")), + ) + return [found] + if kind == "user": + return list(_user(rec)) + return [] diff --git a/python_pkg/token_dashboard/rows.py b/python_pkg/token_dashboard/rows.py new file mode 100644 index 00000000..079264a0 --- /dev/null +++ b/python_pkg/token_dashboard/rows.py @@ -0,0 +1,104 @@ +"""Row types one transcript file contributes to the store (see :mod:`walk`).""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +from python_pkg.token_dashboard.detail_rows import DetailRows + +if TYPE_CHECKING: + from python_pkg.token_dashboard import active + + +@dataclass(frozen=True) +class TurnRow: + """One deduplicated API call, priced.""" + + ts: float + model: str + usd: float + context: int + prompt_idx: int + usage: dict[str, int] + + +@dataclass(frozen=True) +class PromptRow: + """One typed user prompt.""" + + idx: int + ts: float + prompt_id: str + text: str + + +@dataclass +class ToolRow: + """Calls of one tool inside one prompt.""" + + prompt_idx: int + name: str + calls: int = 0 + result_tokens: int = 0 + usd: float = 0.0 + carry_usd: float = 0.0 + # Settled by the carry tracker too, but not stored per tool (the calls + # table holds creation per call). + create_usd: float = 0.0 + carry_calls: int = 0 + + +@dataclass(frozen=True) +class AgentCall: + """An ``Agent`` tool call: the spawn point of a subagent.""" + + tool_use_id: str + agent_id: str + prompt_idx: int + ts: float + description: str + subagent_type: str + model: str + result_ts: float + status: str + + +@dataclass(frozen=True) +class Commit: + """A commit hash printed by a Bash ``git commit`` in this file.""" + + prompt_idx: int + ts: float + end_ts: float + sha: str + repo: str + + +@dataclass +class FileRows: + """The row lists one transcript file contributes.""" + + turns: list[TurnRow] = field(default_factory=list) + prompts: list[PromptRow] = field(default_factory=list) + tools: list[ToolRow] = field(default_factory=list) + agent_calls: list[AgentCall] = field(default_factory=list) + intervals: list[active.Span] = field(default_factory=list) + lever_tables: list[tuple[float, str]] = field(default_factory=list) + commits: list[Commit] = field(default_factory=list) + messages: list[tuple[float, str]] = field(default_factory=list) + detail: DetailRows = field(default_factory=DetailRows) + + +@dataclass +class FileFacts: + """Everything one transcript file contributes to the store.""" + + key: str + session_id: str + parent_session: str | None + cwd: str | None + meta: dict[str, Any] + rows: FileRows = field(default_factory=FileRows) + final_tag: str | None = None + span: tuple[float, float] = (0.0, 0.0) diff --git a/python_pkg/token_dashboard/schema.py b/python_pkg/token_dashboard/schema.py new file mode 100644 index 00000000..52ee17ca --- /dev/null +++ b/python_pkg/token_dashboard/schema.py @@ -0,0 +1,192 @@ +"""SQLite layout of the durable token store. + +Raw tables (``files`` .. ``commits``) hold what one transcript file said and +are replaced wholesale when that file changes. They are **never** deleted when +the file disappears: Claude Code removes transcripts after +``cleanupPeriodDays``, and outliving that is the store's whole purpose. + +Summary tables (``session_summary`` .. ``subagent_link``) are derived from the +raw ones by :mod:`finalize` and may be rebuilt at any time. Aux tables hold the +sources that are not transcripts (snapshots, ledger, levers, autopsy rows). +""" + +from __future__ import annotations + +from pathlib import Path +import sqlite3 + +DEFAULT_DB = Path.home() / "data" / "token-dashboard" / "tokens.sqlite" +VERSION = "3" + +_RAW = """ +CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT); +CREATE TABLE IF NOT EXISTS files( + path TEXT PRIMARY KEY, key TEXT, size INT, mtime REAL); +CREATE TABLE IF NOT EXISTS sessions( + key TEXT PRIMARY KEY, session_id TEXT, main_key TEXT, cwd TEXT, + agent_type TEXT, description TEXT, spawn_tool_use_id TEXT, + parent_agent_id TEXT, agent_model TEXT, first_ts REAL, last_ts REAL, + final_tag TEXT); +CREATE TABLE IF NOT EXISTS turns( + key TEXT, main_key TEXT, is_sub INT, cwd TEXT, ts REAL, day TEXT, + model TEXT, usd REAL, context INT, prompt_idx INT, input INT, + cache_write INT, cache_read INT, output INT); +CREATE INDEX IF NOT EXISTS turns_key ON turns(key); +CREATE INDEX IF NOT EXISTS turns_day ON turns(day); +CREATE TABLE IF NOT EXISTS prompts( + key TEXT, idx INT, ts REAL, prompt_id TEXT, text TEXT); +CREATE INDEX IF NOT EXISTS prompts_key ON prompts(key); +CREATE TABLE IF NOT EXISTS tool_use( + key TEXT, prompt_idx INT, name TEXT, calls INT, result_tokens INT, + usd REAL, carry_usd REAL); +CREATE INDEX IF NOT EXISTS tool_use_key ON tool_use(key); +CREATE TABLE IF NOT EXISTS agent_calls( + key TEXT, tool_use_id TEXT, agent_id TEXT, prompt_idx INT, ts REAL, + description TEXT, subagent_type TEXT, model TEXT, result_ts REAL, + status TEXT); +CREATE INDEX IF NOT EXISTS agent_calls_key ON agent_calls(key); +CREATE TABLE IF NOT EXISTS activity(key TEXT, start REAL, end REAL); +CREATE INDEX IF NOT EXISTS activity_key ON activity(key); +CREATE TABLE IF NOT EXISTS sent_messages(key TEXT, ts REAL, to_agent TEXT); +CREATE TABLE IF NOT EXISTS lever_tables(key TEXT, ts REAL, markdown TEXT); +CREATE TABLE IF NOT EXISTS commits( + key TEXT, prompt_idx INT, ts REAL, end_ts REAL, sha TEXT, repo TEXT); +CREATE TABLE IF NOT EXISTS calls( + key TEXT, main_key TEXT, is_sub INT, cwd TEXT, model TEXT, ts REAL, + day TEXT, prompt_idx INT, end_ts REAL, name TEXT, family TEXT, + target TEXT, sig TEXT, batch INT, result_tokens INT, images INT, + is_error INT, error_kind TEXT, create_usd REAL, carry_usd REAL, + carry_calls INT); +CREATE INDEX IF NOT EXISTS calls_key ON calls(key); +CREATE INDEX IF NOT EXISTS calls_ts ON calls(ts); +CREATE TABLE IF NOT EXISTS marks( + key TEXT, main_key TEXT, is_sub INT, cwd TEXT, model TEXT, ts REAL, + day TEXT, prompt_idx INT, kind TEXT, name TEXT, detail TEXT, count INT, + tokens INT, extra INT, duration_ms INT, retry_usd REAL, create_usd REAL, + carry_usd REAL, carry_calls INT); +CREATE INDEX IF NOT EXISTS marks_key ON marks(key); +CREATE INDEX IF NOT EXISTS marks_ts ON marks(ts); +CREATE TABLE IF NOT EXISTS turn_detail( + key TEXT, main_key TEXT, is_sub INT, cwd TEXT, model TEXT, ts REAL, + day TEXT, prompt_idx INT, latency_s REAL, cw_1h INT, cw_5m INT, + out_text INT, out_tool INT, out_think INT, tool_calls INT); +CREATE INDEX IF NOT EXISTS turn_detail_key ON turn_detail(key); +CREATE INDEX IF NOT EXISTS turn_detail_ts ON turn_detail(ts); +CREATE TABLE IF NOT EXISTS session_commits( + sha TEXT PRIMARY KEY, repo TEXT, ts REAL, key TEXT, prompt_idx INT); +""" + +_DERIVED = """ +CREATE TABLE IF NOT EXISTS session_summary( + key TEXT PRIMARY KEY, cwd TEXT, title TEXT, first_ts REAL, last_ts REAL, + wall_s REAL, active_s REAL, usd REAL, usd_main REAL, usd_sub REAL, + calls INT, prompts INT, subagents INT, models TEXT, final_tag TEXT); +CREATE TABLE IF NOT EXISTS prompt_summary( + key TEXT, idx INT, ts REAL, text TEXT, usd_own REAL, usd_sub REAL, + active_s REAL, wall_s REAL, calls INT); +CREATE INDEX IF NOT EXISTS prompt_summary_key ON prompt_summary(key); +CREATE TABLE IF NOT EXISTS subagent_link( + sub_key TEXT PRIMARY KEY, main_key TEXT, prompt_idx INT, depth INT, + description TEXT, agent_type TEXT, model TEXT, usd REAL, calls INT, + first_ts REAL, last_ts REAL, wall_s REAL, active_s REAL); +CREATE INDEX IF NOT EXISTS subagent_link_main ON subagent_link(main_key); +CREATE TABLE IF NOT EXISTS commit_verdicts( + sha TEXT PRIMARY KEY, repo TEXT, ts REAL, added INT, kept INT, + verdict TEXT); +CREATE VIEW IF NOT EXISTS daily AS + SELECT day, SUM(usd) AS usd, COUNT(*) AS calls, + SUM(CASE WHEN is_sub THEN usd ELSE 0 END) AS usd_sub, + SUM(CASE WHEN model LIKE '%opus%' THEN usd ELSE 0 END) AS usd_opus, + SUM(CASE WHEN context > 300000 THEN usd ELSE 0 END) AS usd_over + FROM turns GROUP BY day; +""" + +_AUX = """ +CREATE TABLE IF NOT EXISTS snapshots( + name TEXT PRIMARY KEY, generated TEXT, since REAL, until REAL, + metrics TEXT); +CREATE TABLE IF NOT EXISTS levers( + source TEXT, date TEXT, lever TEXT, metric TEXT, baseline REAL, + predicted REAL, actual REAL, lower_is_better INT, status TEXT, + outcome TEXT, detail TEXT); +CREATE TABLE IF NOT EXISTS aggregate_sessions( + session_id TEXT PRIMARY KEY, source TEXT, day TEXT, started REAL, + cwd TEXT, title TEXT, input INT, cache_write INT, cache_read INT, + output INT, api_turns INT, records INT, usd_est REAL); +CREATE TABLE IF NOT EXISTS prompt_activity(day TEXT PRIMARY KEY, prompts INT); +CREATE TABLE IF NOT EXISTS events(source TEXT, ts REAL, kind TEXT, detail TEXT); +CREATE UNIQUE INDEX IF NOT EXISTS events_once ON events(source, ts, kind, detail); +CREATE TABLE IF NOT EXISTS context_files( + path TEXT PRIMARY KEY, kind TEXT, standing TEXT, size_limit INT); +CREATE TABLE IF NOT EXISTS context_sizes( + path TEXT, ts REAL, bytes INT, source TEXT, PRIMARY KEY (path, ts, source)); +CREATE TABLE IF NOT EXISTS estimates(session_id TEXT PRIMARY KEY, est_calls INT); +""" + + +# Ordered, additive migrations: MIGRATIONS[n] takes a store from n-1 to n. +# The store must never be deleted to change its schema -- rows for expired +# transcripts cannot be regenerated -- so every change lands here instead. +# New tables need no entry: CREATE TABLE IF NOT EXISTS adds them. +MIGRATIONS: dict[int, tuple[str, ...]] = { + 2: ("ALTER TABLE commits ADD COLUMN end_ts REAL",), + 3: ( + "ALTER TABLE agent_calls ADD COLUMN result_ts REAL", + "ALTER TABLE agent_calls ADD COLUMN status TEXT", + ), +} + + +class SchemaError(RuntimeError): + """The store was written by a newer (or unknown) schema.""" + + def __init__(self, found: object) -> None: + """Name the version found and the one this code knows.""" + super().__init__(f"store schema {found!r}; this code knows {VERSION}") + + +def _stored_version(conn: sqlite3.Connection) -> int | None: + """The schema the store was written with, ``None`` for a fresh file.""" + has_meta = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='meta'" + ).fetchone() + if has_meta is None: + return None + row = conn.execute("SELECT value FROM meta WHERE key='schema'").fetchone() + if row is None or not str(row[0]).isdigit(): + raise SchemaError(row) + return int(row[0]) + + +def _migrate(conn: sqlite3.Connection, stored: int) -> None: + """Apply every migration newer than ``stored``, in order.""" + current = int(VERSION) + if stored > current: + raise SchemaError(stored) + for step in range(stored + 1, current + 1): + for statement in MIGRATIONS.get(step, ()): + table, column = statement.split()[2], statement.split()[5] + present = {r[1] for r in conn.execute(f"PRAGMA table_info({table})")} + if column not in present: + conn.execute(statement) + + +def connect(path: Path, *, readonly: bool = False) -> sqlite3.Connection: + """Open the store; create missing tables and migrate when writing.""" + if readonly: + return sqlite3.connect(f"file:{path}?mode=ro", uri=True) + path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(path) + try: + stored = _stored_version(conn) + # Missing tables first, at today's shape; then columns old tables lack. + conn.executescript(_RAW + _DERIVED + _AUX) + if stored is not None: + _migrate(conn, stored) + except SchemaError: + conn.close() + raise + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("INSERT OR REPLACE INTO meta VALUES ('schema', ?)", (VERSION,)) + conn.commit() + return conn diff --git a/python_pkg/token_dashboard/store.py b/python_pkg/token_dashboard/store.py new file mode 100644 index 00000000..a2961d06 --- /dev/null +++ b/python_pkg/token_dashboard/store.py @@ -0,0 +1,135 @@ +"""Write one file's :class:`FileFacts` into the raw tables, replacing old rows. + +The unit of change is the whole file: a changed transcript is re-walked and +its rows replaced in one transaction. Resuming from a byte offset would need +the dedup state (:class:`TurnMerger`) persisted across runs, and re-walking +even the largest transcript costs a fraction of a second. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from python_pkg.token_dashboard import detail_store +from python_pkg.token_dashboard.detail_store import day_of + +if TYPE_CHECKING: + import sqlite3 + + from python_pkg.token_dashboard.walk import FileFacts + +# One literal statement per raw table: no SQL is assembled from strings. +CLEAR = ( + "DELETE FROM turns WHERE key = ?", + "DELETE FROM prompts WHERE key = ?", + "DELETE FROM tool_use WHERE key = ?", + "DELETE FROM agent_calls WHERE key = ?", + "DELETE FROM activity WHERE key = ?", + "DELETE FROM lever_tables WHERE key = ?", + "DELETE FROM commits WHERE key = ?", + "DELETE FROM sent_messages WHERE key = ?", + *detail_store.CLEAR, +) + + +def write( + conn: sqlite3.Connection, facts: FileFacts, path: str, stat: tuple[int, float] +) -> None: + """Replace every row ``facts.key`` owns. Caller commits.""" + key = facts.key + for statement in CLEAR: + conn.execute(statement, (key,)) + main_key = facts.parent_session or key + meta = facts.meta + conn.execute( + "INSERT OR REPLACE INTO sessions VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + ( + key, + facts.session_id, + main_key, + facts.cwd, + meta.get("agentType"), + meta.get("description"), + meta.get("toolUseId"), + meta.get("parentAgentId"), + meta.get("model"), + *facts.span, + facts.final_tag, + ), + ) + is_sub = int(facts.parent_session is not None) + conn.executemany( + "INSERT INTO turns VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + [ + ( + key, + main_key, + is_sub, + facts.cwd, + t.ts, + day_of(t.ts), + t.model, + t.usd, + t.context, + t.prompt_idx, + t.usage.get("input_tokens", 0), + t.usage.get("cache_creation_input_tokens", 0), + t.usage.get("cache_read_input_tokens", 0), + t.usage.get("output_tokens", 0), + ) + for t in facts.rows.turns + ], + ) + conn.executemany( + "INSERT INTO prompts VALUES (?,?,?,?,?)", + [(key, p.idx, p.ts, p.prompt_id, p.text) for p in facts.rows.prompts], + ) + conn.executemany( + "INSERT INTO tool_use VALUES (?,?,?,?,?,?,?)", + [ + (key, t.prompt_idx, t.name, t.calls, t.result_tokens, t.usd, t.carry_usd) + for t in facts.rows.tools + ], + ) + conn.executemany( + "INSERT INTO agent_calls (key, tool_use_id, agent_id, prompt_idx, ts," + " description, subagent_type, model, result_ts, status)" + " VALUES (?,?,?,?,?,?,?,?,?,?)", + [ + ( + key, + a.tool_use_id, + a.agent_id, + a.prompt_idx, + a.ts, + a.description, + a.subagent_type, + a.model, + a.result_ts, + a.status, + ) + for a in facts.rows.agent_calls + ], + ) + conn.executemany( + "INSERT INTO activity VALUES (?,?,?)", + [(key, s, e) for s, e in facts.rows.intervals], + ) + conn.executemany( + "INSERT INTO lever_tables VALUES (?,?,?)", + [(key, ts, md) for ts, md in facts.rows.lever_tables], + ) + conn.executemany( + "INSERT INTO commits (key, prompt_idx, ts, end_ts, sha, repo)" + " VALUES (?,?,?,?,?,?)", + [ + (key, c.prompt_idx, c.ts, c.end_ts, c.sha, c.repo) + for c in facts.rows.commits + ], + ) + conn.executemany( + "INSERT INTO sent_messages VALUES (?,?,?)", + [(key, ts, to) for ts, to in facts.rows.messages], + ) + detail_store.write(conn, facts, (key, main_key, is_sub, facts.cwd)) + conn.execute("INSERT OR REPLACE INTO files VALUES (?,?,?,?)", (path, key, *stat)) diff --git a/python_pkg/token_dashboard/survival.py b/python_pkg/token_dashboard/survival.py new file mode 100644 index 00000000..e8da8d0c --- /dev/null +++ b/python_pkg/token_dashboard/survival.py @@ -0,0 +1,126 @@ +"""Did a commit made in a session survive? Decided by git, never by a model. + +Rule (fixed here, shown in the dashboard caption): a commit is judged once it +is :data:`WINDOW_DAYS` old, at the last commit of its repository's ``HEAD`` +history that is no more than that old relative to it. + +* ``reverted`` -- a commit inside the window says ``This reverts commit ``. +* ``churned`` -- under :data:`KEEP_SHARE` of the lines it added (counted over + its :data:`MAX_FILES` largest files) still blame to it. +* ``survived`` -- otherwise, including a commit that added no lines. +* ``pending`` -- younger than the window. +* ``unknown`` -- the repo or hash is gone, or the commit is not on ``HEAD`` + (rebased, squashed or on another branch): excluded from the ratio. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +from python_pkg.token_dashboard import gitcmd + +if TYPE_CHECKING: + from collections.abc import Callable + import sqlite3 + +WINDOW_DAYS = 14 +WINDOW_S = WINDOW_DAYS * 86400 +KEEP_SHARE = 0.5 +MAX_FILES = 10 +_NUMSTAT_FIELDS = 3 +FINAL = frozenset({"survived", "churned", "reverted"}) + + +@dataclass(frozen=True) +class Verdict: + """The outcome for one commit.""" + + verdict: str + sha: str = "" + ts: float = 0.0 + added: int = 0 + kept: int = 0 + + +def judge(added: int, kept: int, *, reverted: bool) -> str: + """Apply the survival rule to measured numbers.""" + if reverted: + return "reverted" + if added and kept < KEEP_SHARE * added: + return "churned" + return "survived" + + +def _git(repo: str, *args: str) -> str | None: + """Run git in ``repo``; ``None`` on any failure.""" + return gitcmd.run(repo, *args) + + +def _added(repo: str, sha: str) -> list[tuple[int, str]]: + """``(added lines, path)`` for the commit's largest text files.""" + out = _git(repo, "show", "--numstat", "--format=", sha) or "" + rows = [] + for line in out.splitlines(): + parts = line.split("\t", 2) + if len(parts) == _NUMSTAT_FIELDS and parts[0].isdigit() and int(parts[0]): + rows.append((int(parts[0]), parts[2])) + return sorted(rows, reverse=True)[:MAX_FILES] + + +def _kept(repo: str, rev: str, sha: str, path: str) -> int: + """Lines of ``path`` at ``rev`` that still blame to ``sha``.""" + out = _git(repo, "blame", "--porcelain", rev, "--", path) or "" + prefix = sha + " " + return sum(1 for line in out.splitlines() if line.startswith(prefix)) + + +def evaluate(repo: str, sha: str, now: float) -> Verdict: + """Judge one commit in one repository.""" + if not Path(repo).is_dir(): + return Verdict("unknown") + full = ( + _git(repo, "rev-parse", "--verify", "-q", f"{sha}^{{commit}}") or "" + ).strip() + if not full: + return Verdict("unknown") + ts = float((_git(repo, "show", "-s", "--format=%ct", full) or "0").strip() or 0) + if now - ts < WINDOW_S: + return Verdict("pending", full, ts) + if _git(repo, "merge-base", "--is-ancestor", full, "HEAD") is None: + return Verdict("unknown", full, ts) + until = f"@{int(ts + WINDOW_S)}" + end = (_git(repo, "rev-list", "-1", f"--before={until}", "HEAD") or "").strip() + log = _git(repo, "log", "--format=%B", f"--since=@{int(ts)}", f"--until={until}") + reverted = f"This reverts commit {full}" in (log or "") + files = _added(repo, full) + added = sum(n for n, _ in files) + kept = sum(_kept(repo, end or full, full, path) for _, path in files) + return Verdict(judge(added, kept, reverted=reverted), full, ts, added, kept) + + +def update( + conn: sqlite3.Connection, now: float, deadline: float, clock: Callable[[], float] +) -> bool: + """Judge every session commit without a final verdict, within the deadline. + + Returns whether the backlog was cleared. A verdict once final is never + recomputed, so the backlog is only commits that turned 14 days old. + """ + rows = conn.execute( + "SELECT c.sha, c.repo, c.ts FROM session_commits c LEFT JOIN" + " commit_verdicts v ON v.sha = c.sha WHERE v.verdict IS NULL" + " OR v.verdict = 'pending'" + ).fetchall() + for sha, repo, ts in rows: + if now - ts < WINDOW_S: + continue # cannot be final yet; no git call needed + if clock() > deadline: + return False + found = evaluate(repo, sha, now) + conn.execute( + "INSERT OR REPLACE INTO commit_verdicts VALUES (?,?,?,?,?,?)", + (sha, repo, found.ts or ts, found.added, found.kept, found.verdict), + ) + return True diff --git a/python_pkg/token_dashboard/tests/__init__.py b/python_pkg/token_dashboard/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python_pkg/token_dashboard/tests/conftest.py b/python_pkg/token_dashboard/tests/conftest.py new file mode 100644 index 00000000..398e6232 --- /dev/null +++ b/python_pkg/token_dashboard/tests/conftest.py @@ -0,0 +1,123 @@ +"""Shared dashboard test setup: source isolation and transcript builders. + +The autouse fixture keeps every test away from the real ``~/.claude`` state; +the builders below make synthetic transcript records. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +import json +from typing import TYPE_CHECKING, Any + +import pytest + +from python_pkg.token_dashboard import ab, aux, context_files, walk + +if TYPE_CHECKING: + from pathlib import Path + + from python_pkg.token_dashboard.detail_rows import DetailRows + +BASE = 1_791_600_000.0 # 2026-10-09, a fixed anchor so day buckets are stable + + +@pytest.fixture(autouse=True) +def _isolated_sources(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Point every aux source at an empty tmp directory.""" + claude = tmp_path / "claude-home" + monkeypatch.setattr(aux, "AUTOPSY", claude / "autopsy.jsonl") + monkeypatch.setattr(aux, "PROMPT_HISTORY", claude / "history.jsonl") + monkeypatch.setattr(aux, "TIME_CAP", claude / "time-cap.jsonl") + monkeypatch.setattr(ab, "LOG", claude / "agent-ab.jsonl") + monkeypatch.setattr(context_files, "CLAUDE_DIR", claude) + monkeypatch.setattr(context_files, "SRC_DIR", claude / "src") + + +def iso(offset: float) -> str: + """ISO timestamp ``offset`` seconds after :data:`BASE`.""" + return datetime.fromtimestamp(BASE + offset, tz=UTC).isoformat() + + +def prompt(text: str, at: float, **extra: object) -> dict[str, Any]: + """A typed user prompt.""" + return { + "type": "user", + "timestamp": iso(at), + "cwd": "/repo", + "message": {"role": "user", "content": text}, + **extra, + } + + +def assistant( + mid: str, + at: float, + *, + output: int = 1000, + model: str = "claude-opus-5-5", + blocks: list[dict[str, Any]] | None = None, + context: int = 0, + **usage: object, +) -> dict[str, Any]: + """One assistant record; ``context`` becomes cache-read tokens. + + ``usage`` adds raw usage fields, e.g. a cache-write split. + """ + return { + "type": "assistant", + "timestamp": iso(at), + "message": { + "id": mid, + "model": model, + "usage": { + "output_tokens": output, + "cache_read_input_tokens": context, + **usage, + }, + "content": blocks or [{"type": "text", "text": "ok"}], + }, + } + + +def use(tid: str, name: str, **tool_input: object) -> dict[str, Any]: + """A ``tool_use`` content block.""" + return {"type": "tool_use", "id": tid, "name": name, "input": tool_input} + + +def result( + tid: str, at: float, text: str = "x" * 40, **extra: object +) -> dict[str, Any]: + """A user record carrying one ``tool_result``.""" + return { + "type": "user", + "timestamp": iso(at), + "message": { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": tid, "content": text}], + }, + **extra, + } + + +def write(path: Path, records: list[dict[str, Any]]) -> Path: + """Write records as JSONL, creating parent directories.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(json.dumps(r) for r in records) + "\n", encoding="utf-8") + return path + + +def usd_opus_output(tokens: int) -> float: + """List price of ``tokens`` Opus 5.5 output tokens (USD 20 / MTok).""" + return tokens * 20 / 1_000_000 + + +def attach(at: float, **attachment: object) -> dict[str, Any]: + """An ``attachment`` record.""" + return {"type": "attachment", "timestamp": iso(at), "attachment": attachment} + + +def walk_detail(tmp_path: Path, records: list[dict[str, Any]]) -> DetailRows: + """Walk one main-session file and return its detail rows.""" + path = write(tmp_path / "proj" / "s.jsonl", records) + return walk.walk_file(path, tmp_path).rows.detail diff --git a/python_pkg/token_dashboard/tests/test_ab.py b/python_pkg/token_dashboard/tests/test_ab.py new file mode 100644 index 00000000..3b9988f2 --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_ab.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest + +from python_pkg.token_dashboard import ab, ingest, schema +from python_pkg.token_dashboard.tests.conftest import ( + assistant, + prompt, + result, + use, + write, +) + +if TYPE_CHECKING: + from pathlib import Path + + +def test_wilson_matches_reference_values() -> None: + # Reference: Wilson score interval, z = 1.96 (Newcombe 1998, table I). + low, high = ab.wilson(81, 263) + assert (round(low, 4), round(high, 4)) == (0.2553, 0.3662) + assert ab.wilson(0, 10)[0] == 0.0 + assert ab.wilson(0, 0) == (0.0, 1.0) + + +def test_newcombe_difference_contains_zero_for_equal_rates() -> None: + diff, low, high = ab.diff_ci(5, 50, 5, 50) + assert diff == 0.0 + assert low < 0 < high + + +def test_jaccard_and_correction_phrases() -> None: + assert ab.jaccard("Find RTDB rules for tutor", "find rtdb rules tutor log") >= 0.6 + assert ab.jaccard("Find RTDB rules", "Write the README") == 0.0 + assert ab.is_correction("No, that's wrong") + assert ab.is_correction("you missed the second file") + assert not ab.is_correction("nothing else, thanks") + + +def _arm(n: int, redo: int, corr: int = 0) -> ab.ArmCounts: + return ab.ArmCounts(n=n, redo=redo, correction=corr) + + +@pytest.mark.parametrize( + ("sonnet", "opus", "level"), + [ + (_arm(10, 9), _arm(10, 0), "grey"), + (_arm(40, 20), _arm(40, 2), "red"), + (_arm(40, 2, 20), _arm(40, 2, 1), "red"), + (_arm(40, 6), _arm(40, 4), "green"), + (_arm(40, 0), _arm(40, 20), "green"), # Sonnet better is not a drop + ], +) +def test_banner(sonnet: ab.ArmCounts, opus: ab.ArmCounts, level: str) -> None: + found = ab.banner(sonnet, opus) + assert found["level"] == level + assert f"n={min(sonnet.n, opus.n)}" in found["text"] or level == "red" + + +def test_rebuild_joins_assignments_to_transcripts(tmp_path: Path) -> None: + root = tmp_path / "projects" + first = use("t1", "Agent", description="Review the parser change") + again = use("t2", "Agent", description="Review the parser change again") + msg = use("t3", "SendMessage", to="abc", message="more") + write( + root / "p" / "s1.jsonl", + [ + prompt("go", 0), + assistant("m1", 1, blocks=[first]), + result("t1", 50, "done", toolUseResult={"agentId": "abc"}), + assistant("m2", 60, blocks=[msg]), + result("t3", 61), + prompt("no, that's wrong", 100), + assistant("m3", 101, blocks=[again]), + result( + "t2", 150, "err", toolUseResult={"agentId": "def", "status": "error"} + ), + ], + ) + log = tmp_path / "agent-ab.jsonl" + rows = [ + {"tool_use_id": "t1", "arm": "sonnet", "class": "review", "session_id": "s1"}, + {"tool_use_id": "t2", "arm": "opus", "class": "review", "session_id": "s1"}, + ] + log.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8") + ingest.run(root, tmp_path / "t.sqlite", tmp_path / "report", 30.0) + conn = schema.connect(tmp_path / "t.sqlite") + payload = ab.rebuild(conn, log) + stored = conn.execute( + "SELECT tool_use_id, arm, key, main_key, is_sub, cwd, redo, followup," + " error, correction, tag FROM ab_rows ORDER BY tool_use_id" + ).fetchall() + payload = ab.rebuild(conn, log) # a rebuild replaces, never appends + count = conn.execute("SELECT COUNT(*) FROM ab_rows").fetchone()[0] + conn.close() + assert stored == [ + ("t1", "sonnet", "s1", "s1", 0, "/repo", 1, 1, 0, 1, "untagged"), + ("t2", "opus", "s1", "s1", 0, "/repo", 0, 0, 1, 0, "untagged"), + ] + assert count == 2 + sonnet, opus = payload["arms"]["sonnet"], payload["arms"]["opus"] + assert (sonnet["n"], opus["n"]) == (1, 1) + assert sonnet["redo"]["k"] == 1 + assert sonnet["followup"]["k"] == 1 + assert sonnet["correction"]["k"] == 1 + assert opus["error"]["k"] == 1 + assert payload["banner"]["level"] == "grey" diff --git a/python_pkg/token_dashboard/tests/test_active.py b/python_pkg/token_dashboard/tests/test_active.py new file mode 100644 index 00000000..ace8f013 --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_active.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from python_pkg.token_dashboard import active + + +def test_gap_over_limit_contributes_zero_not_the_limit() -> None: + spans = active.intervals([0, 60, 120, 1000, 1030]) + assert spans == [(0, 120), (1000, 1030)] + assert active.measure(spans) == 150 + + +def test_union_bridges_short_gaps_between_files() -> None: + main = [(0.0, 100.0)] + sub = [(250.0, 400.0)] # 150 s after main went quiet: still active + far = [(2000.0, 2010.0)] + assert active.measure(main + sub + far) == 410 + + +def test_overlapping_intervals_count_once() -> None: + assert active.measure([(0.0, 100.0), (50.0, 150.0)]) == 150 + + +def test_clip_to_a_prompt_window() -> None: + spans = [(0.0, 100.0), (1000.0, 1100.0)] + assert active.measure(spans, 50.0, 1050.0) == 100 + assert active.measure([], 0.0, 10.0) == 0 + + +def test_empty_and_single_stamp() -> None: + assert active.intervals([]) == [] + assert active.intervals([5.0]) == [(5.0, 5.0)] diff --git a/python_pkg/token_dashboard/tests/test_aux.py b/python_pkg/token_dashboard/tests/test_aux.py new file mode 100644 index 00000000..0010d7be --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_aux.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +from datetime import UTC, datetime +import json +from typing import TYPE_CHECKING + +import pytest + +from python_pkg.token_dashboard import aux, schema +from python_pkg.token_dashboard.tests.test_levers import TABLE + +if TYPE_CHECKING: + from collections.abc import Iterator, Sequence + from pathlib import Path + import sqlite3 + +NOON = datetime(2026, 10, 9, 12, tzinfo=UTC) +SECONDS = NOON.timestamp() + + +def _lines(path: Path, rows: Sequence[object], *junk: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + body = [json.dumps(r) for r in rows] + list(junk) + path.write_text("\n".join(body) + "\n", encoding="utf-8") + + +@pytest.fixture +def conn(tmp_path: Path) -> Iterator[sqlite3.Connection]: + opened = schema.connect(tmp_path / "t.sqlite") + yield opened + opened.close() + + +def test_jsonl_skips_blank_malformed_and_non_object_lines(tmp_path: Path) -> None: + path = tmp_path / "x.jsonl" + _lines(path, [{"a": 1}, [1, 2], 3], "", "{broken") + assert list(aux._jsonl(path)) == [{"a": 1}] + assert list(aux._jsonl(tmp_path / "missing.jsonl")) == [] + + +def test_stamp_reads_iso_seconds_milliseconds_and_rejects_the_rest() -> None: + assert aux._stamp(NOON.isoformat()) == SECONDS + assert aux._stamp(SECONDS) == SECONDS + assert aux._stamp(int(SECONDS * 1000)) == pytest.approx(int(SECONDS)) + assert aux._stamp("yesterday") is None + assert aux._stamp(["x"]) is None + assert aux._stamp(None) is None + + +def test_snapshots_copy_window_and_metrics_and_skip_unreadable( + tmp_path: Path, conn: sqlite3.Connection +) -> None: + history = tmp_path / "report" / "history" + history.mkdir(parents=True) + snap = { + "generated": NOON.isoformat(), + "window": {"since": 1.0, "until": 2.0}, + "metrics": {"usd_total": 5}, + } + (history / "2026-10-09.json").write_text(json.dumps(snap)) + (history / "2026-10-10.json").write_text("{not json") + aux.snapshots(conn, tmp_path / "report") + rows = conn.execute("SELECT name, since, until, metrics FROM snapshots").fetchall() + assert rows == [("2026-10-09", 1.0, 2.0, '{"usd_total": 5}')] + + +def test_lever_rows_assign_each_table_to_the_nearest_snapshot( + tmp_path: Path, conn: sqlite3.Connection +) -> None: + report = tmp_path / "report" + (report / "history").mkdir(parents=True) + for day in ("2026-10-02", "2026-10-09"): + stamp = datetime.fromisoformat(f"{day}T12:00:00+00:00").isoformat() + (report / "history" / f"{day}.json").write_text( + json.dumps({"generated": stamp}) + ) + (report / "WEEKLY.md").write_text(TABLE, encoding="utf-8") + aux.snapshots(conn, report) + conn.execute("INSERT INTO lever_tables VALUES ('s', ?, ?)", (SECONDS, TABLE)) + aux.lever_rows(conn, report) + sources = {r[0] for r in conn.execute("SELECT source FROM levers")} + assert "snapshot 2026-10-09" in sources + + +def test_lever_rows_with_no_snapshots_store_nothing( + tmp_path: Path, conn: sqlite3.Connection +) -> None: + conn.execute("INSERT INTO lever_tables VALUES ('s', ?, ?)", (SECONDS, TABLE)) + aux.lever_rows(conn, tmp_path / "report") + assert conn.execute("SELECT COUNT(*) FROM levers").fetchone() == (0,) + + +def _autopsy_row( + sid: str, started: object, msgs: int, api: int | None +) -> dict[str, object]: + return { + "session_id": sid, + "meta": {"started_at": started, "cwd": "/r", "title": sid}, + "tokens": {"input": 100, "cache_write": 200, "cache_read": 300, "output": 400}, + "turns": {"api_turns": api}, + "counts": {"assistant_msgs": msgs}, + } + + +def test_autopsy_scales_by_api_turns_and_calibrates_against_transcripts( + conn: sqlite3.Connection, +) -> None: + rows = [ + _autopsy_row("both", NOON.isoformat(), 10, 5), + _autopsy_row("alone", SECONDS, 4, None), + _autopsy_row("nostart", None, 4, 2), + _autopsy_row("nomsgs", SECONDS, 0, 0), + ] + _lines(aux.AUTOPSY, rows) + conn.execute("INSERT INTO session_summary(key, usd_main) VALUES ('both', 3.0)") + aux.autopsy(conn) + got = dict(conn.execute("SELECT session_id, usd_est FROM aggregate_sessions")) + assert set(got) == {"both", "alone"} + # Same tokens: 5 API turns over 10 records scales the cost by one half. + assert got["both"] == pytest.approx(got["alone"] / 2) + factor = conn.execute("SELECT value FROM meta WHERE key='autopsy_calibration'") + assert float(factor.fetchone()[0]) == pytest.approx(3.0 / got["both"]) + + +def test_autopsy_without_overlap_keeps_factor_one(conn: sqlite3.Connection) -> None: + aux.autopsy(conn) + factor = conn.execute("SELECT value FROM meta WHERE key='autopsy_calibration'") + assert factor.fetchone() == ("1.0",) + + +def test_activity_logs_count_prompts_events_and_first_estimate( + tmp_path: Path, conn: sqlite3.Connection +) -> None: + report = tmp_path / "report" + _lines( + aux.PROMPT_HISTORY, + [{"timestamp": int(SECONDS * 1000)}, {"timestamp": SECONDS}, {"x": 1}], + ) + _lines(aux.TIME_CAP, [{"ts": SECONDS, "event": "kill", "command": "sleep 99"}]) + misses = report / "task-size" / "misses.jsonl" + _lines(misses, [{"ts": NOON.isoformat(), "kind": "under", "first_prompt": "p"}]) + verdicts = [ + {"status": "ok", "session": "a", "verdict": {"est_calls": 12}}, + {"status": "ok", "session": "a", "verdict": {"est_calls": 99}}, + {"status": "ok", "session": "b", "verdict": "none"}, + {"status": "error", "session": "c", "verdict": {"est_calls": 3}}, + ] + _lines(report / "task-size" / "verdicts.jsonl", verdicts) + aux.activity_logs(conn, report) + days = conn.execute("SELECT prompts FROM prompt_activity").fetchall() + events = conn.execute("SELECT source, kind, detail FROM events").fetchall() + estimates = conn.execute("SELECT session_id, est_calls FROM estimates").fetchall() + assert days == [(2,)] + assert events == [ + ("time-cap", "kill", "sleep 99"), + ("task-size-miss", "under", "p"), + ] + assert estimates == [("a", 12)] + + +def test_reloading_keeps_rows_a_trimmed_source_no_longer_has(tmp_path: Path) -> None: + conn = schema.connect(tmp_path / "t.sqlite") + _lines( + aux.PROMPT_HISTORY, + [{"timestamp": 1_700_000_000_000}, {"timestamp": 1_800_000_000}], + ) + aux.activity_logs(conn, tmp_path / "report") + aux.PROMPT_HISTORY.write_text("", encoding="utf-8") # rotated away + aux.activity_logs(conn, tmp_path / "report") + aux.activity_logs(conn, tmp_path / "report") + days = conn.execute("SELECT COUNT(*), SUM(prompts) FROM prompt_activity").fetchone() + conn.close() + assert days == (2, 2) diff --git a/python_pkg/token_dashboard/tests/test_check_window.py b/python_pkg/token_dashboard/tests/test_check_window.py new file mode 100644 index 00000000..29b8c17e --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_check_window.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest + +from python_pkg.token_dashboard import check_window, schema + +if TYPE_CHECKING: + from collections.abc import Mapping + from pathlib import Path + +DAY = 86400.0 + + +def _store(tmp_path: Path, stamps: list[float]) -> Path: + db = tmp_path / "t.sqlite" + conn = schema.connect(db) + conn.executemany( + "INSERT INTO turns(key, ts, usd) VALUES ('k', ?, 1.0)", [(s,) for s in stamps] + ) + conn.commit() + conn.close() + return db + + +def _snapshot(report: Path, name: str, metrics: Mapping[str, float]) -> None: + folder = report / "history" + folder.mkdir(parents=True, exist_ok=True) + snap = {"window": {"since": 0, "until": 10 * DAY}, "metrics": metrics} + (folder / f"{name}.json").write_text(json.dumps(snap), encoding="utf-8") + + +def _run(tmp_path: Path, db: Path, *extra: str) -> int: + return check_window.main( + ["--db", str(db), "--report-dir", str(tmp_path / "report"), *extra] + ) + + +def test_matching_snapshot_exits_zero( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + db = _store(tmp_path, [100.0, 100.0 + DAY]) + good = {"usd_total": 2.0, "usd_per_day": 2.0, "calls": 2} + _snapshot(tmp_path / "report", "2026-10-16", {**good, "calls": 99}) + _snapshot(tmp_path / "report", "2026-10-17", good) + assert _run(tmp_path, db, "--snapshot", "2026-10-17") == 0 + out = capsys.readouterr().out + assert out.count("OK") == 3 + assert "FAIL" not in out + + +def test_drift_beyond_one_percent_in_the_newest_snapshot_fails( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + db = _store(tmp_path, [100.0, 100.0 + DAY]) + _snapshot(tmp_path / "report", "2026-10-16", {"usd_total": 2.0}) + bad = {"usd_total": 2.5, "usd_per_day": 2.0, "calls": 2} + _snapshot(tmp_path / "report", "2026-10-17", bad) + assert _run(tmp_path, db) == 1 + assert "usd_total: store 2.00 vs snapshot 2.50 (20.00%) FAIL" in ( + capsys.readouterr().out + ) + + +def test_empty_window_against_zero_metrics_is_a_match(tmp_path: Path) -> None: + db = _store(tmp_path, []) + zero = {"usd_total": 0, "usd_per_day": 0, "calls": 0} + _snapshot(tmp_path / "report", "2026-10-17", zero) + assert _run(tmp_path, db) == 0 + + +def test_missing_snapshot_or_store_exits_one( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + db = _store(tmp_path, [1.0]) + assert _run(tmp_path, db) == 1 # no snapshot directory at all + _snapshot(tmp_path / "report", "2026-10-17", {"usd_total": 1}) + assert _run(tmp_path, db, "--snapshot", "1999-01-01") == 1 + assert _run(tmp_path, tmp_path / "gone.sqlite") == 1 + assert capsys.readouterr().err.count("missing snapshot or store") == 3 diff --git a/python_pkg/token_dashboard/tests/test_classify.py b/python_pkg/token_dashboard/tests/test_classify.py new file mode 100644 index 00000000..4030316d --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_classify.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import pytest + +from python_pkg.token_dashboard import classify + + +@pytest.mark.parametrize( + ("command", "family"), + [ + ("ls -la", "ls"), + ("git -C /repo status --short | head", "git status"), + ("cd /x && git log --oneline -3", "git log"), + ("~/.claude/scripts/capped.sh python3 -m pytest -q", "python3 -m pytest"), + ( + "CAP_MEM=4G CAP_CPU_PCT=20 ~/.claude/scripts/capped.sh pnpm -s lint", + "pnpm lint", + ), + ("timeout 40 nice -n 19 grep -r x .", "grep"), + ("python3 scripts/check.py --all", "python3 check.py"), + ("bash ~/src/utils/scripts/gate_stamp.sh step x", "bash gate_stamp.sh"), + ("/usr/bin/sqlite3 db 'SELECT 1'", "sqlite3"), + ("cd /x", "cd"), + ("python3 -m", "python3 -m"), + ("python3 <<'EOF'\nprint(1)\nEOF", "python3"), + ("bash -c 'x' > /dev/null", "bash -c"), + ("git", "git"), + ("FOO=1", "(empty)"), + ("", "(empty)"), + ], +) +def test_bash_family(command: str, family: str) -> None: + assert classify.bash_family(command) == family + + +@pytest.mark.parametrize( + ("name", "tool_input", "family", "target"), + [ + ("Bash", {"command": "git status\n"}, "git status", "git status"), + ("Bash", {}, "(empty)", ""), + ( + "mcp__claude-in-chrome__navigate", + {"url": "x"}, + "mcp:claude-in-chrome", + "navigate", + ), + ("Skill", {"skill": "dataviz"}, "skill", "dataviz"), + ( + "Read", + {"file_path": "/h/.claude/rules/a.md"}, + "Read:rules", + "/h/.claude/rules/a.md", + ), + ( + "Read", + {"file_path": "/h/.claude/projects/p/memory/m.md"}, + "Read:memory", + "/h/.claude/projects/p/memory/m.md", + ), + ("Read", {"file_path": "/r/CLAUDE.md"}, "Read:claude-md", "/r/CLAUDE.md"), + ( + "Edit", + {"file_path": "/h/.claude/skills/x/SKILL.md"}, + "Edit:skill", + "/h/.claude/skills/x/SKILL.md", + ), + ( + "Write", + {"file_path": "/h/.claude/settings.json"}, + "Write:claude", + "/h/.claude/settings.json", + ), + ("Read", {"file_path": "/r/shot.PNG"}, "Read:image", "/r/shot.PNG"), + ("Read", {"file_path": "/r/a.py"}, "Read:file", "/r/a.py"), + ("Read", {}, "Read:file", ""), + ("Grep", {"pattern": "def x", "path": "/r"}, "Grep", "def x"), + ( + "WebFetch", + {"url": "https://docs.example.com/a?b"}, + "WebFetch", + "docs.example.com", + ), + ("WebSearch", {"query": "sqlite window"}, "WebSearch", "sqlite window"), + ("Agent", {"subagent_type": "Explore"}, "Agent", "Explore"), + ("Agent", {"isolation": "worktree"}, "Agent", "general-purpose [worktree]"), + ("TaskStop", {"id": 1}, "TaskStop", ""), + ], +) +def test_family_and_target( + name: str, tool_input: dict[str, object], family: str, target: str +) -> None: + assert classify.family(name, tool_input) == family + assert classify.target(name, tool_input) == target + + +def test_target_is_capped() -> None: + long = {"command": "echo " + "x" * 1000} + assert len(classify.target("Bash", long)) == classify.TARGET_CHARS + + +def test_signature_ignores_key_order_but_not_values() -> None: + one = classify.signature("Bash", {"command": "ls", "timeout": 5}) + same = classify.signature("Bash", {"timeout": 5, "command": "ls"}) + other = classify.signature("Bash", {"command": "ls -a", "timeout": 5}) + assert one == same + assert one != other + assert len(one) == classify.SIG_CHARS + + +@pytest.mark.parametrize( + ("text", "kind"), + [ + ("Exit code 1\nboom", "exit_code"), + ("File has not been read yet", "tool_error"), + ("The user doesn't want to proceed with this tool use.", "rejected"), + ("Permission for this action was denied by the classifier", "denied"), + ("PreToolUse:Bash hook error: blocked by policy", "hook"), + ("TIME CAP: this command was killed at 60s", "time_cap"), + ("[Request interrupted by user for tool use]", "interrupted"), + ("something else", "error"), + ], +) +def test_error_kind(text: str, kind: str) -> None: + assert classify.error_kind(text) == kind diff --git a/python_pkg/token_dashboard/tests/test_commits.py b/python_pkg/token_dashboard/tests/test_commits.py new file mode 100644 index 00000000..5ae5b2e3 --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_commits.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from python_pkg.token_dashboard import commits, schema +from python_pkg.token_dashboard.tests.test_efficiency import MONDAY, _commit, _git + +if TYPE_CHECKING: + from pathlib import Path + +TRAILER = "\n\nCo-Authored-By: Claude Opus 5.5 " + + +def test_git_log_attributes_trailer_commits_to_the_commit_command( + tmp_path: Path, +) -> None: + repo = tmp_path / "r" + repo.mkdir() + _git(repo, "init", "-q", when=MONDAY) + (repo / "a").write_text("1") + _git(repo, "add", "a", when=MONDAY) + _git(repo, "commit", "-qm", "ours" + TRAILER, when=MONDAY + 30) + ours = _git(repo, "rev-parse", "HEAD", when=MONDAY) + _commit(repo, "b", "2", MONDAY + 40) # no trailer: a human commit + _git( + repo, + "commit", + "-q", + "--allow-empty", + "-m", + "late" + TRAILER, + when=MONDAY + 9000, + ) + conn = schema.connect(tmp_path / "t.sqlite") + rows = [ + ("s1", 0, MONDAY, MONDAY + 60, "", str(repo)), + ("s2", 3, MONDAY - 5000, MONDAY - 4000, "", str(repo / "sub")), + ("s3", 1, MONDAY, MONDAY + 1, "", str(tmp_path / "missing")), + ] + conn.executemany("INSERT INTO commits VALUES (?,?,?,?,?,?)", rows) + commits.discover(conn) + found = conn.execute("SELECT sha, key, prompt_idx FROM session_commits").fetchall() + conn.close() + assert found == [(ours, "s1", 0)] + + +def test_a_printed_hash_is_attributed_even_without_trailer(tmp_path: Path) -> None: + repo = tmp_path / "r" + repo.mkdir() + _git(repo, "init", "-q", when=MONDAY) + sha = _commit(repo, "a", "1", MONDAY + 5) + conn = schema.connect(tmp_path / "t.sqlite") + conn.execute( + "INSERT INTO commits VALUES (?,?,?,?,?,?)", + ("s1", 2, MONDAY, MONDAY + 9, sha[:7], str(repo)), + ) + commits.discover(conn) + commits.discover(conn) # idempotent + found = conn.execute("SELECT sha, key FROM session_commits").fetchall() + conn.close() + assert found == [(sha, "s1")] diff --git a/python_pkg/token_dashboard/tests/test_context_files.py b/python_pkg/token_dashboard/tests/test_context_files.py new file mode 100644 index 00000000..81e1ddcc --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_context_files.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import json +import os +import subprocess +from typing import TYPE_CHECKING + +from python_pkg.token_dashboard import context_files, gitcmd, schema + +if TYPE_CHECKING: + from pathlib import Path + + +def _tree(tmp_path: Path) -> tuple[Path, Path]: + claude, src = tmp_path / ".claude", tmp_path / "src" + for rel in ( + "CLAUDE.md", + "rules/a.instructions.md", + "rules/deep/b.md", + "memories/mistakes.md", + "skills/x/SKILL.md", + "agents/reviewer.md", + "projects/-home-kuhy/memory/MEMORY.md", + "backups/rules-ondemand/not-loaded.md", + ): + (claude / rel).parent.mkdir(parents=True, exist_ok=True) + (claude / rel).write_text("# x\n" * 10, encoding="utf-8") + (src / "repo").mkdir(parents=True) + (src / "repo" / "CLAUDE.md").write_text("## Commands\n", encoding="utf-8") + (src / "repo" / "AGENTS.md").write_text("agents\n", encoding="utf-8") + return claude, src + + +def test_enumerates_every_loaded_file_and_nothing_on_demand(tmp_path: Path) -> None: + claude, src = _tree(tmp_path) + found = { + f.path.replace(str(tmp_path), ""): f for f in context_files.find(claude, src) + } + assert set(found) == { + "/.claude/CLAUDE.md", + "/.claude/rules/a.instructions.md", + "/.claude/rules/deep/b.md", + "/.claude/memories/mistakes.md", + "/.claude/skills/x/SKILL.md", + "/.claude/agents/reviewer.md", + "/.claude/projects/-home-kuhy/memory/MEMORY.md", + "/src/repo/CLAUDE.md", + "/src/repo/AGENTS.md", + } + assert found["/.claude/rules/deep/b.md"].standing == "always" + assert found["/src/repo/CLAUDE.md"].standing == "in repo" + assert found["/.claude/skills/x/SKILL.md"].standing == "description" + assert found["/.claude/projects/-home-kuhy/memory/MEMORY.md"].limit == 24_400 + + +def _git(repo: Path, *args: str, when: int) -> None: + env = {**os.environ, "GIT_AUTHOR_DATE": f"@{when} +0000"} + env |= {"GIT_COMMITTER_DATE": f"@{when} +0000", "HOME": str(repo)} + env |= {"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t"} + env |= {"GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"} + subprocess.run( + [gitcmd.GIT, "-C", str(repo), *args], env=env, check=True, capture_output=True + ) + + +def test_size_history_from_git_and_prune_sizes(tmp_path: Path) -> None: + claude, src = _tree(tmp_path) + _git(claude, "init", "-q", when=1000) + _git(claude, "add", "CLAUDE.md", when=1000) + _git(claude, "commit", "-qm", "a", when=1000) + (claude / "CLAUDE.md").write_text("longer\n" * 50, encoding="utf-8") + _git(claude, "commit", "-qam", "b", when=2000) + prune = claude / "token-report" / "prune" / "2026-10-12" + prune.mkdir(parents=True) + (prune / "sizes.json").write_text(json.dumps({"file_bytes": {"CLAUDE.md": 77}})) + conn = schema.connect(tmp_path / "t.sqlite") + context_files.refresh(conn, claude, src) + rows = conn.execute( + "SELECT ts, bytes, source FROM context_sizes WHERE path LIKE '%/CLAUDE.md'" + " AND path NOT LIKE '%/src/%' ORDER BY ts" + ).fetchall() + listed = conn.execute("SELECT COUNT(*) FROM context_files").fetchone()[0] + context_files.refresh(conn, claude, src) # incremental: no duplicates + again = conn.execute("SELECT COUNT(*) FROM context_sizes").fetchone()[0] + conn.close() + assert [(r[1], r[2]) for r in rows[:2]] == [(40, "git"), (350, "git")] + assert (77, "prune 2026-10-12") in [(r[1], r[2]) for r in rows] + assert listed == 9 + assert again == len(rows) + + +def test_price_mix_weights_cache_read_price_by_calls(tmp_path: Path) -> None: + conn = schema.connect(tmp_path / "t.sqlite") + rows = [ + ("claude-opus-5-5", 100.0), + ("claude-opus-5-5", 101.0), + ("claude-haiku-4-5", 102.0), + ("claude-haiku-4-5", 1.0), + ] + conn.executemany("INSERT INTO turns (model, ts) VALUES (?, ?)", rows) + price = context_files.price_mix(conn, now=200.0, days=150 / 86400) + conn.close() + assert round(price, 4) == round((0.2 * 2 + 0.1) / 3, 4) diff --git a/python_pkg/token_dashboard/tests/test_detail.py b/python_pkg/token_dashboard/tests/test_detail.py new file mode 100644 index 00000000..b1c6c208 --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_detail.py @@ -0,0 +1,164 @@ +"""Per-call and per-turn detail rows the walker keeps.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from python_pkg.token_dashboard import detail +from python_pkg.token_dashboard.tests.conftest import ( + assistant, + iso, + prompt, + result, + use, + walk_detail, +) + +if TYPE_CHECKING: + from pathlib import Path + +ONE_HOUR_WRITE = { + "cache_creation_input_tokens": 1000, + "cache_creation": {"ephemeral_1h_input_tokens": 1000}, +} + + +def test_result_creation_and_carry_are_priced_by_hand(tmp_path: Path) -> None: + rows = walk_detail( + tmp_path, + [ + prompt("go", 0), + assistant("m1", 2, blocks=[use("t1", "Bash", command="git status")]), + result("t1", 5, "x" * 398), # json.dumps -> 400 chars -> 100 tokens + assistant("m2", 9, **ONE_HOUR_WRITE), + assistant("m3", 12, context=2000), + ], + ) + (call,) = rows.calls + assert (call.family, call.target, call.batch) == ("git status", "git status", 1) + assert call.result_tokens == 100 + assert call.end_ts - call.where.ts == pytest.approx(3) + # Written once by m2 at the 1h rate (2x input $4/MTok) ... + assert call.cost.create_usd == pytest.approx(100 * 2.0 * 4 / 1e6) + # ... then cache-read by m2 and m3 at $0.20/MTok. + assert call.cost.carry_usd == pytest.approx(100 * 2 * 0.2 / 1e6) + assert call.cost.carry_calls == 2 + assert call.where.model == "claude-opus-5-5" + + +def test_compaction_stops_the_carry(tmp_path: Path) -> None: + boundary = { + "type": "system", + "subtype": "compact_boundary", + "timestamp": iso(4), + "compactMetadata": { + "trigger": "auto", + "preTokens": 100000, + "postTokens": 9000, + "durationMs": 5000, + }, + } + read = use("t1", "Read", file_path="/a") + rows = walk_detail( + tmp_path, + [ + prompt("go", 0), + assistant("m1", 1, blocks=[read], context=100000), + result("t1", 2, "x" * 398), + assistant("m2", 3, context=100000), + boundary, + assistant("m3", 5, context=9000), + ], + ) + assert rows.calls[0].cost.carry_calls == 1 + (mark,) = [m for m in rows.marks if m.kind == "compaction"] + assert (mark.name, mark.tokens, mark.extra) == ("auto", 100000, 9000) + assert mark.duration_ms == 5000 + assert mark.cost.carry_usd == 0 # the summary call is not context + + +def test_batch_errors_images_and_signatures(tmp_path: Path) -> None: + image = {"type": "image", "source": {"type": "base64", "data": "A" * 9000}} + blocks = [use("a", "Bash", command="ls"), use("b", "mcp__chrome__shot")] + failed = { + "type": "tool_result", + "tool_use_id": "a", + "content": "Exit code 2", + "is_error": True, + } + shot = { + "type": "tool_result", + "tool_use_id": "b", + "content": [image, {"type": "text", "text": "ok"}], + } + rows = walk_detail( + tmp_path, + [ + prompt("go", 0), + assistant("m1", 1, blocks=blocks[:1]), + assistant("m1", 1, blocks=blocks[1:]), + {"type": "user", "timestamp": iso(2), "message": {"content": [failed]}}, + {"type": "user", "timestamp": iso(3), "message": {"content": [shot]}}, + assistant("m2", 4), + ], + ) + by_name = {c.name: c for c in rows.calls} + assert {c.batch for c in rows.calls} == {2} + assert by_name["Bash"].error_kind == "exit_code" + found = by_name["mcp__chrome__shot"] + assert (found.family, found.target, found.images) == ("mcp:chrome", "shot", 1) + assert found.error_kind == "" + assert found.result_tokens < 3000 # an image is priced by area, not base64 + assert by_name["Bash"].sig != found.sig + + +def test_turn_detail_latency_cache_split_and_output_split(tmp_path: Path) -> None: + text = {"type": "text", "text": "y" * 400} # ~100 tokens of reply + thinking = {"type": "thinking", "thinking": ""} + rows = walk_detail( + tmp_path, + [ + prompt("go", 0), + assistant("m1", 7, **ONE_HOUR_WRITE, blocks=[thinking]), + assistant("m1", 8, blocks=[text]), + assistant("m2", 9, output=10, blocks=[text]), + assistant("m3", 10, output=50, blocks=[{"type": "server_tool_use"}]), + assistant("m4", 11, output=40, blocks=[thinking]), + ], + ) + first, second, third, fourth = rows.turns + assert first.latency_s == pytest.approx(7) + assert (first.cw_1h, first.cw_5m) == (1000, 0) + assert (first.out_text, first.out_tool, first.out_think) == (100, 0, 900) + # More visible text than billed output: scaled down, no thinking left. + assert (second.out_text, second.out_think) == (10, 0) + assert second.latency_s == pytest.approx(1) + # Nothing visible: all of it is text, or thinking when a block says so. + assert (third.out_text, third.out_think) == (50, 0) + assert (fourth.out_text, fourth.out_think) == (0, 40) + + +def test_text_and_tool_output_without_thinking_split_by_length( + tmp_path: Path, +) -> None: + blocks = [{"type": "text", "text": "y" * 400}, use("t", "Bash", command="z" * 385)] + rows = walk_detail( + tmp_path, [prompt("go", 0), assistant("m1", 1, output=1000, blocks=blocks)] + ) + (turn,) = rows.turns + assert (turn.out_text, turn.out_tool, turn.out_think) == (500, 500, 0) + assert turn.tool_calls == 1 + + +def test_write_multiplier_blends_the_split() -> None: + assert detail.write_multiplier({}) == pytest.approx(1.25) + half = {"cache_creation_input_tokens": 100, "cache_creation_1h": 50} + assert detail.write_multiplier(half) == pytest.approx((50 * 2 + 50 * 1.25) / 100) + + +def test_result_for_an_unknown_call_is_ignored() -> None: + found = detail.Detail() + found.result({"tool_use_id": "nope", "content": "x"}, 1.0) + assert found.rows.calls == [] diff --git a/python_pkg/token_dashboard/tests/test_detail_marks.py b/python_pkg/token_dashboard/tests/test_detail_marks.py new file mode 100644 index 00000000..6c1c59be --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_detail_marks.py @@ -0,0 +1,198 @@ +"""Context marks: hooks, injected attachments, API errors, interruptions.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from python_pkg.token_dashboard.tests.conftest import ( + BASE, + assistant, + attach, + iso, + prompt, + result, + use, + walk_detail, +) + +if TYPE_CHECKING: + from pathlib import Path + +CTX = "cd chain wrapped in a subshell" + + +def _hook_records() -> list[dict[str, object]]: + run = {"type": "hook_success", "hookName": "PreToolUse:Bash"} + run |= {"hookEvent": "PreToolUse", "toolUseID": "t1", "content": ""} + stop = {"blockingError": "no status tag", "command": "python3 /h/tag_stop.py"} + return [ + attach( + 0, + type="hook_success", + hookName="SessionStart:startup", + hookEvent="SessionStart", + toolUseID="s", + content="x" * 400, + stdout="x" * 400, + command="bash /h/.claude/hooks/start.sh", + durationMs=30, + ), + prompt("go", 1), + assistant("m1", 2, blocks=[use("t1", "Bash", command="cd /x && ls")]), + attach( + 3, + type="hook_additional_context", + hookName="PreToolUse:Bash", + hookEvent="PreToolUse", + toolUseID="t1", + content=[CTX], + ), + attach( + 3, + **run, + stdout='{"additionalContext":"' + CTX + '"}', + command="bash /h/.claude/hooks/cd_chain.sh", + durationMs=15, + ), + attach(3, **run, stdout="{}", command="bash /h/other.sh", durationMs="7"), + result("t1", 4), + assistant("m2", 5), + attach( + 6, + type="hook_blocking_error", + hookName="Stop", + toolUseID="z", + blockingError=stop, + ), + assistant("m3", 7, output=500), + assistant("m4", 8, output=500), + prompt("next", 20), + assistant("m5", 21), + attach(22, type="hook_system_message", hookName="UserPromptSubmit"), + attach( + 22, type="hook_non_blocking_error", hookName="PostToolUse", content="oops" + ), + attach(22, type="nested_memory", path="/r/CLAUDE.md", content={"c": "z"}), + attach(22, type="prompt_snapshot", systemPrompt=["huge"]), + ] + + +def test_hooks_are_marked_matched_and_their_retry_priced(tmp_path: Path) -> None: + rows = walk_detail(tmp_path, _hook_records()) + marks = {(m.kind, m.detail): m for m in rows.marks} + start = marks[("hook_run", "start.sh")] + assert (start.tokens, start.duration_ms, start.where.prompt_idx) == (100, 30, -1) + assert start.cost.carry_usd > 0 + assert start.where.model == "claude-opus-5-5" # filled from the first turn + assert marks[("hook_context", "cd_chain.sh")].tokens == len(CTX) // 4 + assert marks[("hook_run", "other.sh")].duration_ms == 7 + assert marks[("hook_run", "cd_chain.sh")].tokens == 0 # JSON is not injected + bounce = marks[("hook_block", "tag_stop.py")] + assert bounce.name == "Stop" + assert bounce.retry_usd == pytest.approx(2 * 500 * 20 / 1e6) # m3 + m4 only + assert ("hook_msg", "") in marks + assert marks[("hook_error", "")].tokens == 1 + assert marks[("attach", "/r/CLAUDE.md")].name == "nested_memory" + assert not [m for m in rows.marks if m.name == "prompt_snapshot"] + + +def test_pretool_block_retry_is_only_the_next_call(tmp_path: Path) -> None: + rows = walk_detail( + tmp_path, + [ + prompt("go", 0), + assistant("m1", 1), + attach( + 2, + type="hook_blocking_error", + hookName="PreToolUse:Bash", + toolUseID="q", + blockingError="denied", + ), + assistant("m2", 3, output=100), + assistant("m3", 4, output=100), + ], + ) + (block,) = rows.marks + assert (block.detail, block.tokens) == ("", 1) + assert block.retry_usd == pytest.approx(100 * 20 / 1e6) + + +def test_api_errors_interruptions_and_pasted_images(tmp_path: Path) -> None: + pasted = {"type": "image", "source": {"type": "base64", "data": "A"}} + limit = {"isApiErrorMessage": True, "error": "rate_limit"} + limit |= {"apiError": "usage_limit_reached"} + text = [{"type": "text", "text": "Overloaded"}] + rows = walk_detail( + tmp_path, + [ + { + "type": "user", + "timestamp": iso(0), + "message": {"content": [{"type": "text", "text": "look"}, pasted]}, + }, + {"type": "assistant", "timestamp": iso(1), **limit, "message": {}}, + { + "type": "assistant", + "timestamp": iso(2), + "isApiErrorMessage": True, + "message": {"content": text}, + }, + { + "type": "user", + "timestamp": iso(3), + "message": { + "content": [ + {"type": "text", "text": "[Request interrupted by user]"} + ] + }, + }, + assistant("m1", 4), + ], + ) + kinds = sorted((m.kind, m.name, m.detail) for m in rows.marks) + assert kinds == [ + ("api_error", "rate_limit", "usage_limit_reached"), + ("api_error", "unknown", "Overloaded"), + ("image", "user", ""), + ("interrupt", "user", ""), + ] + assert rows.marks[0].where.ts == pytest.approx(BASE) + assert rows.marks[0].tokens > 0 + + +def test_repeats_merge_and_odd_records_are_tolerated(tmp_path: Path) -> None: + reminder = {"type": "total_tokens_reminder", "left": 5} + odd = {"hookName": "PostToolUse", "toolUseID": "u"} + rows = walk_detail( + tmp_path, + [ + prompt("go", 0), + attach(1, **reminder), + attach(2, **reminder), + attach( + 2, + type="hook_success", + **odd, + content="", + stdout="", + command="", + durationMs="n/a", + ), + attach(2, type="hook_additional_context", **odd, content=""), + attach(2, type="hook_additional_context", **odd, content="unmatched"), + attach(2, type="hook_cancelled", **odd), + {"type": "system", "subtype": "turn_duration", "timestamp": iso(2)}, + assistant("m1", 3, blocks=[{"type": "server_tool_use", "id": "w"}]), + ], + ) + (merged,) = [m for m in rows.marks if m.name == "total_tokens_reminder"] + assert merged.count == 2 + assert merged.where.ts == pytest.approx(BASE + 1) + assert merged.cost.carry_calls == 2 + run = next(m for m in rows.marks if m.kind == "hook_run") + assert (run.duration_ms, run.detail) == (0, "") + contexts = [m for m in rows.marks if m.kind == "hook_context"] + assert {m.detail for m in contexts} == {""} diff --git a/python_pkg/token_dashboard/tests/test_detail_store.py b/python_pkg/token_dashboard/tests/test_detail_store.py new file mode 100644 index 00000000..d4e99631 --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_detail_store.py @@ -0,0 +1,79 @@ +"""The detail tables are written per file, replaced on re-walk, filterable.""" + +from __future__ import annotations + +from contextlib import closing +import sqlite3 +from typing import TYPE_CHECKING + +from python_pkg.token_dashboard import ingest +from python_pkg.token_dashboard.tests.conftest import ( + assistant, + iso, + prompt, + result, + use, + write, +) + +if TYPE_CHECKING: + from pathlib import Path + +COUNTS = ( + "SELECT (SELECT COUNT(*) FROM calls), (SELECT COUNT(*) FROM marks)," + " (SELECT COUNT(*) FROM turn_detail)" +) + + +def _records(extra: int) -> list[dict[str, object]]: + hook = { + "type": "attachment", + "timestamp": iso(1), + "attachment": {"type": "hook_system_message", "hookName": "Stop"}, + } + return [ + prompt("go", 0), + hook, + assistant("m1", 2, blocks=[use("t1", "Bash", command="ls")]), + result("t1", 3), + *[assistant(f"x{i}", 4 + i) for i in range(extra)], + ] + + +def _ingest(tmp_path: Path) -> closing[sqlite3.Connection]: + ingest.run( + root=tmp_path / "projects", + db_path=tmp_path / "t.sqlite", + report_dir=tmp_path / "reports", + budget_s=30.0, + ) + return closing(sqlite3.connect(tmp_path / "t.sqlite")) + + +def test_rewalk_replaces_detail_rows(tmp_path: Path) -> None: + path = tmp_path / "projects" / "proj" / "s1.jsonl" + write(path, _records(1)) + with _ingest(tmp_path) as conn: + assert conn.execute(COUNTS).fetchone() == (1, 1, 2) + write(path, _records(2)) + with _ingest(tmp_path) as conn: + assert conn.execute(COUNTS).fetchone() == (1, 1, 3) + + +def test_detail_rows_carry_the_shared_filter_columns(tmp_path: Path) -> None: + write(tmp_path / "projects" / "proj" / "s1.jsonl", _records(1)) + with _ingest(tmp_path) as conn: + call = conn.execute( + "SELECT key, main_key, is_sub, cwd, model, day, family, target, batch," + " carry_calls FROM calls" + ).fetchone() + mark = conn.execute( + "SELECT key, main_key, is_sub, cwd, model, kind, name, count FROM marks" + ).fetchone() + turn = conn.execute( + "SELECT key, is_sub, cwd, model, latency_s FROM turn_detail ORDER BY ts" + ).fetchone() + assert call[:5] == ("s1", "s1", 0, "/repo", "claude-opus-5-5") + assert call[6:] == ("ls", "ls", 1, 1) + assert mark == ("s1", "s1", 0, "/repo", "claude-opus-5-5", "hook_msg", "Stop", 1) + assert turn == ("s1", 0, "/repo", "claude-opus-5-5", 1.0) diff --git a/python_pkg/token_dashboard/tests/test_efficiency.py b/python_pkg/token_dashboard/tests/test_efficiency.py new file mode 100644 index 00000000..d0db538f --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_efficiency.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +from datetime import UTC, datetime +import os +from pathlib import Path +import subprocess + +import pytest + +from python_pkg.token_dashboard import efficiency, gitcmd, survival + +DAY = 86400.0 +MONDAY = datetime(2026, 10, 5, 12, tzinfo=UTC).timestamp() + + +def _s(ts: float, usd: float, tag: str | None) -> efficiency.SessionCost: + return efficiency.SessionCost(key=f"k{ts}{tag}", first_ts=ts, usd=usd, tag=tag) + + +def test_cost_per_done_counts_failed_sessions_against_done_ones() -> None: + rows = [ + _s(MONDAY, 10.0, "DONE"), + _s(MONDAY + 1, 6.0, "QUESTION"), + _s(MONDAY + 2, 4.0, "DONE"), + _s(MONDAY + 3, 99.0, None), # pre-convention: excluded from the ratio + ] + (week,) = efficiency.done_weekly(rows) + assert week["done"] == 2 + assert week["other"] == 1 + assert week["untagged"] == 1 + assert week["usd_per_done"] == pytest.approx(10.0) + assert week["done_share"] == pytest.approx(2 / 3) + + +def test_week_with_no_done_has_no_ratio() -> None: + (week,) = efficiency.done_weekly([_s(MONDAY, 5.0, "RUNNING")]) + assert week["usd_per_done"] is None + + +def test_cost_per_surviving_commit_and_pending() -> None: + week_usd = {efficiency.week_of(MONDAY): 30.0} + verdicts = [ + (MONDAY, "survived"), + (MONDAY, "survived"), + (MONDAY, "churned"), + (MONDAY, "pending"), + ] + (week,) = efficiency.commits_weekly(verdicts, week_usd) + assert (week["survived"], week["lost"], week["pending"]) == (2, 1, 1) + assert week["usd_per_survivor"] == pytest.approx(15.0) + + +def test_cost_per_estimated_unit() -> None: + rows = [_s(MONDAY, 8.0, None), _s(MONDAY + 5, 2.0, "DONE")] + estimates = {rows[0].key: 30, rows[1].key: 10} + (week,) = efficiency.estimate_weekly(rows, estimates) + assert week["est_calls"] == 40 + assert week["usd_per_unit"] == pytest.approx(0.25) + + +@pytest.mark.parametrize( + ("added", "kept", "reverted", "expected"), + [ + (10, 10, False, "survived"), + (10, 5, False, "survived"), + (10, 4, False, "churned"), + (10, 10, True, "reverted"), + (0, 0, False, "survived"), + ], +) +def test_survival_rule(added: int, kept: int, *, reverted: bool, expected: str) -> None: + assert survival.judge(added, kept, reverted=reverted) == expected + + +def _git(repo: Path, *args: str, when: float) -> str: + stamp = f"@{int(when)} +0000" + env = {**os.environ, "GIT_AUTHOR_DATE": stamp, "GIT_COMMITTER_DATE": stamp} + env |= {"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "HOME": str(repo)} + env |= {"GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"} + done = subprocess.run( + [gitcmd.GIT, "-C", str(repo), *args], + env=env, + check=True, + capture_output=True, + text=True, + ) + return done.stdout.strip() + + +def _commit(repo: Path, name: str, text: str, when: float) -> str: + (repo / name).write_text(text, encoding="utf-8") + _git(repo, "add", name, when=when) + _git(repo, "commit", "-qm", name, when=when) + return _git(repo, "rev-parse", "HEAD", when=when) + + +def test_git_survival_against_a_real_repo(tmp_path: Path) -> None: + repo = tmp_path / "r" + repo.mkdir() + _git(repo, "init", "-q", when=MONDAY) + t0 = MONDAY - 40 * DAY + kept = _commit(repo, "a.txt", "1\n2\n3\n4\n", t0) + churn = _commit(repo, "b.txt", "1\n2\n3\n4\n", t0 + 1) + _commit(repo, "b.txt", "x\ny\nz\n4\n", t0 + 2 * DAY) + young = _commit(repo, "c.txt", "1\n", MONDAY) + now = MONDAY + 1 + assert survival.evaluate(str(repo), kept[:8], now).verdict == "survived" + churned = survival.evaluate(str(repo), churn, now) + assert (churned.verdict, churned.added, churned.kept) == ("churned", 4, 1) + assert survival.evaluate(str(repo), young, now).verdict == "pending" + assert survival.evaluate(str(repo), "deadbeef", now).verdict == "unknown" + assert survival.evaluate(str(tmp_path / "nope"), kept, now).verdict == "unknown" + + +def test_git_revert_within_window_is_reverted(tmp_path: Path) -> None: + repo = tmp_path / "r" + repo.mkdir() + _git(repo, "init", "-q", when=MONDAY) + t0 = MONDAY - 40 * DAY + _commit(repo, "a.txt", "base\n", t0 - 10) + bad = _commit(repo, "b.txt", "1\n", t0) + _git(repo, "revert", "--no-edit", bad, when=t0 + DAY) + assert survival.evaluate(str(repo), bad, MONDAY).verdict == "reverted" + + +def test_update_judges_old_commits_and_stops_at_the_deadline(tmp_path: Path) -> None: + from python_pkg.token_dashboard import schema + + repo = tmp_path / "r" + repo.mkdir() + _git(repo, "init", "-q", when=MONDAY) + old = _commit(repo, "a.txt", "1\n", MONDAY - 40 * DAY) + conn = schema.connect(tmp_path / "t.sqlite") + rows = [ + (old, str(repo), MONDAY - 40 * DAY, "s", 0), + ("f" * 40, "x", MONDAY, "s", 0), + ] + conn.executemany("INSERT INTO session_commits VALUES (?,?,?,?,?)", rows) + assert not survival.update(conn, MONDAY, deadline=0.0, clock=lambda: 1.0) + assert survival.update(conn, MONDAY, deadline=2.0, clock=lambda: 1.0) + efficiency.rebuild(conn, MONDAY) + verdicts = conn.execute("SELECT verdict FROM commit_verdicts").fetchall() + payload = conn.execute("SELECT value FROM meta WHERE key='efficiency'").fetchone() + conn.close() + assert verdicts == [("survived",)] + assert '"pending": 1' in payload[0] diff --git a/python_pkg/token_dashboard/tests/test_gaps_git.py b/python_pkg/token_dashboard/tests/test_gaps_git.py new file mode 100644 index 00000000..4b880918 --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_gaps_git.py @@ -0,0 +1,54 @@ +"""Git-backed edge cases: caching, binary files, commits outside HEAD.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from python_pkg.token_dashboard import commits, survival +from python_pkg.token_dashboard.tests.test_efficiency import DAY, MONDAY, _commit, _git + +if TYPE_CHECKING: + from pathlib import Path + + +def _repo(tmp_path: Path) -> Path: + repo = tmp_path / "r" + repo.mkdir() + _git(repo, "init", "-q", when=MONDAY) + return repo + + +def test_toplevel_is_resolved_once_per_hint(tmp_path: Path) -> None: + repo = _repo(tmp_path) + _commit(repo, "a", "1", MONDAY) + cache: dict[str, str | None] = {} + first = commits._toplevel(str(repo), cache) + assert first == str(repo.resolve()) + cache[str(repo)] = "cached" + assert commits._toplevel(str(repo), cache) == "cached" + assert commits._toplevel("", cache) is None + + +def test_binary_and_empty_files_do_not_count_as_added_lines(tmp_path: Path) -> None: + repo = _repo(tmp_path) + old = MONDAY - 40 * DAY + (repo / "blob.bin").write_bytes(b"\x00\x01\x02") + (repo / "empty.txt").write_text("", encoding="utf-8") + (repo / "t.txt").write_text("1\n2\n", encoding="utf-8") + _git(repo, "add", ".", when=old) + _git(repo, "commit", "-qm", "mixed", when=old) + sha = _git(repo, "rev-parse", "HEAD", when=old) + found = survival.evaluate(str(repo), sha, MONDAY) + assert (found.verdict, found.added, found.kept) == ("survived", 2, 2) + + +def test_a_commit_that_is_not_an_ancestor_of_head_is_unknown(tmp_path: Path) -> None: + repo = _repo(tmp_path) + old = MONDAY - 40 * DAY + _commit(repo, "a", "1", old) + trunk = _git(repo, "branch", "--show-current", when=old) + _git(repo, "checkout", "-qb", "side", when=old) + side = _commit(repo, "b", "2", old + 1) + _git(repo, "checkout", "-q", trunk, when=old) + found = survival.evaluate(str(repo), side, MONDAY) + assert (found.verdict, found.sha) == ("unknown", side) diff --git a/python_pkg/token_dashboard/tests/test_gaps_ingest.py b/python_pkg/token_dashboard/tests/test_gaps_ingest.py new file mode 100644 index 00000000..01427a6b --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_gaps_ingest.py @@ -0,0 +1,116 @@ +"""Edge cases of the walker, the summaries and the aux readers.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +from python_pkg.token_dashboard import ab, context_files, finalize, ingest, schema, walk +from python_pkg.token_dashboard.tests.conftest import ( + assistant, + prompt, + result, + use, + write, +) + +if TYPE_CHECKING: + from pathlib import Path + + +def test_timestampless_orphans_and_garbage_lines_are_tolerated(tmp_path: Path) -> None: + path = tmp_path / "proj" / "s.jsonl" + path.parent.mkdir() + lines = ['{"type": "summary", "summary": "x"}', "{broken", ""] + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + facts = walk.walk_file(path, tmp_path) + assert facts.span == (0.0, 0.0) + assert facts.rows.turns == [] + assert facts.final_tag is None + + +def test_a_result_without_its_call_is_ignored(tmp_path: Path) -> None: + path = write(tmp_path / "proj" / "s.jsonl", [prompt("go", 0), result("nope", 1)]) + facts = walk.walk_file(path, tmp_path) + assert facts.rows.tools == [] + + +def test_a_nested_subagent_resolves_to_the_prompt_that_started_the_chain( + tmp_path: Path, +) -> None: + root = tmp_path / "projects" + outer = use("ta", "Agent", description="outer") + inner = use("tb", "Agent", description="inner") + write( + root / "p" / "s1.jsonl", + [ + assistant("m0", 0), # a turn before any prompt + prompt("go", 5), + assistant("m1", 10, blocks=[outer]), + result("ta", 11, "ok", toolUseResult={"agentId": "aaa"}), + ], + ) + subs = root / "p" / "s1" / "subagents" + write( + subs / "agent-aaa.jsonl", + [ + assistant("a1", 20, blocks=[inner]), + result("tb", 21, toolUseResult={"agentId": "bbb"}), + ], + ) + write(subs / "agent-bbb.jsonl", [assistant("b1", 30)]) + ingest.run(root, tmp_path / "t.sqlite", tmp_path / "report", 30.0) + conn = schema.connect(tmp_path / "t.sqlite") + links = conn.execute( + "SELECT sub_key, prompt_idx, depth FROM subagent_link ORDER BY depth" + ).fetchall() + first = conn.execute( + "SELECT idx, text FROM prompt_summary WHERE key='s1' ORDER BY idx" + ).fetchone() + conn.close() + assert links == [("s1/agent-aaa", 0, 1), ("s1/agent-bbb", 0, 2)] + assert first == (-1, "(before the first prompt)") + + +def test_refreshing_an_unknown_session_changes_nothing(tmp_path: Path) -> None: + conn = schema.connect(tmp_path / "t.sqlite") + finalize.refresh(conn, "ghost") + assert conn.execute("SELECT COUNT(*) FROM session_summary").fetchone() == (0,) + conn.close() + + +def test_unreadable_prune_sizes_are_skipped(tmp_path: Path) -> None: + claude, src = tmp_path / "claude", tmp_path / "src" + prune = claude / "token-report" / "prune" + for name, body in ( + ("a", "{broken"), + ("b", "[1]"), + ("c", '{"file_bytes": {"X.md": 9}}'), + ): + (prune / name).mkdir(parents=True) + (prune / name / "sizes.json").write_text(body, encoding="utf-8") + conn = schema.connect(tmp_path / "t.sqlite") + context_files.refresh(conn, claude, src) + sources = conn.execute("SELECT source FROM context_sizes").fetchall() + conn.close() + assert sources == [("prune c",)] + + +def test_unmatched_and_malformed_assignments_are_counted_unjoined( + tmp_path: Path, +) -> None: + log = tmp_path / "agent-ab.jsonl" + rows = [ + {"tool_use_id": "ghost", "arm": "sonnet"}, + {"tool_use_id": "t1", "arm": "haiku"}, + {"tool_use_id": "t2"}, + [1], + ] + log.write_text("\n".join(json.dumps(r) for r in rows) + "\n{broken\n") + conn = schema.connect(tmp_path / "t.sqlite") + payload = ab.rebuild(conn, log) + default = ab.rebuild(conn) # the isolated, absent default log + conn.close() + assert payload["unjoined"] == 2 + assert payload["arms"]["sonnet"]["n"] == 0 + assert default["unjoined"] == 0 diff --git a/python_pkg/token_dashboard/tests/test_gaps_units.py b/python_pkg/token_dashboard/tests/test_gaps_units.py new file mode 100644 index 00000000..b20d4c55 --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_gaps_units.py @@ -0,0 +1,94 @@ +"""Edge cases of the small pure helpers: blocks, gitcmd, levers, schema, efficiency.""" + +from __future__ import annotations + +import json +import sqlite3 +import subprocess +from typing import TYPE_CHECKING + +import pytest + +from python_pkg.token_dashboard import blocks, efficiency, gitcmd, levers, schema + +if TYPE_CHECKING: + from pathlib import Path + + +def test_text_of_joins_text_blocks_and_ignores_the_rest() -> None: + content = [ + {"type": "text", "text": "one"}, + {"type": "image"}, + "stray", + {"type": "text", "text": "two"}, + ] + assert blocks.text_of(content) == "one\ntwo" + assert blocks.text_of("plain") == "plain" + assert blocks.text_of(None) == "" + assert blocks.text_of({"type": "text", "text": "dict"}) == "" + + +def test_compact_summaries_and_tool_results_are_not_prompts() -> None: + message = {"role": "user", "content": "summary of the chat"} + summary = {"type": "user", "isCompactSummary": True, "message": message} + assert blocks.prompt_text(summary) is None + visible = {"type": "user", "isVisibleInTranscriptOnly": True, "message": message} + assert blocks.prompt_text(visible) is None + tool = [{"type": "tool_result", "tool_use_id": "t", "content": "x"}] + reply = {"type": "user", "message": {"content": tool}} + assert blocks.prompt_text(reply) is None + + +def test_repo_hint_falls_back_to_cwd_and_resolves_relative_targets() -> None: + assert blocks.repo_hint("git commit -m x", None) == "" + assert blocks.repo_hint("git commit -m x", "/work/r") == "/work/r" + assert blocks.repo_hint("cd sub && git commit", "/work/r") == "/work/r/sub" + assert blocks.repo_hint("git -C /abs/repo commit", "/work/r") == "/abs/repo" + + +def test_lever_table_needs_the_heading_and_at_least_one_row() -> None: + heading = f"## {blocks.LEVER_HEADING}\n\n" + assert blocks.lever_table("nothing here") is None + assert blocks.lever_table(heading + "just prose") is None + assert ( + blocks.lever_table(heading + "| a | b |\n| c | d |") == "| a | b |\n| c | d |" + ) + + +@pytest.mark.parametrize( + "error", [OSError("no git"), subprocess.TimeoutExpired("git", 1)] +) +def test_git_run_never_raises( + error: Exception, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + def boom(*_args: object, **_kwargs: object) -> None: + raise error + + monkeypatch.setattr("python_pkg.token_dashboard.gitcmd.subprocess.run", boom) + assert gitcmd.run(tmp_path, "status") is None + + +def test_unreadable_or_non_object_snapshots_are_skipped(tmp_path: Path) -> None: + history = tmp_path / "history" + history.mkdir() + (history / "2026-10-01.json").write_text("{broken") + (history / "2026-10-02.json").write_text(json.dumps([1, 2])) + (history / "2026-10-03.json").write_text(json.dumps({"metrics": {}})) + assert levers._snapshots(tmp_path) == [("2026-10-03", {"metrics": {}})] + + +def test_readonly_store_refuses_writes(tmp_path: Path) -> None: + db = tmp_path / "t.sqlite" + schema.connect(db).close() + conn = schema.connect(db, readonly=True) + try: + assert conn.execute("SELECT COUNT(*) FROM turns").fetchone() == (0,) + with pytest.raises(sqlite3.OperationalError, match="readonly"): + conn.execute("INSERT INTO meta VALUES ('k', 'v')") + finally: + conn.close() + + +def test_commits_with_an_unrecognised_verdict_count_as_unknown() -> None: + (week,) = efficiency.commits_weekly([(1.0, "mystery")], {}) + assert (week["unknown"], week["survived"], week["lost"]) == (1, 0, 0) diff --git a/python_pkg/token_dashboard/tests/test_ingest.py b/python_pkg/token_dashboard/tests/test_ingest.py new file mode 100644 index 00000000..4e4be8d4 --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_ingest.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import json +import sqlite3 +from typing import TYPE_CHECKING + +import pytest + +from python_pkg.token_dashboard import ingest +from python_pkg.token_dashboard.tests.conftest import ( + assistant, + prompt, + result, + usd_opus_output, + use, + write, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + from pathlib import Path + + +def _tree(root: Path) -> None: + agent = use("ta", "Agent", description="Find x", subagent_type="Explore") + write( + root / "proj" / "s1.jsonl", + [ + prompt("first", 0), + assistant("m1", 10, blocks=[agent]), + result("ta", 11, "launched", toolUseResult={"agentId": "abc"}), + assistant("m2", 20), + prompt("second", 1000), + assistant("m3", 1010, blocks=[{"type": "text", "text": "DONE: x"}]), + ], + ) + subdir = root / "proj" / "s1" / "subagents" + write(subdir / "agent-abc.jsonl", [prompt("task", 12), assistant("a1", 200)]) + (subdir / "agent-abc.meta.json").write_text( + json.dumps({"agentType": "Explore", "description": "Find x"}), + encoding="utf-8", + ) + # An agent that outlived /clear: nothing in s1 spawned it. + write(subdir / "agent-zzz.jsonl", [prompt("orphan", 30), assistant("z1", 40)]) + + +def _run(tmp_path: Path, budget: float = 30.0) -> ingest.Outcome: + return ingest.run( + root=tmp_path / "projects", + db_path=tmp_path / "t.sqlite", + report_dir=tmp_path / "report", + budget_s=budget, + ) + + +@pytest.fixture +def db(tmp_path: Path) -> Iterator[Callable[[], sqlite3.Connection]]: + opened: list[sqlite3.Connection] = [] + + def _open() -> sqlite3.Connection: + opened.append(sqlite3.connect(tmp_path / "t.sqlite")) + return opened[-1] + + yield _open + for conn in opened: + conn.close() + + +def test_session_cost_reconciles_to_prompts_plus_unattributed( + tmp_path: Path, db: Callable[[], sqlite3.Connection] +) -> None: + _tree(tmp_path / "projects") + outcome = _run(tmp_path) + assert outcome.complete + conn = db() + usd, active_s, wall_s, tag = conn.execute( + "SELECT usd, active_s, wall_s, final_tag FROM session_summary WHERE key='s1'" + ).fetchone() + assert usd == pytest.approx(usd_opus_output(1000) * 5) + prompts = conn.execute( + "SELECT idx, usd_own + usd_sub FROM prompt_summary WHERE key='s1' ORDER BY idx" + ).fetchall() + unattributed = conn.execute( + "SELECT SUM(usd) FROM subagent_link WHERE main_key='s1' AND prompt_idx IS NULL" + ).fetchone()[0] + assert sum(v for _, v in prompts) + unattributed == pytest.approx(usd) + # Prompt 0 carries its own two turns plus the Explore agent it spawned. + assert prompts[0][1] == pytest.approx(usd_opus_output(1000) * 3) + assert wall_s == 1010 + # Union of main + subagent activity: 0..200 bridged, then 1000..1010. + assert active_s == pytest.approx(210) + assert tag == "DONE" + + +def test_subagent_links_to_its_prompt_via_agent_id( + tmp_path: Path, db: Callable[[], sqlite3.Connection] +) -> None: + _tree(tmp_path / "projects") + _run(tmp_path) + rows = dict( + db().execute("SELECT sub_key, prompt_idx FROM subagent_link").fetchall() + ) + assert rows == {"s1/agent-abc": 0, "s1/agent-zzz": None} + + +def test_rerun_skips_unchanged_and_rows_outlive_deleted_files( + tmp_path: Path, db: Callable[[], sqlite3.Connection] +) -> None: + _tree(tmp_path / "projects") + _run(tmp_path) + assert _run(tmp_path).parsed == 0 + (tmp_path / "projects" / "proj" / "s1.jsonl").unlink() + _run(tmp_path) + count = db().execute("SELECT COUNT(*) FROM turns").fetchone()[0] + assert count == 5 + + +def test_changed_file_is_reparsed_without_duplicating_rows( + tmp_path: Path, db: Callable[[], sqlite3.Connection] +) -> None: + _tree(tmp_path / "projects") + _run(tmp_path) + path = tmp_path / "projects" / "proj" / "s1.jsonl" + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(assistant("m4", 1020)) + "\n") + assert _run(tmp_path).parsed == 1 + count = db().execute("SELECT COUNT(*) FROM turns").fetchone()[0] + assert count == 6 + + +def test_zero_budget_leaves_work_pending_and_resumes(tmp_path: Path) -> None: + _tree(tmp_path / "projects") + first = _run(tmp_path, budget=0.0) + assert not first.complete + assert first.parsed == 1 # always makes progress + rest = _run(tmp_path) + assert rest.complete + assert first.parsed + rest.parsed == 3 + + +def test_daily_rollup_matches_turns( + tmp_path: Path, db: Callable[[], sqlite3.Connection] +) -> None: + _tree(tmp_path / "projects") + _run(tmp_path) + conn = db() + total, calls, sub = conn.execute( + "SELECT SUM(usd), SUM(calls), SUM(usd_sub) FROM daily" + ).fetchone() + assert total == pytest.approx(usd_opus_output(1000) * 5) + assert calls == 5 + assert sub == pytest.approx(usd_opus_output(1000) * 2) + + +def test_cli_exit_codes(tmp_path: Path) -> None: + _tree(tmp_path / "projects") + args = [ + "--root", + str(tmp_path / "projects"), + "--db", + str(tmp_path / "t.sqlite"), + "--report-dir", + str(tmp_path / "report"), + ] + assert ingest.main([*args, "--budget", "0"]) == ingest.EXIT_PENDING + assert ingest.main(args) == 0 + + +def test_session_with_no_prompts_and_no_main_file( + tmp_path: Path, db: Callable[[], sqlite3.Connection] +) -> None: + sub = tmp_path / "projects" / "proj" / "gone" / "subagents" / "agent-q.jsonl" + write(sub, [assistant("q1", 5), assistant("q2", 6)]) + assert _run(tmp_path).complete + usd, prompts = ( + db() + .execute("SELECT usd, prompts FROM session_summary WHERE key='gone'") + .fetchone() + ) + assert usd == pytest.approx(usd_opus_output(1000) * 2) + assert prompts == 0 diff --git a/python_pkg/token_dashboard/tests/test_levers.py b/python_pkg/token_dashboard/tests/test_levers.py new file mode 100644 index 00000000..d5822d45 --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_levers.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +from python_pkg.token_dashboard import levers + +if TYPE_CHECKING: + from pathlib import Path + +TABLE = """## Candidate levers (rule-based) + +| category | lever | evidence | est $/day | action | +|---|---|---|---|---| +| cheaper agents | Opus runs most subagent work | subagents 64% | $14.47 | route | +| agent mechanics | Large Bash results | 40% of Bash | - | read ranges | + +## Next +""" + + +def test_parse_candidate_table() -> None: + rows = levers.parse_candidates(TABLE) + assert [r["title"] for r in rows] == [ + "Opus runs most subagent work", + "Large Bash results", + ] + assert rows[0]["est_usd_per_day"] == 14.47 + assert rows[1]["est_usd_per_day"] is None + assert rows[0]["metric"] == "subagent_opus_usd_share" + + +def test_ledger_date_falls_back_to_text_then_unknown() -> None: + assert levers.ledger_date({"date": "2026-10-04", "lever": "x"}) == "2026-10-04" + assert levers.ledger_date({"lever": "weekly prefix prune 2026-10-05"}) == ( + "2026-10-05" + ) + assert levers.ledger_date({"lever": "no date"}) is None + + +def _report(tmp_path: Path) -> Path: + report = tmp_path / "report" + (report / "history").mkdir(parents=True) + ledger = [ + { + "date": "2026-10-04", + "lever": "routing", + "metric": "subagent_opus_usd_share", + "baseline": 0.98, + "predicted": 0.55, + }, + { + "lever": "weekly prefix prune 2026-10-05", + "metric": "standing_tokens_est", + "baseline": 22187, + "predicted": 19620, + }, + ] + (report / "ledger.jsonl").write_text( + "\n".join(json.dumps(e) for e in ledger), encoding="utf-8" + ) + for day, value in (("2026-10-10", 0.8), ("2026-10-17", 0.84)): + snap = { + "generated": f"{day}T10:00:00+00:00", + "metrics": {"subagent_opus_usd_share": value, "standing_tokens_est": 23670}, + } + (report / "history" / f"{day}.json").write_text(json.dumps(snap)) + for day in ("2026-10-05", "2026-10-12"): + (report / "prune" / day).mkdir(parents=True) + (report / "prune" / day / "PROPOSAL.md").write_text( + f"# Prefix prune proposal {day}\n\nModest week. Estimated ~ -0.35k.\n" + ) + return report + + +def test_build_rows_status_and_outcome(tmp_path: Path) -> None: + report = _report(tmp_path) + tables = {"2026-10-10": TABLE} + rows = levers.build(report, tables) + applied = [r for r in rows if r["source"] == "ledger"] + assert [r["lever"] for r in applied] == [ + "routing", + "weekly prefix prune 2026-10-05", + ] + assert applied[0]["outcome"] == "helped" + assert applied[1]["outcome"] == "regressed" + assert applied[1]["date"] == "2026-10-05" + status = {r["source"]: r["status"] for r in rows if r["source"].startswith("prune")} + assert status == {"prune 2026-10-05": "applied", "prune 2026-10-12": "proposed"} + # The 10-10 Opus candidate was followed by no ledger line on its metric + # and a newer snapshot exists, so it was not applied. + opus = next(r for r in rows if r["lever"] == "Opus runs most subagent work") + assert opus["status"] == "not applied" + + +def test_structured_levers_in_a_snapshot_win_over_markdown(tmp_path: Path) -> None: + report = _report(tmp_path) + snap_path = report / "history" / "2026-10-17.json" + snap = json.loads(snap_path.read_text()) + snap["levers"] = [ + { + "category": "c", + "title": "Structured one", + "evidence": "e", + "action": "a", + "est_usd_per_day": 1.5, + "metric": "usd_per_day", + }, + ] + snap_path.write_text(json.dumps(snap)) + rows = levers.build(report, {}) + found = next(r for r in rows if r["lever"] == "Structured one") + assert found["status"] == "proposed" + assert found["source"] == "snapshot 2026-10-17" diff --git a/python_pkg/token_dashboard/tests/test_schema_migrate.py b/python_pkg/token_dashboard/tests/test_schema_migrate.py new file mode 100644 index 00000000..a1834eae --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_schema_migrate.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import sqlite3 +from typing import TYPE_CHECKING + +import pytest + +from python_pkg.token_dashboard import schema + +if TYPE_CHECKING: + from pathlib import Path + + +def _v1_store(path: Path) -> None: + """A store as schema v1 wrote it: commits without end_ts, no A/B columns.""" + conn = sqlite3.connect(path) + conn.executescript( + "CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT);" + "INSERT INTO meta VALUES ('schema', '1');" + "CREATE TABLE commits(key TEXT, prompt_idx INT, ts REAL, sha TEXT," + " repo TEXT);" + "INSERT INTO commits VALUES ('s1', 0, 5.0, 'abc', '/r');" + "CREATE TABLE agent_calls(key TEXT, tool_use_id TEXT, agent_id TEXT," + " prompt_idx INT, ts REAL, description TEXT, subagent_type TEXT," + " model TEXT);" + "INSERT INTO agent_calls VALUES ('s1','t','a',0,1.0,'d','x','m');" + ) + conn.commit() + conn.close() + + +def test_old_store_is_migrated_in_place_and_keeps_its_rows(tmp_path: Path) -> None: + path = tmp_path / "t.sqlite" + _v1_store(path) + conn = schema.connect(path) + kept = conn.execute("SELECT key, sha, end_ts FROM commits").fetchall() + calls = conn.execute("SELECT tool_use_id, result_ts, status FROM agent_calls") + version = conn.execute("SELECT value FROM meta WHERE key='schema'").fetchone() + assert kept == [("s1", "abc", None)] + assert calls.fetchall() == [("t", None, None)] + assert version == (schema.VERSION,) + conn.close() + + +def test_newer_store_is_refused_not_relabelled(tmp_path: Path) -> None: + path = tmp_path / "t.sqlite" + conn = sqlite3.connect(path) + conn.executescript( + "CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT);" + "INSERT INTO meta VALUES ('schema', '999');" + ) + conn.commit() + conn.close() + with pytest.raises(schema.SchemaError): + schema.connect(path) + check = sqlite3.connect(path) + assert check.execute("SELECT value FROM meta").fetchone() == ("999",) + check.close() + + +def _meta_only(path: Path, version: str | None) -> None: + conn = sqlite3.connect(path) + conn.execute("CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT)") + if version is not None: + conn.execute("INSERT INTO meta VALUES ('schema', ?)", (version,)) + conn.commit() + conn.close() + + +def test_meta_without_a_version_is_refused(tmp_path: Path) -> None: + _meta_only(tmp_path / "t.sqlite", None) + with pytest.raises(schema.SchemaError): + schema.connect(tmp_path / "t.sqlite") + + +def test_table_missing_from_an_old_store_is_created_at_todays_shape( + tmp_path: Path, +) -> None: + _meta_only(tmp_path / "t.sqlite", "1") + conn = schema.connect(tmp_path / "t.sqlite") + columns = {r[1] for r in conn.execute("PRAGMA table_info(commits)")} + conn.close() + assert "end_ts" in columns diff --git a/python_pkg/token_dashboard/tests/test_walk.py b/python_pkg/token_dashboard/tests/test_walk.py new file mode 100644 index 00000000..2e94dbd5 --- /dev/null +++ b/python_pkg/token_dashboard/tests/test_walk.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from python_pkg.token_dashboard import walk +from python_pkg.token_dashboard.tests.conftest import ( + assistant, + prompt, + result, + usd_opus_output, + use, + write, +) + +if TYPE_CHECKING: + from pathlib import Path + + +def _main(tmp_path: Path, records: list[dict[str, object]]) -> walk.FileFacts: + path = write(tmp_path / "proj" / "sess-1.jsonl", records) + return walk.walk_file(path, tmp_path) + + +def test_one_api_message_written_as_three_records_is_priced_once( + tmp_path: Path, +) -> None: + blocks = [use("t1", "Bash", command="ls"), use("t2", "Read", file_path="/a")] + facts = _main( + tmp_path, + [ + prompt("do it", 0), + assistant("m1", 1, blocks=[{"type": "thinking", "thinking": ""}]), + assistant("m1", 1, blocks=blocks[:1]), + assistant("m1", 1, blocks=blocks[1:]), + result("t1", 2), + result("t2", 2), + ], + ) + assert len(facts.rows.turns) == 1 + assert facts.rows.turns[0].usd == pytest.approx(usd_opus_output(1000)) + # The one turn's dollars split evenly across the two tools it issued. + by_name = {t.name: t for t in facts.rows.tools} + assert by_name["Bash"].usd == pytest.approx(usd_opus_output(500)) + assert by_name["Bash"].calls == 1 + assert by_name["Read"].calls == 1 + assert by_name["Read"].result_tokens > 0 + + +def test_prompts_split_turns_and_text_only_turns_go_to_no_tool( + tmp_path: Path, +) -> None: + facts = _main( + tmp_path, + [ + prompt("first " + "a" * 300, 0, promptId="p1"), + assistant("m1", 10), + prompt("second", 100), + assistant("m2", 110), + assistant("m3", 120), + ], + ) + assert [p.idx for p in facts.rows.prompts] == [0, 1] + assert len(facts.rows.prompts[0].text) <= walk.PROMPT_CHARS + assert facts.rows.prompts[0].prompt_id == "p1" + assert [t.prompt_idx for t in facts.rows.turns] == [0, 1, 1] + assert {t.name for t in facts.rows.tools} == {walk.NO_TOOL} + + +def test_tool_results_and_meta_records_are_not_prompts(tmp_path: Path) -> None: + facts = _main( + tmp_path, + [ + prompt("real", 0), + prompt("injected", 1), + prompt("hidden", 2, isMeta=True), + prompt("[Request interrupted by user]", 3), + prompt("/clear", 4), + assistant("m1", 5, blocks=[use("t1", "Bash", command="ls")]), + result("t1", 6), + ], + ) + assert [p.text for p in facts.rows.prompts] == ["real", "/clear"] + + +def test_agent_call_and_commit_and_final_tag_are_captured(tmp_path: Path) -> None: + agent = use("ta", "Agent", description="Find x", subagent_type="Explore") + commit = use("tc", "Bash", command="git -C ~/src/foo commit -m x") + facts = _main( + tmp_path, + [ + prompt("go", 0), + assistant("m1", 1, blocks=[agent]), + result("ta", 2, "launched", toolUseResult={"agentId": "abc"}), + assistant("m2", 3, blocks=[commit]), + result("tc", 4, "[main 1a2b3c4d] x\n 1 file changed"), + assistant( + "m3", 5, blocks=[{"type": "text", "text": "x\nDONE: ok\nNOTE: y"}] + ), + ], + ) + call = facts.rows.agent_calls[0] + assert (call.tool_use_id, call.agent_id, call.description) == ( + "ta", + "abc", + "Find x", + ) + assert call.subagent_type == "Explore" + assert facts.rows.commits[0].sha == "1a2b3c4d" + assert facts.rows.commits[0].repo.endswith("/src/foo") + assert facts.final_tag == "DONE" + + +@pytest.mark.parametrize( + ("text", "tag"), + [ + ("all good", None), + ("**NEEDS ACTION:** run it", "NEEDS ACTION"), + ("RUNNING: tests", "RUNNING"), + ], +) +def test_final_tag_variants(tmp_path: Path, text: str, tag: str | None) -> None: + blocks = [{"type": "text", "text": text}] + facts = _main(tmp_path, [prompt("go", 0), assistant("m1", 1, blocks=blocks)]) + assert facts.final_tag == tag + + +def test_lever_table_in_a_tool_result_is_kept(tmp_path: Path) -> None: + table = ( + "## Candidate levers (rule-based)\n\n| category | lever |\n|---|---|\n" + "| a | b |\n\nafter" + ) + facts = _main( + tmp_path, + [ + prompt("go", 0), + assistant("m1", 1, blocks=[use("t1", "Bash", command="x")]), + result("t1", 2, table), + ], + ) + assert facts.rows.lever_tables[0][1].splitlines()[-1] == "| a | b |" + + +def test_activity_intervals_break_on_gaps_over_five_minutes(tmp_path: Path) -> None: + facts = _main( + tmp_path, + [prompt("a", 0), assistant("m1", 60), prompt("b", 1000), assistant("m2", 1100)], + ) + spans = [(round(e - s)) for s, e in facts.rows.intervals] + assert spans == [60, 100] + assert facts.span[1] - facts.span[0] == 1100 + + +def test_subagent_file_reads_meta_and_parent(tmp_path: Path) -> None: + sub = tmp_path / "proj" / "sess-1" / "subagents" / "agent-abc.jsonl" + write(sub, [prompt("task", 0), assistant("m1", 1, model="claude-sonnet-5-5")]) + sub.with_suffix(".meta.json").write_text( + '{"agentType": "Explore", "description": "Find", "toolUseId": "ta"}', + encoding="utf-8", + ) + facts = walk.walk_file(sub, tmp_path) + assert facts.key == "sess-1/agent-abc" + assert facts.parent_session == "sess-1" + assert facts.meta["toolUseId"] == "ta" + assert facts.rows.prompts == [] diff --git a/python_pkg/token_dashboard/walk.py b/python_pkg/token_dashboard/walk.py new file mode 100644 index 00000000..79f7cfed --- /dev/null +++ b/python_pkg/token_dashboard/walk.py @@ -0,0 +1,226 @@ +"""Walk one transcript file into the rows the store keeps for it. + +Reuses token_audit's record primitives -- line loading, usage extraction, +per-API-message dedup (:class:`TurnMerger`) and list-price pricing -- rather +than :func:`parse.iter_events`, because the dashboard also needs what that +stream drops: every record's timestamp (active time), prompt boundaries, the +``Agent`` calls that spawned subagents, and commit hashes. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from python_pkg.token_audit import discover, parse, pricing, records, turns +from python_pkg.token_dashboard import active, blocks, carry, detail +from python_pkg.token_dashboard.blocks import PROMPT_CHARS +from python_pkg.token_dashboard.rows import ( + AgentCall, + Commit, + FileFacts, + PromptRow, + ToolRow, + TurnRow, +) + +if TYPE_CHECKING: + from pathlib import Path + + from python_pkg.token_audit.model import Turn + +__all__ = ["NO_TOOL", "PROMPT_CHARS", "FileFacts", "walk_file"] + +NO_TOOL = "(no tool)" +AGENT_TOOLS = frozenset({"Agent", "Task"}) +_BOOK = pricing.load() + + +class _Walker: + """Single pass over a file's records, holding the cross-record state.""" + + def __init__(self, facts: FileFacts) -> None: + self.facts = facts + self.prompt = -1 + self.ts = 0.0 + self.stamps: list[float] = [] + self.pending: dict[str, dict[str, Any]] = {} + self.names: dict[str, list[str]] = {} + self.use_ts: dict[str, float] = {} + self.tools: dict[tuple[int, str], ToolRow] = {} + self.carry = carry.Tracker() + self.detail = detail.Detail() + self.merger = records.TurnMerger() + self.last_text = "" + + def tool(self, name: str) -> ToolRow: + """The accumulator for ``name`` in the current prompt.""" + return self.tools.setdefault((self.prompt, name), ToolRow(self.prompt, name)) + + def release(self, turn: Turn | None) -> None: + """Price a completed API call and split it across its tools.""" + if turn is None: + return + cost = pricing.usd(turn, _BOOK) + usage = turn.usage + self.facts.rows.turns.append( + TurnRow( + turn.timestamp or self.ts, + turn.model, + cost, + turn.context, + self.prompt, + usage, + ) + ) + self.carry.turn(turn.model, turn.context) + self.detail.turn(turn, cost, self.prompt) + names = self.names.pop(turn.message_id, []) or [NO_TOOL] + for name in names: + self.tool(name).usd += cost / len(names) + + def record(self, rec: dict[str, Any]) -> None: + """Account one record.""" + stamp = records.timestamp(rec) + if stamp is not None: + self.ts = stamp + self.stamps.append(stamp) + if self.facts.cwd is None and isinstance(rec.get("cwd"), str): + self.facts.cwd = rec["cwd"] + if rec.get("type") == "user": + self.release(self.merger.flush()) + self.user(rec) + self.detail.record(rec, self.ts, self.prompt) + for block in parse.content_blocks(rec): + self.block(rec, block) + found = turns.usage_of(rec) + if found is not None: + self.release(self.merger.push(turns.build_turn(rec, *found))) + + def user(self, rec: dict[str, Any]) -> None: + """Open a new prompt when this user record is one.""" + text = blocks.prompt_text(rec) + if text is None or self.facts.parent_session is not None: + return + self.prompt += 1 + prompt_id = rec.get("promptId") + self.facts.rows.prompts.append( + PromptRow(self.prompt, self.ts, str(prompt_id or ""), text) + ) + + def block(self, rec: dict[str, Any], block: dict[str, Any]) -> None: + """Remember tool calls, pair results, keep the last reply text.""" + kind = block.get("type") + if kind == "tool_use": + self.tool_use(rec, block) + elif kind == "tool_result": + self.tool_result(rec, block) + elif kind == "text" and rec.get("type") == "assistant": + self.last_text = str(block.get("text") or "") + + def tool_use(self, rec: dict[str, Any], block: dict[str, Any]) -> None: + """Hold a call until its result arrives; note which message issued it.""" + call_id = str(block.get("id") or "") + self.pending[call_id] = block + self.use_ts[call_id] = self.ts + message = rec.get("message") + mid = message.get("id") if isinstance(message, dict) else None + name = str(block.get("name") or "unknown") + self.names.setdefault(str(mid or ""), []).append(name) + self.detail.use(rec, block, self.ts, self.prompt) + + def tool_result(self, rec: dict[str, Any], block: dict[str, Any]) -> None: + """Pair a result with its call and record what it tells us.""" + call_id = str(block.get("tool_use_id") or "") + use = self.pending.get(call_id) + paired = parse.pair_result(block, self.pending) + if use is None or paired is None: + return + self.detail.result(block, self.ts) + row = self.tool(paired.name) + row.calls += 1 + row.result_tokens += paired.result_tokens + self.carry.result(row, paired.result_tokens) + text = blocks.text_of(block.get("content")) + raw_input = use.get("input") + tool_input: dict[str, Any] = raw_input if isinstance(raw_input, dict) else {} + if paired.name in AGENT_TOOLS: + failed = bool(block.get("is_error")) + self.agent_call(rec, call_id, tool_input, failed=failed) + elif paired.name == "SendMessage": + self.facts.rows.messages.append((self.ts, str(tool_input.get("to") or ""))) + if paired.commit: + # Every commit command is kept, hash or not: most run through + # finish_auto.sh with a tailed or backgrounded output, so the hash + # is usually found later by :mod:`commits` from git log instead. + hint = blocks.repo_hint(str(tool_input.get("command")), self.facts.cwd) + started = self.use_ts.pop(call_id, self.ts) + self.facts.rows.commits += [ + Commit(self.prompt, started, self.ts, sha, hint) + for sha in blocks.commit_shas(text) or [""] + ] + table = blocks.lever_table(text) + if table is not None: + self.facts.rows.lever_tables.append((self.ts, table)) + + def agent_call( + self, + rec: dict[str, Any], + call_id: str, + tool_input: dict[str, Any], + *, + failed: bool, + ) -> None: + """Record a subagent spawn with the ``agentId`` its result names.""" + result = rec.get("toolUseResult") + result = result if isinstance(result, dict) else {} + agent_id = result.get("agentId") + status = "error" if failed else str(result.get("status") or "") + self.facts.rows.agent_calls.append( + AgentCall( + tool_use_id=call_id, + agent_id=str(agent_id or ""), + prompt_idx=self.prompt, + ts=self.use_ts.get(call_id, self.ts), + description=str(tool_input.get("description") or ""), + subagent_type=str(tool_input.get("subagent_type") or ""), + model=str(tool_input.get("model") or ""), + result_ts=self.ts, + status=status, + ) + ) + + def finish(self) -> FileFacts: + """Flush state into the facts.""" + self.release(self.merger.flush()) + facts = self.facts + facts.rows.tools = list(self.tools.values()) + self.carry.settle(_BOOK) + facts.rows.detail = self.detail.finish(_BOOK) + facts.rows.intervals = active.intervals(self.stamps) + if self.stamps: + facts.span = (min(self.stamps), max(self.stamps)) + facts.final_tag = blocks.final_tag(self.last_text) + return facts + + +def walk_file(path: Path, root: Path) -> FileFacts: + """Read one transcript (main or subagent) into :class:`FileFacts`.""" + del root # keys are derived from the path's own layout + if path.parent.name == "subagents": + parent = path.parent.parent.name + facts = FileFacts( + key=f"{parent}/{path.stem}", + session_id=parent, + parent_session=parent, + cwd=None, + meta=discover.agent_meta(path), + ) + else: + facts = FileFacts(path.stem, path.stem, None, None, {}) + walker = _Walker(facts) + with path.open(encoding="utf-8", errors="replace") as handle: + for line in handle: + rec = parse.load_line(line) + if rec is not None: + walker.record(rec) + return walker.finish()