diff --git a/python_pkg/token_audit/__init__.py b/python_pkg/token_audit/__init__.py deleted file mode 100644 index 78437f6b..00000000 --- a/python_pkg/token_audit/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Measure what Claude Code sessions actually cost, from local transcripts. - -Deterministic analysis with no model in the loop: the CLI parses -``~/.claude/projects/*/*.jsonl`` and ranks spend by project, tool, MCP server, -skill, subagent and session shape. Running the audit therefore costs nothing, -which is the point — an LLM summarising its own token usage would be the -expense it is trying to measure. - -Usage:: - - python3 -m token_audit # rolling 7 days - python3 -m token_audit --days 30 - python3 -m token_audit --json -""" - -from __future__ import annotations - -__all__ = ["__version__"] - -__version__ = "0.1.0" diff --git a/python_pkg/token_audit/__main__.py b/python_pkg/token_audit/__main__.py deleted file mode 100644 index 6f8d8b0f..00000000 --- a/python_pkg/token_audit/__main__.py +++ /dev/null @@ -1,188 +0,0 @@ -"""Command-line entry point for the token audit. - -Fails closed. If the per-axis rollup does not reconcile with the raw usage -figures the API reported, the run exits non-zero instead of printing a report -that quietly lost or double-counted sessions — a plausible-looking wrong number -is worse than no number, because it gets acted on. -""" - -from __future__ import annotations - -import argparse -from dataclasses import asdict -import json -from pathlib import Path -import sys -import time -from typing import TYPE_CHECKING - -from python_pkg.token_audit import ( - attribute, - breakdown, - discover, - history, - imagecost, - notes, - parse, - pricing, - report, - review, -) - -if TYPE_CHECKING: - from python_pkg.token_audit.model import Session - -TRANSCRIPT_ROOT = Path.home() / ".claude" / "projects" -DEFAULT_DAYS = 7 -SECONDS_PER_DAY = 86400 -# Largest acceptable relative gap between the axis rollup and the raw usage -# totals. The two are computed independently, so anything above rounding noise -# means a session was mis-attributed. -MAX_DRIFT = 0.01 -EXIT_DRIFT = 2 -EXIT_NO_DATA = 3 - - -def _build_parser() -> argparse.ArgumentParser: - """Define the CLI surface.""" - parser = argparse.ArgumentParser( - prog="token_audit", - description="Rank Claude Code token spend from local transcripts.", - ) - parser.add_argument( - "--days", - type=float, - default=DEFAULT_DAYS, - help=f"rolling window size in days (default {DEFAULT_DAYS})", - ) - parser.add_argument( - "--since", - type=float, - default=None, - help="window start as a unix timestamp (overrides --days)", - ) - parser.add_argument( - "--until", - type=float, - default=None, - help="window end as a unix timestamp (default: now)", - ) - parser.add_argument( - "--root", - type=Path, - default=TRANSCRIPT_ROOT, - help="transcript directory to scan", - ) - parser.add_argument( - "--out", - type=Path, - default=report.REPORT_DIR, - help="directory to write the report into", - ) - parser.add_argument( - "--json", - action="store_true", - help="print the machine-readable snapshot instead of the table", - ) - parser.add_argument( - "--recommend", - action="store_true", - help="append rule-based candidate levers to the report", - ) - parser.add_argument( - "--no-write", - action="store_true", - help="analyse without writing report files", - ) - return parser - - -def _collect(root: Path, since: float, until: float) -> list[Session]: - """Load and annotate every transcript inside the window.""" - sessions: list[Session] = [] - window = (since, until) - for path in discover.find_transcripts(root, since): - session = parse.load_session(path, window) - if not session.turns: - continue - # Re-walk the file for image cost: it needs the interleaved event order - # that load_session flattens into separate lists. - imagecost.annotate(session, parse.iter_events(path, window)) - sessions.append(session) - return sessions - - -def main(argv: list[str] | None = None) -> int: - """Run the audit and return a process exit code.""" - args = _build_parser().parse_args(argv) - started = time.monotonic() - until = args.until if args.until is not None else time.time() - # Default start: where the previous analysis ended, so repeated runs tile - # time without gaps or overlap. --days applies only on the very first run. - last_end = history.default_since(args.out) - if args.since is not None: - since = args.since - elif last_end is not None: - since = last_end - else: - since = until - args.days * SECONDS_PER_DAY - - sessions = _collect(args.root, since, until) - if not sessions: - sys.stderr.write(f"No transcripts with usage data under {args.root}\n") - return EXIT_NO_DATA - - totals, axes = attribute.build(sessions) - drift = attribute.reconcile(totals, sessions) - window = report.Window(since=since, until=until) - snap = report.snapshot(totals, axes, window, len(sessions)) - book = pricing.load() - priced = breakdown.build(sessions, book) - snap["v"] = review.SNAPSHOT_VERSION - snap["effective_window"] = {"since": priced.earliest, "until": priced.latest} - snap["metrics"] = breakdown.metrics(priced) | review.extra_metrics( - totals, axes, priced.days - ) - noted = notes.section(args.root, book, sessions, (since, until)) - if noted is not None: - snap["metrics"] |= noted[0] - previous = history.last_snapshot(args.out) - found = review.candidates(snap, previous) if args.recommend else None - if found is not None: - # Persisted so the dashboard reads proposed levers as data instead of - # re-parsing this run's markdown table. - snap["levers"] = [asdict(lever) for lever in found] - markdown = report.render(totals, axes, window, sessions, snap, None) - markdown += "\n".join( - [ - "", - *breakdown.render(priced), - "", - *review.render(snap, previous, levers=args.recommend, found=found), - *(noted[1] if noted is not None else []), - "", - ], - ) - - if not args.no_write: - written = report.write(markdown, snap, args.out) - history.save_snapshot(snap, args.out) - if noted is not None: - notes.write_json(noted[2], args.out) - sys.stderr.write(f"Wrote {written}\n") - - sys.stdout.write(json.dumps(snap, indent=2) if args.json else markdown) - elapsed = time.monotonic() - started - sys.stderr.write(f"Analysed {len(sessions)} sessions in {elapsed:.2f}s\n") - - if drift > MAX_DRIFT: - sys.stderr.write( - f"RECONCILIATION FAILED: axis rollup differs from raw usage by " - f"{drift * 100:.2f}% (limit {MAX_DRIFT * 100:.0f}%)\n", - ) - return EXIT_DRIFT - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_pkg/token_audit/attribute.py b/python_pkg/token_audit/attribute.py deleted file mode 100644 index 295f3b03..00000000 --- a/python_pkg/token_audit/attribute.py +++ /dev/null @@ -1,205 +0,0 @@ -"""Roll sessions up along the axes the report ranks. - -Each axis answers a different "should I change something?" question, so they -are kept separate rather than merged into one table: - -* **project** (cwd) — which work was expensive, not which tool was. -* **tool** — where result payloads come from; ranked by estimated result size. -* **mcp** — invocation counts per server, the only way to tell a registered - server apart from a *used* one. -* **skill** — how often each skill was actually invoked. -* **sidechain** — what subagents cost, which is easy to assume is large and in - practice may be zero. -* **cold start** — the fixed prefix every new session pays before doing work. - -Only :func:`totals` feeds the reconciliation gate; everything else is ranking. -""" - -from __future__ import annotations - -from collections import Counter -from dataclasses import dataclass, field -from typing import TYPE_CHECKING - -from python_pkg.token_audit.model import TOKEN_KINDS, Session - -if TYPE_CHECKING: - from collections.abc import Iterable - -# A session past this many assistant turns is "long". Chosen from measurement: -# in the observed window, sessions above it were ~90% of all weighted spend, -# because per-turn cost scales with accumulated context. -LONG_SESSION_TURNS = 400 - - -@dataclass -class Totals: - """Raw token counts summed straight from the API's usage blocks.""" - - by_kind: Counter[str] = field(default_factory=Counter) - weighted: float = 0.0 - turns: int = 0 - - def add(self, session: Session) -> None: - """Accumulate one session's usage.""" - for turn in session.turns: - for kind in TOKEN_KINDS: - self.by_kind[kind] += turn.usage.get(kind, 0) - self.weighted += turn.cost - self.turns += 1 - - -@dataclass -class Tally: - """A running "how much, over how many" pair. - - Every headline driver is reported the same way — a cost or token total plus - the number of things that produced it — so they share one type instead of - two loose fields each. - """ - - total: float = 0.0 - count: int = 0 - - def add(self, amount: float) -> None: - """Record one more contribution.""" - self.total += amount - self.count += 1 - - -@dataclass -class Batching: - """How often independent tool calls shared one API message. - - An unbatched call re-sends the whole conversation to earn a single result, - so this is a direct measure of avoidable turns rather than a style metric. - """ - - messages: int = 0 - batched: int = 0 - calls: int = 0 - - @property - def share(self) -> float: - """Fraction of tool messages that carried more than one call.""" - return self.batched / self.messages if self.messages else 0.0 - - @property - def per_message(self) -> float: - """Mean tool calls per API message; 1.0 means nothing was ever batched.""" - return self.calls / self.messages if self.messages else 0.0 - - -BIG_RESULT = 2000 - - -@dataclass -class ToolPayloads: - """Tool-result tokens and call counts per tool name.""" - - tokens: Counter[str] = field(default_factory=Counter) - calls: Counter[str] = field(default_factory=Counter) - # Tokens from results over BIG_RESULT: the part a range/head would have - # cut (whole-file cat/sed dumps). - big_tokens: Counter[str] = field(default_factory=Counter) - - -@dataclass -class Axes: - """Every ranking the report needs, in one pass over the sessions.""" - - project: Counter[str] = field(default_factory=Counter) - tools: ToolPayloads = field(default_factory=ToolPayloads) - mcp_calls: Counter[str] = field(default_factory=Counter) - skills: Counter[str] = field(default_factory=Counter) - models: Counter[str] = field(default_factory=Counter) - sidechain: Tally = field(default_factory=Tally) - cold_start: Tally = field(default_factory=Tally) - images: Tally = field(default_factory=Tally) - long_sessions: Tally = field(default_factory=Tally) - # Cost-weighted, compaction-aware image re-send estimate. Tracked apart from - # ``images`` (which counts tokens read once) because the two answer - # different questions and only this one is comparable to session cost. - image_cost: float = 0.0 - batching: Batching = field(default_factory=Batching) - - -def build(sessions: Iterable[Session]) -> tuple[Totals, Axes]: - """Compute reconciled totals and every ranking axis together.""" - totals = Totals() - axes = Axes() - for session in sessions: - totals.add(session) - _add_turns(axes, session) - _add_tools(axes, session) - _add_session_shape(axes, session) - return totals, axes - - -def _add_turns(axes: Axes, session: Session) -> None: - """Attribute per-turn cost to project, model and sidechain.""" - axes.project[session.cwd or "unknown"] += int(session.cost) - for turn in session.turns: - axes.models[turn.model] += turn.usage.get("output_tokens", 0) - if turn.is_sidechain: - axes.sidechain.add(turn.cost) - _add_batching(axes, session) - - -def _add_batching(axes: Axes, session: Session) -> None: - """Count how many API messages carried more than one tool call. - - Grouped by ``message_id`` because the transcript splits one API message - across several records -- one per ``tool_use`` block. Counting records - instead reports every message as carrying exactly one call, which is how a - real 2.8% batching rate reads as 0.0%. - """ - per_message: Counter[str] = Counter() - for turn in session.turns: - if turn.tool_calls and turn.message_id: - per_message[turn.message_id] += turn.tool_calls - for calls in per_message.values(): - axes.batching.messages += 1 - axes.batching.calls += calls - if calls > 1: - axes.batching.batched += 1 - - -def _add_tools(axes: Axes, session: Session) -> None: - """Attribute tool-result payloads to tools, MCP servers and skills.""" - for call in session.tools: - axes.tools.tokens[call.name] += call.result_tokens - axes.tools.calls[call.name] += 1 - if call.result_tokens > BIG_RESULT: - axes.tools.big_tokens[call.name] += call.result_tokens - server = call.mcp_server - if server is not None: - axes.mcp_calls[server] += 1 - if call.skill is not None: - axes.skills[call.skill] += 1 - if call.is_image: - axes.images.add(call.result_tokens) - - -def _add_session_shape(axes: Axes, session: Session) -> None: - """Attribute whole-session properties: cold start, length, image cost.""" - cold = session.cold_start_tokens - if cold: - axes.cold_start.add(cold) - axes.image_cost += session.image_cost - if session.turn_count > LONG_SESSION_TURNS: - axes.long_sessions.add(session.cost) - - -def reconcile(totals: Totals, sessions: Iterable[Session]) -> float: - """Return the relative drift between axis sums and the raw usage totals. - - The report is only worth acting on if its parts add up to its whole, so the - CLI turns this into a hard gate rather than a warning. Comparing the - per-project rollup against the independently summed usage blocks catches - any session that was counted twice or dropped. - """ - summed = sum(session.cost for session in sessions) - if totals.weighted == 0: - return 0.0 - return abs(summed - totals.weighted) / totals.weighted diff --git a/python_pkg/token_audit/breakdown.py b/python_pkg/token_audit/breakdown.py deleted file mode 100644 index f37aeb3e..00000000 --- a/python_pkg/token_audit/breakdown.py +++ /dev/null @@ -1,242 +0,0 @@ -"""Dollar-denominated metrics: per model, per subagent, per call position. - -:mod:`attribute` answers "where did the weighted tokens go"; this module puts -list-price dollars on the same sessions so that model choice becomes visible, -and computes the flat ``metrics`` dict every snapshot stores. Ledger entries -name a key in that dict, so a key here is a contract -- rename one and old -ledger lines silently become "unmeasured". -""" - -from __future__ import annotations - -from collections import Counter -from dataclasses import dataclass, field, replace -from pathlib import Path -from typing import TYPE_CHECKING - -from python_pkg.token_audit.pricing import usd - -if TYPE_CHECKING: - from collections.abc import Sequence - - from python_pkg.token_audit.model import Session - from python_pkg.token_audit.pricing import PriceBook - -SECONDS_PER_DAY = 86400 -# Context size above which a call is "over budget" -- the session gate blocks -# here. Measured 2026-10-04: per-call cost 21k weighted for calls 1-100 versus -# 51k for calls 201-400, driven by context size, not by call count. -CONTEXT_BUDGET = 300_000 -# Model a mechanical subagent would move to; used only to size that lever. -CHEAPER_MODEL = "claude-sonnet-5-5" -# Call-position buckets for the cost curve (upper bounds, inclusive). -BUCKETS: tuple[int, ...] = (100, 200, 400, 600, 1000) - - -@dataclass -class Tally: - """Dollars and API calls accumulated along one axis.""" - - usd: float = 0.0 - calls: int = 0 - - def add(self, cost: float) -> None: - """Count one call costing ``cost`` dollars.""" - self.usd += cost - self.calls += 1 - - -@dataclass -class Span: - """Earliest and latest record timestamp seen (0.0 means none yet).""" - - earliest: float = 0.0 - latest: float = 0.0 - - -@dataclass -class Breakdown: - """Everything priced in one pass over the sessions.""" - - main: Tally = field(default_factory=Tally) - sub: Tally = field(default_factory=Tally) - over_budget: Tally = field(default_factory=Tally) - home_usd: float = 0.0 - by_model: Counter[str] = field(default_factory=Counter) - by_agent_type: Counter[str] = field(default_factory=Counter) - by_description: Counter[str] = field(default_factory=Counter) - sub_opus_usd: float = 0.0 - # The same Opus subagent turns re-priced at the reference cheaper model: - # what routing them there would have cost, for sizing that lever exactly. - sub_opus_repriced_usd: float = 0.0 - buckets: dict[str, Tally] = field(default_factory=dict) - commits: int = 0 - span: Span = field(default_factory=Span) - - @property - def usd_main(self) -> float: - """Main-conversation spend.""" - return self.main.usd - - @property - def usd_sub(self) -> float: - """Subagent spend.""" - return self.sub.usd - - @property - def calls_main(self) -> int: - """Main-conversation API calls.""" - return self.main.calls - - @property - def calls_sub(self) -> int: - """Subagent API calls.""" - return self.sub.calls - - @property - def over_budget_calls(self) -> int: - """Calls whose context exceeded :data:`CONTEXT_BUDGET`.""" - return self.over_budget.calls - - @property - def over_budget_usd(self) -> float: - """Spend carried by calls over the context budget.""" - return self.over_budget.usd - - @property - def earliest(self) -> float: - """Earliest record timestamp seen.""" - return self.span.earliest - - @property - def latest(self) -> float: - """Latest record timestamp seen.""" - return self.span.latest - - @property - def usd_total(self) -> float: - """Main plus subagent spend.""" - return self.usd_main + self.usd_sub - - @property - def calls(self) -> int: - """Real API calls, main plus subagent.""" - return self.calls_main + self.calls_sub - - @property - def days(self) -> float: - """Span actually covered by records, at least a minute's worth.""" - return max(self.latest - self.earliest, 60.0) / SECONDS_PER_DAY - - -def _bucket(index: int) -> str: - """Label the call-position bucket a 1-based call index falls in.""" - low = 1 - for high in BUCKETS: - if index <= high: - return f"{low}-{high}" - low = high + 1 - return f"{low}+" - - -def _stamp(found: Breakdown, stamp: float) -> None: - """Widen the observed record span to include ``stamp``.""" - if not stamp: - return - span = found.span - span.earliest = min(span.earliest or stamp, stamp) - span.latest = max(span.latest, stamp) - - -def build(sessions: Sequence[Session], book: PriceBook) -> Breakdown: - """Price every turn and roll it up along each axis.""" - found = Breakdown() - home = str(Path.home()) - for session in sessions: - is_sub = session.parent_id is not None - found.commits += sum(1 for call in session.tools if call.commit) - for index, turn in enumerate(session.turns, start=1): - cost = usd(turn, book) - _stamp(found, turn.timestamp) - found.by_model[turn.model] += cost - if turn.context > CONTEXT_BUDGET: - found.over_budget.add(cost) - if session.cwd == home: - found.home_usd += cost - if is_sub: - found.sub.add(cost) - found.by_agent_type[session.agent_type or "unknown"] += cost - found.by_description[session.description or "?"] += cost - if "opus" in turn.model: - found.sub_opus_usd += cost - cheap = replace(turn, model=CHEAPER_MODEL) - found.sub_opus_repriced_usd += usd(cheap, book) - else: - found.main.add(cost) - found.buckets.setdefault(_bucket(index), Tally()).add(cost) - return found - - -def _share(part: float, whole: float) -> float: - """``part / whole`` rounded for storage, 0 when ``whole`` is 0.""" - return round(part / whole, 4) if whole else 0.0 - - -def metrics(found: Breakdown) -> dict[str, float]: - """The flat metric dict snapshots store and ledger entries reference.""" - total = found.usd_total - opus_main = sum(c for m, c in found.by_model.items() if "opus" in m) - return { - "usd_total": round(total, 2), - "usd_per_day": round(total / found.days, 2), - "usd_per_call": round(total / found.calls, 5) if found.calls else 0.0, - "calls": found.calls, - "calls_per_day": round(found.calls / found.days, 1), - "commits": found.commits, - "usd_per_commit": round(total / found.commits, 2) if found.commits else 0.0, - "subagent_usd_share": _share(found.usd_sub, total), - "subagent_opus_usd_share": _share(found.sub_opus_usd, found.usd_sub), - "opus_usd_share": _share(opus_main, total), - "home_cwd_usd_share": _share(found.home_usd, total), - "over_budget_call_share": _share(found.over_budget_calls, found.calls), - "over_budget_usd_share": _share(found.over_budget_usd, total), - "subagent_sonnet_saving_usd": round( - found.sub_opus_usd - found.sub_opus_repriced_usd, 2 - ), - } - - -def _money(rows: list[tuple[str, float]], total: float) -> list[str]: - """Render ``(label, usd)`` rows as a markdown table with shares.""" - out = ["| item | USD | share |", "|---|---|---|"] - out += [f"| {k} | ${v:,.2f} | {_share(v, total) * 100:.1f}% |" for k, v in rows] - return out - - -def render(found: Breakdown, top: int = 12) -> list[str]: - """Markdown sections for the dollar view.""" - total = found.usd_total - lines = [ - "## Dollar view (list prices, deduplicated per API message)", - "", - f"**${total:,.2f}** over {found.days:.1f} days of records " - f"({found.calls:,} API calls; main ${found.usd_main:,.2f}, " - f"subagents ${found.usd_sub:,.2f}).", - f"Calls with context over {CONTEXT_BUDGET:,} tokens: " - f"{found.over_budget_calls:,} carrying " - f"{_share(found.over_budget_usd, total) * 100:.1f}% of spend.", - "", - "### By model", - "", - ] - lines += _money(found.by_model.most_common(top), total) - lines += ["", "### Subagents by type", ""] - lines += _money(found.by_agent_type.most_common(top), total) - lines += ["", "### Costliest subagent tasks", ""] - lines += _money(found.by_description.most_common(top), total) - lines += ["", "### Cost per call by position in session (main only)", ""] - lines += ["| calls | USD per call | calls |", "|---|---|---|"] - for label in sorted(found.buckets, key=lambda b: int(b.split("-")[0].rstrip("+"))): - tally = found.buckets[label] - lines.append(f"| {label} | ${tally.usd / tally.calls:.4f} | {tally.calls:,} |") - return lines diff --git a/python_pkg/token_audit/checkup.py b/python_pkg/token_audit/checkup.py deleted file mode 100644 index 1aa04a2c..00000000 --- a/python_pkg/token_audit/checkup.py +++ /dev/null @@ -1,71 +0,0 @@ -"""The weekly standing-cost checkup. - -Complements the spend report. That report answers "where did the tokens go"; -this answers "what is still being paid for every turn regardless of use", which -is the question earlier audits kept answering by memory and kept getting wrong. - -Run as ``python3 -m python_pkg.token_audit.checkup``. -""" - -from __future__ import annotations - -import sys - -from python_pkg.token_audit import surfaces, unused -from python_pkg.token_audit.probe import weighted_pct - - -def _fmt_usage(rows: list[unused.Usage]) -> list[str]: - """Render call counts, marking only rows dead in BOTH windows.""" - header = ( - f"| name | {unused.SHORT_WINDOW_DAYS}d | {unused.LONG_WINDOW_DAYS}d | verdict |" - ) - lines = [header, "|---|---|---|---|"] - for row in rows: - verdict = "**park**" if row.dead else "keep" - lines.append( - f"| {row.name} | {row.short_calls} | {row.long_calls} | {verdict} |" - ) - return lines - - -def build(turns: int, weighted_total: int) -> str: - """Render the checkup as markdown.""" - out = ["# Standing-cost checkup", ""] - out.append( - "Every row below is billed on each of " - f"{turns:,} turns. Shares are weighted (cache-read weight 0.1)." - ) - out.append("") - out.append("## Surfaces") - out.append("| surface | est tokens | share | detail |") - out.append("|---|---|---|---|") - for surface in surfaces.collect(): - share = weighted_pct(surface.est_tokens, turns, weighted_total) - est = f"{surface.est_tokens:,}" if surface.est_tokens else "A/B only" - out.append(f"| {surface.name} | {est} | {share:.2f}% | {surface.detail} |") - out += ["", "## MCP servers", ""] - out += _fmt_usage(unused.mcp_usage()) - out += ["", "## Skills", ""] - out += _fmt_usage(unused.skill_usage()) - out += [ - "", - "_A surface marked `A/B only` cannot be sized from disk; measure it", - "with `probe.measure()`, which runs non-strict so project `.mcp.json`", - "files are included._", - "", - ] - return "\n".join(out) - - -def main(argv: list[str] | None = None) -> int: - """Print the checkup, taking turns and weighted total from the last report.""" - argv = sys.argv[1:] if argv is None else argv - turns = int(argv[0]) if argv else 44311 - total = int(argv[1]) if len(argv) > 1 else 1230346533 - sys.stdout.write(build(turns, total)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/python_pkg/token_audit/discover.py b/python_pkg/token_audit/discover.py deleted file mode 100644 index 01c52ad9..00000000 --- a/python_pkg/token_audit/discover.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Find transcript files and read the per-file metadata around them. - -Two layouts hold billed turns: - -* ``//.jsonl`` -- the main conversation. -* ``///subagents/agent-.jsonl`` -- one per - ``Agent`` call, with a sibling ``agent-.meta.json`` naming the agentType - and description. Before 2026-10-04 the audit globbed only the first layout - and reported subagents at 0.0% while they were 35% of real spend. - -A background agent outlives ``/clear``: the rest of its transcript lands in the -*new* session's ``subagents/`` under the same ``agent-`` name, starting -mid-conversation and with no ``.meta.json`` (or one without a description). -:func:`agent_meta` borrows the label from the spawning session's copy; on -2026-10-10 that was all 14 unlabelled files, 5.2% of a week's spend. -""" - -from __future__ import annotations - -import json -import re -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from pathlib import Path - -PATTERNS: tuple[str, ...] = ("*/*.jsonl", "*/*/subagents/agent-*.jsonl") - -# Harness-injected ``...`` blocks (````, -# ````): identical across agents, so never a useful label. -WRAPPER = re.compile(r"<([\w-]+)[^>]*>.*?", re.DOTALL) - - -def find_transcripts(root: Path, since: float) -> list[Path]: - """Return transcripts that may hold records after ``since``, newest first. - - A file last written before ``since`` cannot hold an in-window record, so - mtime prunes those. There is deliberately no mtime upper bound: a file - still being appended to after the window closed holds in-window records - too, and the per-record timestamp filter in :func:`parse.iter_events` - trims whatever falls outside. - """ - found: list[tuple[float, Path]] = [] - for pattern in PATTERNS: - for path in sorted(root.glob(pattern)): - mtime = path.stat().st_mtime - if mtime >= since: - found.append((mtime, path)) - return [path for _, path in sorted(found, reverse=True)] - - -def _record(line: str) -> dict[str, Any] | None: - """Parse one JSON object, or ``None``.""" - try: - loaded = json.loads(line) - except ValueError: - return None - return loaded if isinstance(loaded, dict) else None - - -def first_cwd(path: Path) -> str | None: - """Return the working directory a transcript was recorded in.""" - with path.open(encoding="utf-8", errors="replace") as handle: - for line in handle: - record = _record(line) - if record is not None: - cwd = record.get("cwd") - if isinstance(cwd, str): - return cwd - return None - - -def _meta_file(meta: Path) -> dict[str, Any]: - """Read one ``.meta.json``, ``{}`` when missing or unreadable.""" - if not meta.exists(): - return {} - return _record(meta.read_text(encoding="utf-8", errors="replace")) or {} - - -def agent_meta(path: Path) -> dict[str, Any]: - """Return a subagent transcript's metadata, or ``{}``. - - The sibling ``.meta.json`` wins; any field it lacks is filled from the same - agent's meta in another session directory (see the module docstring). - """ - own = _meta_file(path.with_suffix(".meta.json")) - if own.get("description"): - return own - projects = path.parents[3] - for other in sorted(projects.glob(f"*/*/subagents/{path.stem}.meta.json")): - borrowed = _meta_file(other) - if borrowed.get("description"): - return {**borrowed, **{k: v for k, v in own.items() if v}} - return own - - -def _prompt_text(record: dict[str, Any]) -> str | None: - """The typed text of a ``user`` record, ignoring tool results.""" - if record.get("type") != "user": - return None - message = record.get("message") - content = message.get("content") if isinstance(message, dict) else None - if isinstance(content, str): - return content - if isinstance(content, list): - for block in content: - if isinstance(block, dict) and block.get("type") == "text": - return str(block.get("text") or "") - return None - - -def first_prompt(path: Path, limit: int = 60) -> str: - """First non-blank line of a transcript's first user prompt, truncated. - - The label of last resort for a subagent nobody described: better than - ``?`` because it names the work, and an empty result means none exists. - """ - with path.open(encoding="utf-8", errors="replace") as handle: - for line in handle: - record = _record(line) - text = _prompt_text(record) if record is not None else None - text = WRAPPER.sub("", text or "") - lines = [part.strip() for part in text.splitlines()] - first = next((part for part in lines if part), "") - if first: - return first if len(first) <= limit else first[: limit - 1] + "…" - return "" diff --git a/python_pkg/token_audit/history.py b/python_pkg/token_audit/history.py deleted file mode 100644 index eb633d9b..00000000 --- a/python_pkg/token_audit/history.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Dated snapshots and the ledger of applied changes. - -Until 2026-10-04 every run overwrote one ``weekly.json``, so "did last month's -change help?" had no data to answer it -- and Claude Code deletes transcripts -after ~30 days, so the raw data cannot be re-read later either. Two durable -files fix that: - -* ``history/YYYY-MM-DD[-HHMM].json`` -- one snapshot per run, never rewritten. - The newest one's window end is the next run's default start, which is how - ``/token-optimization`` "figures out when the last analysis was". -* ``ledger.jsonl`` -- one line per applied lever: which snapshot metric it was - meant to move, the baseline value, the predicted value, and how to undo it. - :func:`score` turns each line into helped / no effect / regressed against the - latest snapshot, in code, so the verdict is never an LLM's impression. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from datetime import UTC, datetime -import json -from pathlib import Path -from typing import Any - -from python_pkg.token_audit.probe import PREDICTION_TOLERANCE - -REPORT_DIR = Path.home() / ".claude" / "token-report" -HISTORY_NAME = "history" -LEDGER_NAME = "ledger.jsonl" -# A metric must move by more than this fraction of its baseline to count as a -# change at all; smaller moves are within week-to-week noise. -NOISE = 0.03 - - -def _root(directory: Path | None) -> Path: - """Resolve the report directory at call time so it can be redirected.""" - return REPORT_DIR if directory is None else directory - - -def save_snapshot(snap: dict[str, Any], directory: Path | None = None) -> Path: - """Write ``snap`` under ``history/`` without ever overwriting a file.""" - folder = _root(directory) / HISTORY_NAME - folder.mkdir(parents=True, exist_ok=True) - now = datetime.now(tz=UTC) - path = folder / f"{now:%Y-%m-%d}.json" - if path.exists(): - path = folder / f"{now:%Y-%m-%d-%H%M%S}.json" - path.write_text(json.dumps(snap, indent=2), encoding="utf-8") - return path - - -def last_snapshot(directory: Path | None = None) -> dict[str, Any] | None: - """Return the newest readable snapshot, or ``None`` if there is none.""" - folder = _root(directory) / HISTORY_NAME - if not folder.is_dir(): - return None - for path in sorted(folder.glob("*.json"), reverse=True): - try: - loaded = json.loads(path.read_text(encoding="utf-8")) - except ValueError: - continue - if isinstance(loaded, dict): - return loaded - return None - - -def default_since(directory: Path | None = None) -> float | None: - """The previous analysis's window end, i.e. where this one should start.""" - snap = last_snapshot(directory) - if snap is None: - return None - until = snap.get("window", {}).get("until") - return float(until) if isinstance(until, (int, float)) else None - - -def read_ledger(directory: Path | None = None) -> list[dict[str, Any]]: - """Return every well-formed ledger entry, oldest first.""" - path = _root(directory) / LEDGER_NAME - if not path.exists(): - return [] - entries: list[dict[str, Any]] = [] - for line in path.read_text(encoding="utf-8").splitlines(): - try: - entry = json.loads(line) - except ValueError: - continue - if isinstance(entry, dict) and "metric" in entry and "baseline" in entry: - entries.append(entry) - return entries - - -def append_ledger(entry: dict[str, Any], directory: Path | None = None) -> None: - """Append one applied lever; ``date`` is filled in when missing.""" - directory = _root(directory) - directory.mkdir(parents=True, exist_ok=True) - record = {"date": datetime.now(tz=UTC).strftime("%Y-%m-%d"), **entry} - with (directory / LEDGER_NAME).open("a", encoding="utf-8") as handle: - handle.write(json.dumps(record) + "\n") - - -@dataclass(frozen=True) -class Verdict: - """How one applied lever turned out against the latest snapshot.""" - - lever: str - metric: str - baseline: float - predicted: float | None - actual: float | None - outcome: str - note: str = "" - - -def _outcome(baseline: float, actual: float, *, lower_is_better: bool) -> str: - """Classify a metric move as helped / no effect / regressed.""" - if not baseline: - return "no baseline" - change = (actual - baseline) / abs(baseline) - if abs(change) <= NOISE: - return "no effect" - improved = change < 0 if lower_is_better else change > 0 - return "helped" if improved else "regressed" - - -def _prediction_note(baseline: float, predicted: float | None, actual: float) -> str: - """Flag a prediction that missed the measured move by more than tolerance.""" - if predicted is None: - return "" - want = baseline - predicted - got = baseline - actual - if not want: - return "" - drift = abs(got - want) / abs(want) - if drift <= PREDICTION_TOLERANCE: - return "prediction held" - return f"prediction off by {drift:.0%}" - - -def score(entries: list[dict[str, Any]], snap: dict[str, Any]) -> list[Verdict]: - """Score each ledger entry against the snapshot's current metric values.""" - metrics = snap.get("metrics", {}) - verdicts: list[Verdict] = [] - for entry in entries: - metric = str(entry["metric"]) - baseline = float(entry["baseline"]) - raw_predicted = entry.get("predicted") - predicted = float(raw_predicted) if raw_predicted is not None else None - value = metrics.get(metric) - if not isinstance(value, (int, float)): - verdicts.append( - Verdict( - str(entry.get("lever", "?")), - metric, - baseline, - predicted, - None, - "unmeasured", - "metric absent from snapshot", - ), - ) - continue - actual = float(value) - lower = bool(entry.get("lower_is_better", True)) - verdicts.append( - Verdict( - lever=str(entry.get("lever", "?")), - metric=metric, - baseline=baseline, - predicted=predicted, - actual=actual, - outcome=_outcome(baseline, actual, lower_is_better=lower), - note=_prediction_note(baseline, predicted, actual), - ), - ) - return verdicts diff --git a/python_pkg/token_audit/imagecost.py b/python_pkg/token_audit/imagecost.py deleted file mode 100644 index e3ab91fb..00000000 --- a/python_pkg/token_audit/imagecost.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Estimate what images cost *after* the turn that loaded them. - -An image does not leave the context once read. Every subsequent request in the -session re-sends it as cache-read, so a screenshot loaded at turn 20 of a -1000-turn session is paid for ~980 more times. At the cache-read weight of 0.1 -that turns ~2.5k tokens into ~250k, which is why images are worth tracking as -their own axis rather than lumping them in with text tool results. - -Measured over a real 7-day window this amplification was ~1.6% of weighted -spend — real, but an order of magnitude below session length. An earlier -version of this module reported ~27%, because it sized images by their base64 -payload length; see :func:`parse._result_tokens` for why that is ~20x too high. -The lesson is kept here deliberately: an estimate that is never checked against -billed tokens can invent a headline finding out of nothing. - -Two honest limits on the number this produces: - -* It is an **estimate**, derived from result sizes rather than billed tokens. - It is deliberately never folded into the reconciled totals, which come only - from the API's own usage figures. -* Compaction discards history. An image loaded before a compaction stops being - re-sent, so charging it forever would roughly double the figure and yields - the impossible result of sessions costing more than they measurably did. - Compaction is detected from a sharp drop in context size. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from python_pkg.token_audit.model import ( - COMPACTION_DROP_RATIO, - WEIGHTS, - Session, - ToolCall, - Turn, -) - -if TYPE_CHECKING: - from collections.abc import Iterable - -# Images are re-sent from cache, so each later turn charges them at cache-read. -_REREAD_WEIGHT = WEIGHTS["cache_read_input_tokens"] - - -def _is_compaction(context: int, previous: int | None) -> bool: - """Whether this turn's context implies history was just discarded. - - Context grows monotonically as a session accumulates messages, so the only - way it falls sharply is compaction (or a fresh branch). A ratio test rather - than an absolute threshold keeps this valid for both small and huge - sessions. - """ - return previous is not None and context < previous * COMPACTION_DROP_RATIO - - -def image_cost(events: Iterable[tuple[str, object]]) -> float: - """Return cost-weighted tokens spent re-sending images across turns. - - ``events`` must preserve transcript order — the whole calculation is about - how many turns followed each image before compaction cleared it. - """ - live: list[int] = [] - previous: int | None = None - total = 0.0 - for kind, event in events: - if kind == "tool": - if isinstance(event, ToolCall) and event.is_image: - live.append(event.result_tokens) - elif isinstance(event, Turn): - if _is_compaction(event.context, previous): - live.clear() - # Charge every still-live image once for this turn: it was part of - # the context this request re-sent. - total += sum(live) * _REREAD_WEIGHT - previous = event.context - return total - - -def annotate(session: Session, events: Iterable[tuple[str, object]]) -> Session: - """Attach the image amplification estimate to a session.""" - session.image_cost = image_cost(events) - return session diff --git a/python_pkg/token_audit/model.py b/python_pkg/token_audit/model.py deleted file mode 100644 index 4bb28132..00000000 --- a/python_pkg/token_audit/model.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Types and cost weights shared by every stage of the audit. - -The whole package exists to answer one question — *what did last week actually -cost?* — and the answer depends entirely on how the four token classes are -weighted. Ranking by raw token count is actively misleading here: in a measured -7-day window cache reads were 99% of raw volume but only ~82% of weighted cost, -while output was 1.5% of volume and ~7.5% of cost. A report that ranked by raw -counts would point at the wrong culprits, so the weights live here as the single -source of truth rather than being re-derived per module. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field - -# Relative price of each token class, normalised so plain input == 1.0. These -# mirror Anthropic's published multipliers (cache write 1.25x, cache read 0.1x, -# output 5x); they are ratios, not currency, so the report stays valid whatever -# the absolute per-token price happens to be. -WEIGHTS: dict[str, float] = { - "input_tokens": 1.0, - "cache_creation_input_tokens": 1.25, - "cache_read_input_tokens": 0.1, - "output_tokens": 5.0, -} - -# Token classes in the order they should appear in reports. -TOKEN_KINDS: tuple[str, ...] = ( - "input_tokens", - "cache_creation_input_tokens", - "cache_read_input_tokens", - "output_tokens", -) - -# A turn whose context is below this fraction of the previous turn's is treated -# as sitting after a compaction boundary. Compaction discards earlier messages, -# so images loaded before it stop being re-sent and must stop being charged. -# Without this the image estimate roughly doubles and produces the nonsense -# result of single sessions costing >100% of their own measured spend. -COMPACTION_DROP_RATIO = 0.7 - -# Extensions whose Read results arrive as images rather than text. -IMAGE_SUFFIXES: frozenset[str] = frozenset( - {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp"}, -) - - -def weighted(usage: dict[str, int]) -> float: - """Return the cost-weighted token total for one request's usage block.""" - return sum(WEIGHTS[kind] * usage.get(kind, 0) for kind in TOKEN_KINDS) - - -@dataclass(frozen=True) -class Turn: - """One assistant response, with the token accounting the API reported. - - ``context`` is what the request actually re-sent (cache read + cache - creation). It is the quantity that grows as a session accumulates history, - and the quantity that collapses at a compaction boundary, so it drives both - the session-length analysis and the image-amplification model. - - ``message_id`` is the API message id. One API message can be written to the - transcript as SEVERAL JSONL records -- one per ``tool_use`` block -- all - sharing the id. Counting records therefore overstates turns and makes every - turn look like it carried exactly one tool call. Grouping by this id is what - makes the batching ratio come out right (measured 2026-08-28: 2.8% batched, - not the 0.0% a per-record count reports). - """ - - usage: dict[str, int] - context: int - model: str - is_sidechain: bool - message_id: str = "" - tool_calls: int = 0 - timestamp: float = 0.0 - - @property - def cost(self) -> float: - """Cost-weighted tokens attributable to this turn.""" - return weighted(self.usage) - - -@dataclass(frozen=True) -class ToolCall: - """A tool invocation paired with the size of the result it returned. - - ``result_tokens`` is an estimate: transcripts store the rendered result, so - the character count divided by four is the best available proxy. It is used - only for ranking tools against each other, never added to the reconciled - totals, which come exclusively from the API's own usage numbers. - """ - - name: str - result_tokens: int - path: str | None = None - skill: str | None = None - # A Bash call that ran ``git commit`` or ``finish_auto.sh``: the closest - # transcript-visible proxy for "an issue got solved", used for USD/commit. - commit: bool = False - - @property - def is_image(self) -> bool: - """Whether this call returned an image rather than text.""" - if self.path is None: - return False - dot = self.path.rfind(".") - return dot != -1 and self.path[dot:].lower() in IMAGE_SUFFIXES - - @property - def mcp_server(self) -> str | None: - """The MCP server this tool belongs to, or ``None`` for builtins.""" - parts = self.name.split("__") - expected_parts = 3 - return parts[1] if len(parts) >= expected_parts and parts[0] == "mcp" else None - - -@dataclass -class Session: - """Everything one transcript file contributes to the audit.""" - - session_id: str - path: str - cwd: str | None = None - turns: list[Turn] = field(default_factory=list) - tools: list[ToolCall] = field(default_factory=list) - # Set only for subagent transcripts: the spawning session and the - # agentType / description from the sibling ``.meta.json``. - parent_id: str | None = None - agent_type: str | None = None - description: str | None = None - # Cost-weighted tokens spent re-sending images that were still live in - # context. Computed by :mod:`imagecost`, which needs the interleaved order - # of images and turns and so cannot be derived from these lists alone. - image_cost: float = 0.0 - - @property - def cost(self) -> float: - """Cost-weighted tokens for the whole session.""" - return sum(turn.cost for turn in self.turns) - - @property - def turn_count(self) -> int: - """Number of assistant responses in the session.""" - return len(self.turns) - - @property - def max_context(self) -> int: - """Largest context this session ever re-sent in a single request.""" - return max((turn.context for turn in self.turns), default=0) - - @property - def cold_start_tokens(self) -> int: - """Tokens written to cache before any cache existed to read from. - - The first request of a session pays to cache the whole fixed prefix — - system prompt, tool schemas, CLAUDE.md, rules and memories. Isolating it - shows what every new session costs before it does any work. - """ - return sum( - turn.usage.get("cache_creation_input_tokens", 0) - for turn in self.turns - if turn.usage.get("cache_read_input_tokens", 0) == 0 - ) diff --git a/python_pkg/token_audit/notes.py b/python_pkg/token_audit/notes.py deleted file mode 100644 index fb29c608..00000000 --- a/python_pkg/token_audit/notes.py +++ /dev/null @@ -1,198 +0,0 @@ -"""USD per completed todo note: per-note rows, window metrics and outputs. - -``usd_per_note`` = lifetime linked USD of notes done or abandoned in the window -over the notes done in the window that have a linked session (lower is -better). ``note_linked_usd_share`` is the share of window spend linked to any -note: how far to trust the first number. -""" - -from __future__ import annotations - -import argparse -from dataclasses import asdict, dataclass -from datetime import UTC, datetime -import json -from pathlib import Path -import sys -import time -from typing import TYPE_CHECKING - -from python_pkg.token_audit import history, notes_collect, pricing -from python_pkg.token_audit.notes_backlog import TERMINAL - -if TYPE_CHECKING: - from collections.abc import Mapping, Sequence - - from python_pkg.token_audit.model import Session - from python_pkg.token_audit.notes_backlog import Note - from python_pkg.token_audit.notes_cost import Spend - from python_pkg.token_audit.notes_link import Link - -JSON_NAME = "notes.json" -TOP = 15 -TITLE_CHARS = 60 - - -@dataclass(frozen=True) -class NoteRow: - """One note with the spend of every session linked to it.""" - - id: str - title: str - status: str - priority: str - done_at: float | None - sessions: list[str] - usd: float - tokens: int - - -def backlog_path_for(home: Path) -> Path: - """BACKLOG.md under a given home directory.""" - return home / "src" / "todo" / "BACKLOG.md" - - -def note_rows( - notes: list[Note], links: Mapping[str, Link], spend: Mapping[str, Spend] -) -> list[NoteRow]: - """Notes with >=1 linked session; a session's spend splits equally.""" - usd: dict[str, float] = {} - toks: dict[str, float] = {} - sessions: dict[str, list[str]] = {} - for sid in sorted(links): - share = len(links[sid].notes) - cost = spend.get(sid) - for note_id in links[sid].notes: - sessions.setdefault(note_id, []).append(sid) - usd[note_id] = usd.get(note_id, 0.0) + (cost.usd / share if cost else 0.0) - toks[note_id] = toks.get(note_id, 0) + (cost.tokens / share if cost else 0) - rows = [ - NoteRow( - n.id, - n.title, - n.status, - n.priority, - n.done_at, - sessions[n.id], - round(usd[n.id], 4), - round(toks[n.id]), - ) - for n in notes - if n.id in sessions - ] - return sorted(rows, key=lambda r: -r.usd) - - -def _in(stamp: float | None, window: tuple[float, float]) -> bool: - return stamp is not None and window[0] <= stamp <= window[1] - - -def metrics( - notes: list[Note], - rows: list[NoteRow], - window: tuple[float, float], - window_usd: Mapping[str, float], - linked: set[str], -) -> dict[str, float]: - """The snapshot metrics; ``usd_per_note`` is absent when unmeasurable.""" - done = {n.id for n in notes if n.status == "done" and _in(n.done_at, window)} - closed = {n.id for n in notes if n.status == "abandoned" and _in(n.done_at, window)} - linked_done = [r for r in rows if r.id in done] - spent = sum(r.usd for r in rows if r.id in done | closed) - total = sum(window_usd.values()) - linked_usd = sum(v for k, v in window_usd.items() if k in linked) - out: dict[str, float] = { - "note_linked_usd_share": round(linked_usd / total, 4) if total else 0.0, - "notes_done": len(done), - "notes_done_linked": len(linked_done), - } - if linked_done: - out = {"usd_per_note": round(spent / len(linked_done), 2)} | out - return out - - -def _day(stamp: float | None) -> str: - if not stamp: - return "-" - return datetime.fromtimestamp(stamp, tz=UTC).astimezone().strftime("%Y-%m-%d") - - -def _title(text: str) -> str: - text = text.replace("|", "\\|") - return text if len(text) <= TITLE_CHARS else text[: TITLE_CHARS - 1] + "…" - - -def render( - rows: list[NoteRow], found: Mapping[str, float], rules: Mapping[str, int] -) -> list[str]: - """The WEEKLY.md section: metrics, link sources, top closed notes by USD.""" - per_note = found.get("usd_per_note") - lines = [ - "## Cost per todo note", - "", - f"- usd_per_note: {'n/a' if per_note is None else f'${per_note:,.2f}'}" - " (lower is better)", - f"- note_linked_usd_share: {found['note_linked_usd_share'] * 100:.1f}%", - f"- notes_done: {found['notes_done']} " - f"({found.get('notes_done_linked', 0)} with a linked session)", - "- links by rule: " - + " ".join(f"{k}={rules.get(k, 0)}" for k in ("a", "b", "c", "d")), - "", - "| note | status | done | sessions | USD | tokens |", - "|---|---|---|---|---|---|", - ] - lines += [ - f"| {_title(r.title)} | {r.status} | {_day(r.done_at)} | " - f"{len(r.sessions)} | ${r.usd:,.2f} | {r.tokens:,} |" - for r in rows[:TOP] - ] - return lines - - -def write_json(rows: list[NoteRow], out: Path) -> Path: - """Write every linked note for the dashboard.""" - out.mkdir(parents=True, exist_ok=True) - path = out / JSON_NAME - data = {"generated": time.time(), "notes": [asdict(r) for r in rows]} - path.write_text(json.dumps(data, indent=1), encoding="utf-8") - return path - - -def section( - root: Path, - book: pricing.PriceBook, - sessions: Sequence[Session], - window: tuple[float, float], -) -> tuple[dict[str, float], list[str], list[NoteRow]] | None: - """Metrics, WEEKLY.md lines and rows for one audit window.""" - started = time.monotonic() - found = notes_collect.collect(root, book, sessions, window[0]) - if found is None: - return None - rows = note_rows(found.notes, found.links, found.spend) - spent = notes_collect.window_usd(sessions, book) - got = metrics(found.notes, rows, window, spent, set(found.links)) - rules = " ".join(f"{k}={v}" for k, v in sorted(found.rules.items())) - sys.stderr.write( - f"notes: {len(found.links)} sessions linked ({rules}) " - f"in {time.monotonic() - started:.2f}s\n" - ) - closed = [r for r in rows if r.status in TERMINAL and _in(r.done_at, window)] - return got, ["", *render(closed, got, found.rules)], rows - - -def main(argv: list[str] | None = None) -> int: - """Rewrite notes.json from all transcripts, independent of any window.""" - parser = argparse.ArgumentParser(prog="token_audit.notes") - parser.add_argument("--root", type=Path, default=notes_collect.TRANSCRIPTS) - parser.add_argument("--out", type=Path, default=history.REPORT_DIR) - args = parser.parse_args(argv) - found = notes_collect.collect(args.root, pricing.load(), [], float("inf")) - if found is not None: - written = write_json(note_rows(found.notes, found.links, found.spend), args.out) - sys.stderr.write(f"Wrote {written}\n") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_pkg/token_audit/notes_backlog.py b/python_pkg/token_audit/notes_backlog.py deleted file mode 100644 index 7dd15fd9..00000000 --- a/python_pkg/token_audit/notes_backlog.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Todo notes plus their done date: the status HLC in the first dump showing it. - -The app keeps no done timestamp; the BACKLOG parse copies todo_mcp/_backlog.py. -""" - -from __future__ import annotations - -from dataclasses import dataclass, replace -from datetime import UTC, datetime -import gzip -import json -from pathlib import Path -import re -from typing import Any - -STATUSES = ("todo", "inProgress", "done", "abandoned") -TERMINAL = frozenset({"done", "abandoned"}) -_MARKER = re.compile(r"^[ \t]*$", re.MULTILINE) -_ATTR = re.compile(r'(\w+)="([^"]*)"') -_HLC = re.compile(r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?)Z") -_DUMP_GLOB = "kuhy-syncs-*.json.gz" -_FIELD_LEN = 2 - - -@dataclass(frozen=True) -class Note: - """One backlog note; ``done_at`` is set only for done/abandoned notes.""" - - id: str - title: str - body: str - priority: str - status: str - updated: float | None - done_at: float | None = None - - -def backlog_path() -> Path: - """Where the app's Settings -> Export notes writes BACKLOG.md.""" - return Path.home() / "src" / "todo" / "BACKLOG.md" - - -def dumps_dir() -> Path: - """Where the daily Firebase RTDB backups land.""" - return Path.home() / "data" / "cloud" / "firebase_backups" - - -def _local_time(raw: str | None) -> float | None: - if not raw: - return None - try: - return datetime.fromisoformat(raw).timestamp() - except ValueError: - return None - - -def parse_backlog(text: str) -> list[Note]: - """Every ```` block in an exported BACKLOG.md.""" - markers = list(_MARKER.finditer(text)) - out: list[Note] = [] - for index, marker in enumerate(markers): - attrs = dict(_ATTR.findall(marker.group(1))) - end = markers[index + 1].start() if index + 1 < len(markers) else len(text) - body = text[marker.end() : end].strip() - status = attrs.get("status", "todo") - out.append( - Note( - id=attrs.get("id", ""), - title=body.splitlines()[0].lstrip("#").strip() if body else "", - body=body, - priority=attrs.get("priority", "medium"), - status=status if status in STATUSES else "todo", - updated=_local_time(attrs.get("updated")), - ) - ) - return out - - -def hlc_time(hlc: object) -> float | None: - """The wall-clock part of an HLC string (``Z--``).""" - match = _HLC.match(hlc) if isinstance(hlc, str) else None - if match is None: - return None - return datetime.fromisoformat(match.group(1)).replace(tzinfo=UTC).timestamp() - - -def _status_field(note: object) -> tuple[int, str] | None: - if not isinstance(note, dict) or note.get("deleted"): - return None - field = (note.get("fields") or {}).get("status") - if not isinstance(field, list) or len(field) != _FIELD_LEN: - return None - value, hlc = field - return (value, hlc) if isinstance(value, int) and isinstance(hlc, str) else None - - -def merge_devices(devices: dict[str, Any]) -> dict[str, tuple[int, str]]: - """Last-writer-wins status per note across the per-device documents.""" - merged: dict[str, tuple[int, str]] = {} - for raw in devices.values(): - try: - doc = json.loads(raw) if isinstance(raw, str) else raw - except ValueError: - continue - for note_id, note in (doc if isinstance(doc, dict) else {}).items(): - found = _status_field(note) - if found is not None and ( - note_id not in merged or found[1] > merged[note_id][1] - ): - merged[note_id] = found - return merged - - -def _dump_statuses(path: Path) -> dict[str, tuple[int, str]]: - try: - data = json.loads(gzip.decompress(path.read_bytes())) - except OSError, ValueError, EOFError: - return {} - devices = ((data or {}).get("todo-sync") or {}).get("notes") or {} - return merge_devices(devices) if isinstance(devices, dict) else {} - - -def first_seen(directory: Path) -> dict[str, dict[str, float]]: - """Per note and terminal status: its status HLC in the first dump showing it.""" - seen: dict[str, dict[str, float]] = {} - for path in sorted(directory.glob(_DUMP_GLOB)) if directory.is_dir() else []: - for note_id, (value, hlc) in _dump_statuses(path).items(): - status = STATUSES[value] if 0 <= value < len(STATUSES) else "todo" - stamp = hlc_time(hlc) - if status in TERMINAL and stamp is not None: - seen.setdefault(note_id, {}).setdefault(status, stamp) - return seen - - -def _dated(note: Note, seen: dict[str, dict[str, float]]) -> Note: - if note.status not in TERMINAL: - return note - return replace(note, done_at=seen.get(note.id, {}).get(note.status, note.updated)) - - -def load_notes(backlog: Path, dumps: Path) -> list[Note]: - """BACKLOG notes with ``done_at`` filled for done/abandoned ones.""" - seen = first_seen(dumps) - text = backlog.read_text(encoding="utf-8") - return [_dated(note, seen) for note in parse_backlog(text)] diff --git a/python_pkg/token_audit/notes_collect.py b/python_pkg/token_audit/notes_collect.py deleted file mode 100644 index fd289d4a..00000000 --- a/python_pkg/token_audit/notes_collect.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Gather notes, session links and linked spend for the notes report.""" - -from __future__ import annotations - -from collections import Counter -from dataclasses import dataclass -from pathlib import Path -import sys -from typing import TYPE_CHECKING - -from python_pkg.token_audit import notes_backlog, notes_cost, notes_link, notes_scan - -if TYPE_CHECKING: - from collections.abc import Sequence - - from python_pkg.token_audit.model import Session - from python_pkg.token_audit.notes_backlog import Note - from python_pkg.token_audit.notes_cost import Spend - from python_pkg.token_audit.notes_link import Link - from python_pkg.token_audit.pricing import PriceBook - -TRANSCRIPTS = Path.home() / ".claude" / "projects" - - -@dataclass(frozen=True) -class Collected: - """Everything the notes metrics are computed from.""" - - notes: list[Note] - links: dict[str, Link] - spend: dict[str, Spend] - - @property - def rules(self) -> Counter[str]: - """How many sessions each rule linked.""" - return Counter(link.rule for link in self.links.values()) - - -def _reusable( - sessions: Sequence[Session], - facts: dict[str, notes_scan.Facts], - since: float, - book: PriceBook, -) -> dict[str, Spend]: - out: dict[str, Spend] = {} - for session in sessions: - fact = facts.get(session.parent_id or session.session_id) - if fact is not None and fact.start >= since: - out[session.path] = notes_cost.of_session(session, book) - return out - - -def collect( - root: Path, book: PriceBook, sessions: Sequence[Session], since: float -) -> Collected | None: - """Notes, links and lifetime spend; ``None`` when BACKLOG.md is missing.""" - backlog = notes_backlog.backlog_path() - if not backlog.exists(): - sys.stderr.write(f"notes: skipped, no {backlog}\n") - return None - notes = notes_backlog.load_notes(backlog, notes_backlog.dumps_dir()) - facts = notes_scan.scan_root(root) - links = notes_link.link_all(facts, notes) - mains = {sid: fact.path for sid, fact in facts.items()} - reuse = _reusable(sessions, facts, since, book) - spend = notes_cost.lifetime(mains, set(links), book, reuse) - return Collected(notes, links, spend) - - -def window_usd(sessions: Sequence[Session], book: PriceBook) -> dict[str, float]: - """In-window USD per main session id, subagents folded into their parent.""" - out: dict[str, float] = {} - for session in sessions: - key = session.parent_id or session.session_id - out[key] = out.get(key, 0.0) + notes_cost.of_session(session, book).usd - return out diff --git a/python_pkg/token_audit/notes_cost.py b/python_pkg/token_audit/notes_cost.py deleted file mode 100644 index 36c472f5..00000000 --- a/python_pkg/token_audit/notes_cost.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Lifetime USD and tokens of the sessions linked to notes (main + subagents). - -Window-free on purpose: a note finished this week was often worked last week. -Files already loaded whole by the audit are reused instead of re-parsed. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import TYPE_CHECKING - -from python_pkg.token_audit import parse, pricing -from python_pkg.token_audit.model import TOKEN_KINDS - -if TYPE_CHECKING: - from pathlib import Path - - from python_pkg.token_audit.model import Session - from python_pkg.token_audit.pricing import PriceBook - - -@dataclass(frozen=True) -class Spend: - """Dollars at list price and raw tokens (all four classes).""" - - usd: float = 0.0 - tokens: int = 0 - - def __add__(self, other: Spend) -> Spend: - """Sum two spends.""" - return Spend(self.usd + other.usd, self.tokens + other.tokens) - - -def of_session(session: Session, book: PriceBook) -> Spend: - """Spend of every turn a loaded session holds.""" - return Spend( - sum(pricing.usd(turn, book) for turn in session.turns), - sum(turn.usage.get(kind, 0) for turn in session.turns for kind in TOKEN_KINDS), - ) - - -def files_of(main: Path) -> list[Path]: - """A main transcript plus every subagent transcript filed under it.""" - return [main, *sorted(main.with_suffix("").glob("subagents/agent-*.jsonl"))] - - -def lifetime( - mains: dict[str, Path], - wanted: set[str], - book: PriceBook, - reuse: dict[str, Spend], -) -> dict[str, Spend]: - """Whole-file spend per wanted session id, reusing ``reuse[path]`` hits.""" - out: dict[str, Spend] = {} - for sid in sorted(wanted & mains.keys()): - total = Spend() - for path in files_of(mains[sid]): - hit = reuse.get(str(path)) - total += ( - hit if hit is not None else of_session(parse.load_session(path), book) - ) - out[sid] = total - return out diff --git a/python_pkg/token_audit/notes_link.py b/python_pkg/token_audit/notes_link.py deleted file mode 100644 index 88a484b9..00000000 --- a/python_pkg/token_audit/notes_link.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Link sessions to todo notes; the first rule that yields a note decides. - -a: a Bash call ran ``note_claim.sh ``; b: a typed prompt names -``todo-images//``; c: the first prompt fuzzy-matches one note; -d: an ``Implement|Continue `` session inherits its writer's notes. -""" - -from __future__ import annotations - -from dataclasses import dataclass -import functools -import re -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from collections.abc import Iterable - - from python_pkg.token_audit.notes_backlog import Note - from python_pkg.token_audit.notes_scan import Facts - -FUZZY_THRESHOLD = 0.5 -MIN_TOKENS = 4 -MAX_IMAGE_NOTES = 2 -_TOKEN = re.compile(r"[^\W_]{3,}") -_STOP = frozenset( - [ - "the", - "and", - "for", - "that", - "this", - "with", - "you", - "are", - "not", - "but", - "have", - "from", - "what", - "when", - "where", - "should", - "would", - "can", - "will", - "just", - "all", - "any", - "its", - "into", - "also", - "then", - "than", - "them", - "they", - "their", - "there", - "was", - "were", - "been", - "being", - "lets", - "let", - "make", - "made", - "does", - "dont", - ] -) - - -@dataclass(frozen=True) -class Link: - """The notes one session worked on and the rule that found them.""" - - notes: tuple[str, ...] - rule: str - - -@functools.cache -def tokens(text: str) -> frozenset[str]: - """Normalised content words: lower-case, 3+ letters, no stopwords/digits.""" - return frozenset( - t for t in _TOKEN.findall(text.lower()) if t not in _STOP and not t.isdigit() - ) - - -def _dice(left: frozenset[str], right: frozenset[str]) -> float: - total = len(left) + len(right) - return 2 * len(left & right) / total if total else 0.0 - - -def score(prompt: str, note: Note) -> float: - """Body overlap, or the prompt's first line against the note title.""" - body = _dice(tokens(prompt), tokens(note.body)) - title = tokens(note.title) - if len(title) < MIN_TOKENS: - return body - first_line = prompt.strip().splitlines()[0] if prompt.strip() else "" - return max(body, _dice(tokens(first_line), title)) - - -def resolve(token: str, ids: Iterable[str]) -> str | None: - """The one note id ``token`` names (full id or unique prefix).""" - token = token.lower() - found = [i for i in ids if i.lower().startswith(token)] - return found[0] if len(found) == 1 else None - - -def _explicit(facts: Facts, ids: list[str]) -> tuple[str, ...]: - found = (resolve(token, ids) for token in facts.claims) - return tuple(dict.fromkeys(i for i in found if i is not None)) - - -def _images(facts: Facts, ids: set[str]) -> tuple[str, ...]: - known = tuple(i for i in facts.images if i in ids) - return known if len(known) <= MAX_IMAGE_NOTES else () - - -def _fuzzy(facts: Facts, notes: list[Note]) -> tuple[str, ...]: - if facts.handoff or len(tokens(facts.first_prompt)) < MIN_TOKENS: - return () - scored = [(score(facts.first_prompt, n), n.updated or 0.0, n.id) for n in notes] - best = max(scored, default=(0.0, 0.0, "")) - return (best[2],) if best[0] >= FUZZY_THRESHOLD else () - - -def _writers(facts: dict[str, Facts]) -> dict[str, tuple[float, str]]: - first: dict[str, tuple[float, str]] = {} - for sid, fact in facts.items(): - for rel, stamp in fact.creates.items(): - first[rel] = min(first.get(rel, (stamp, sid)), (stamp, sid)) - return first - - -def link_all(facts: dict[str, Facts], notes: list[Note]) -> dict[str, Link]: - """Links per session id, resolved in start order so writers come first.""" - ids = [n.id for n in notes if n.id] - writers = _writers(facts) - links: dict[str, Link] = {} - for sid in sorted(facts, key=lambda s: (facts[s].start, s)): - fact = facts[sid] - for rule, found in ( - ("a", _explicit(fact, ids)), - ("b", _images(fact, set(ids))), - ("c", _fuzzy(fact, notes)), - ): - if found: - links[sid] = Link(found, rule) - break - else: - writer = writers.get(fact.handoff or "") - if writer and writer[1] != sid and writer[0] <= fact.start: - inherited = links.get(writer[1]) - if inherited is not None: - links[sid] = Link(inherited.notes, "d") - return links diff --git a/python_pkg/token_audit/notes_scan.py b/python_pkg/token_audit/notes_scan.py deleted file mode 100644 index 5bdd2a65..00000000 --- a/python_pkg/token_audit/notes_scan.py +++ /dev/null @@ -1,144 +0,0 @@ -"""Per-session facts the note join needs, read from every main transcript. - -Only typed user prompts and assistant tool_use blocks are read; tool results -are never evidence (a session that reads BACKLOG.md names dozens of ids). -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -import re -from typing import TYPE_CHECKING, Any - -from python_pkg.token_audit import parse, records - -if TYPE_CHECKING: - from pathlib import Path - -CLAIM = re.compile(r"note_claim\.sh\s+[\"']?([0-9A-Fa-f][0-9A-Fa-f-]{7,35})") -IMAGE = re.compile(r"todo-images/([0-9a-f]{8}-[0-9a-f-]{27})/") -HANDOFF_PROMPT = re.compile(r"^\s*(?:implement|continue)\s+(\S+)", re.IGNORECASE) -HANDOFF_DIR = ".claude/handoff/" -_WRAPPER = re.compile( - r"<(system-reminder|command-[\w-]+|local-command[\w-]*)[^>]*>.*?", re.DOTALL -) -_PASTE_TAG = re.compile(r"]*>") -_BASH_CREATE = re.compile(r"(?:(?&\d])>(?!>)|\btee\s+(?!-a))\s*[\"']?(\S+)") -_SKIP_PREFIX = ("Caveat:",) - - -@dataclass -class Facts: - """What one main session contributes to the join.""" - - session_id: str - path: Path - start: float = 0.0 - first_prompt: str = "" - handoff: str | None = None - claims: list[str] = field(default_factory=list) - images: list[str] = field(default_factory=list) - creates: dict[str, float] = field(default_factory=dict) - - -def handoff_rel(raw: str) -> str | None: - """The path below ``~/.claude/handoff/``, or ``None`` if not under it.""" - _, sep, rel = raw.strip("\"'`").partition(HANDOFF_DIR) - return rel.rstrip(".,;)") or None if sep else None - - -def prompt_text(record: dict[str, Any]) -> str | None: - """Typed prompt text, wrappers removed; ``None`` for anything else.""" - if record.get("type") != "user" or record.get("isSidechain"): - return None - if record.get("isMeta"): - return None - content = (record.get("message") or {}).get("content") - if isinstance(content, list): - if any(b.get("type") == "tool_result" for b in content if isinstance(b, dict)): - return None - content = "\n".join( - str(b.get("text") or "") - for b in content - if isinstance(b, dict) and b.get("type") == "text" - ) - if not isinstance(content, str): - return None - text = _PASTE_TAG.sub("", _WRAPPER.sub("", content)).strip() - return "" if text.startswith(_SKIP_PREFIX) else text - - -def _created_paths(name: str, tool_input: dict[str, Any]) -> list[str]: - if name == "Write": - path = tool_input.get("file_path") - return [path] if isinstance(path, str) else [] - command = tool_input.get("command") - if name != "Bash" or not isinstance(command, str) or HANDOFF_DIR not in command: - return [] - return _BASH_CREATE.findall(command) - - -def _tool_uses(record: dict[str, Any]) -> list[dict[str, Any]]: - if record.get("type") != "assistant": - return [] - content = (record.get("message") or {}).get("content") - blocks = content if isinstance(content, list) else [] - return [b for b in blocks if isinstance(b, dict) and b.get("type") == "tool_use"] - - -def _use(facts: Facts, record: dict[str, Any], *, main: bool) -> None: - stamp = records.timestamp(record) or 0.0 - for block in _tool_uses(record): - tool_input = block.get("input") if isinstance(block.get("input"), dict) else {} - name = str(block.get("name")) - if main and name == "Bash": - facts.claims += CLAIM.findall(str(tool_input.get("command") or "")) - for raw in _created_paths(name, tool_input): - rel = handoff_rel(raw) - if rel is not None and rel not in facts.creates: - facts.creates[rel] = stamp - - -def _prompt(facts: Facts, record: dict[str, Any]) -> None: - text = prompt_text(record) - if not text: - return - facts.images += [i for i in IMAGE.findall(text) if i not in facts.images] - if not facts.first_prompt: - facts.first_prompt = text - match = HANDOFF_PROMPT.match(text) - facts.handoff = handoff_rel(match.group(1)) if match else None - - -def _wanted(line: str) -> bool: - if '"tool_use"' in line: - return "note_claim" in line or HANDOFF_DIR in line - return '"type":"user"' in line and '"tool_result"' not in line - - -def scan(path: Path, *, main: bool = True) -> Facts: - """Read one transcript's prompts, claims and handoff-file creations.""" - facts = Facts(session_id=path.stem, path=path) - with path.open(encoding="utf-8", errors="replace") as handle: - for line in handle: - if not facts.start and '"timestamp"' in line: - facts.start = records.timestamp(parse.load_line(line) or {}) or 0.0 - if not _wanted(line): - continue - record = parse.load_line(line) or {} - _use(facts, record, main=main) - if main: - _prompt(facts, record) - return facts - - -def scan_root(root: Path) -> dict[str, Facts]: - """Facts per main session; subagent creations fold into their parent.""" - found = {p.stem: scan(p) for p in sorted(root.glob("*/*.jsonl"))} - for sub in sorted(root.glob("*/*/subagents/agent-*.jsonl")): - parent = found.get(sub.parent.parent.name) - if parent is None: - continue - for rel, stamp in scan(sub, main=False).creates.items(): - parent.creates[rel] = min(stamp, parent.creates.get(rel, stamp)) - return found diff --git a/python_pkg/token_audit/parse.py b/python_pkg/token_audit/parse.py deleted file mode 100644 index 0d43f2d4..00000000 --- a/python_pkg/token_audit/parse.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Read Claude Code transcripts into :class:`Session` objects. - -Transcripts are JSONL under ``~/.claude/projects//.jsonl``, -one JSON object per line, appended live. Three properties of that format drive -the design here: - -1. **Lines can be malformed.** A transcript being written while it is read can - end in a partial line, and older sessions contain records this package does - not model. Every line is therefore parsed defensively and skipped on error; - losing one line is always better than aborting a weekly report. -2. **A tool call and its result are in different messages.** ``tool_use`` blocks - carry the name and input, ``tool_result`` blocks carry the payload and only a - ``tool_use_id`` back-reference. Attributing result size to the right tool - requires holding an id map for the length of the file. -3. **Order matters.** The image-cost model needs to know how many turns followed - each image, so :func:`iter_events` preserves the interleaving rather than - returning separate lists. -""" - -from __future__ import annotations - -import json -import re -from typing import TYPE_CHECKING, Any - -from python_pkg.token_audit import discover, records, turns -from python_pkg.token_audit.model import IMAGE_SUFFIXES, Session, ToolCall, Turn - -if TYPE_CHECKING: - from collections.abc import Iterator - from pathlib import Path - -# Divisor turning rendered characters into an approximate token count. Four is -# the usual English-text rule of thumb and is only ever used for ranking tools, -# never for the reconciled totals. -CHARS_PER_TOKEN = 4 - -# Median billed size of one screenshot, measured from real transcripts by -# isolating turns whose only new content was a single image (n=484, median -# 2,566, p90 2,692). Anthropic bills images by pixel area, so a constant is a -# far better estimate than the base64 payload length, which is ~20x too high. -# Halving a screenshot's dimensions cuts this roughly fourfold. -IMAGE_TOKENS = 2566 - -# Commands that land work: a plain commit or the scripted wrap-up. -COMMIT = re.compile(r"\bgit(\s+-C\s+\S+)?\s+commit\b|finish_auto\.sh") - - -def _load_line(line: str) -> dict[str, Any] | None: - """Parse one JSONL line, returning ``None`` if it is not a usable record.""" - try: - record = json.loads(line) - except ValueError, UnicodeDecodeError: - return None - return record if isinstance(record, dict) else None - - -def _blocks(record: dict[str, Any]) -> list[dict[str, Any]]: - """Return the content blocks of a record's message, if it has any.""" - message = record.get("message") - if not isinstance(message, dict): - return [] - content = message.get("content") - if not isinstance(content, list): - return [] - return [block for block in content if isinstance(block, dict)] - - -def _result_tokens(block: dict[str, Any], *, is_image: bool = False) -> int: - """Estimate how many tokens a tool result occupied in context. - - Text is charged by length, but an **image is not**. An image arrives as a - base64 blob whose character count has nothing to do with its billed size — - Anthropic charges by pixel area. Measuring images by payload length - overstates them by roughly 20x (a 240KB PNG looks like ~82k tokens against - a real cost near 2.5k), which is enough to make screenshots look like the - dominant cost driver when they are not. Validated against real transcripts - by isolating turns whose only new content was one image: the context grew - ~1.6-3k tokens, never the tens of thousands the payload length implied. - """ - content = block.get("content") - if content is None: - return 0 - if is_image: - return IMAGE_TOKENS - return len(json.dumps(content)) // CHARS_PER_TOKEN - - -def iter_events( - path: Path, - window: records.Window | None = None, -) -> Iterator[tuple[str, object]]: - """Yield ``("turn", Turn)`` and ``("tool", ToolCall)`` in transcript order. - - Emitting a single ordered stream — rather than two lists — is what lets - :mod:`imagecost` count how many turns each image survived for. Turns are - one per API message (see :mod:`records`), and only records inside - ``window`` are emitted. - """ - pending: dict[str, dict[str, Any]] = {} - merger = records.TurnMerger() - with path.open(encoding="utf-8", errors="replace") as handle: - for line in handle: - record = _load_line(line) - if record is None or not records.in_window(record, window): - continue - pairs: list[tuple[str, object]] = [] - for block in _blocks(record): - pairs += _block_events(block, pending, merger) - yield from pairs - found = turns.usage_of(record) - if found is not None: - done = merger.push(turns.build_turn(record, *found)) - if done is not None: - yield "turn", done - held = merger.flush() - if held is not None: - yield "turn", held - - -def _block_events( - block: dict[str, Any], - pending: dict[str, dict[str, Any]], - merger: records.TurnMerger, -) -> list[tuple[str, object]]: - """Events one content block emits: remember a ``tool_use``, pair a result. - - A ``tool_result`` ends the assistant message before it, so the held turn is - released first to keep the stream in transcript order. - """ - kind = block.get("type") - if kind == "tool_use": - call_id = block.get("id") - if isinstance(call_id, str): - pending[call_id] = block - return [] - if kind != "tool_result": - return [] - out: list[tuple[str, object]] = [] - held = merger.flush() - if held is not None: - out.append(("turn", held)) - call = _pair_result(block, pending) - if call is not None: - out.append(("tool", call)) - return out - - -def _pair_result( - block: dict[str, Any], - pending: dict[str, dict[str, Any]], -) -> ToolCall | None: - """Join a ``tool_result`` back to the ``tool_use`` that produced it.""" - use = pending.pop(block.get("tool_use_id", ""), None) - if use is None: - return None - tool_input = use.get("input") - tool_input = tool_input if isinstance(tool_input, dict) else {} - path = tool_input.get("file_path") - skill = tool_input.get("skill") - command = tool_input.get("command") - call_path = path if isinstance(path, str) else None - return ToolCall( - name=str(use.get("name") or "unknown"), - result_tokens=_result_tokens(block, is_image=_looks_like_image(call_path)), - path=call_path, - skill=skill if isinstance(skill, str) else None, - commit=isinstance(command, str) and COMMIT.search(command) is not None, - ) - - -def _looks_like_image(path: str | None) -> bool: - """Whether a tool-result path refers to an image, by extension.""" - if path is None: - return False - dot = path.rfind(".") - return dot != -1 and path[dot:].lower() in IMAGE_SUFFIXES - - -def load_session(path: Path, window: records.Window | None = None) -> Session: - """Read one transcript file into a :class:`Session`. - - A subagent transcript (``/subagents/agent-*.jsonl``) is tagged - with its parent session id and the ``agentType``/``description`` the - spawning ``Agent`` call recorded in the sibling ``.meta.json``. With no - description anywhere, the first prompt line stands in, then the agent id. - """ - session = Session( - session_id=path.stem, path=str(path), cwd=discover.first_cwd(path) - ) - if path.parent.name == "subagents": - meta = discover.agent_meta(path) - session.parent_id = path.parent.parent.name - session.agent_type = str(meta.get("agentType") or "unknown") - session.description = str( - meta.get("description") - or discover.first_prompt(path) - or f"(no label) {path.stem}" - ) - for kind, event in iter_events(path, window): - if kind == "turn" and isinstance(event, Turn): - session.turns.append(event) - elif isinstance(event, ToolCall): - session.tools.append(event) - return session - - -# Public names for the record primitives, shared with token_dashboard's walker -# (which needs prompt boundaries and timestamps that iter_events drops). -load_line = _load_line -content_blocks = _blocks -pair_result = _pair_result diff --git a/python_pkg/token_audit/prices.json b/python_pkg/token_audit/prices.json deleted file mode 100644 index edc4a984..00000000 --- a/python_pkg/token_audit/prices.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "_source": "claude-api skill model table (cached 2026-09-25), read 2026-10-04. USD per million tokens. Cache writes are priced as multiples of input: 5m = 1.25x, 1h = 2x (verified 2026-10-04: a claude -p probe's total_cost_usd matched opus-5-5 at 2x for its 1h write to 7 digits). Entries marked assumed_cache_read use the standard 0.1x-of-input ratio because the table does not list them.", - "default": "claude-opus-5-5", - "models": { - "claude-fable-5-1": { "input": 10.0, "output": 50.0, "cache_read": 0.25 }, - "claude-opus-5-5": { "input": 4.0, "output": 20.0, "cache_read": 0.2 }, - "claude-opus-5": { - "input": 5.0, - "output": 25.0, - "cache_read": 0.5, - "assumed_cache_read": true - }, - "claude-sonnet-5-5": { "input": 2.0, "output": 10.0, "cache_read": 0.2 }, - "claude-sonnet-5": { - "input": 2.0, - "output": 10.0, - "cache_read": 0.2, - "assumed_cache_read": true - }, - "claude-haiku-4-5": { - "input": 1.0, - "output": 5.0, - "cache_read": 0.1, - "assumed_cache_read": true - } - } -} diff --git a/python_pkg/token_audit/pricing.py b/python_pkg/token_audit/pricing.py deleted file mode 100644 index bcf7a09f..00000000 --- a/python_pkg/token_audit/pricing.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Price turns in US dollars, per model. - -The token-class weights in :mod:`model` are model-blind: an Opus call and a -Haiku call with identical usage weigh the same, so the weighted total cannot -say what moving work to a cheaper model would save. Dollars can. They are API -list prices -- on a subscription they are not what is billed, but they are the -best available proxy for how fast a usage limit drains, and they make Opus, -Sonnet and Haiku spend comparable. - -One measured surprise this exposed: Sonnet 5.5 cache reads cost the same -$0.20/MTok as Opus 5.5. Cache reads are most of the bill, so moving a long -agent to Sonnet saves far less than the 2x the headline prices suggest. -""" - -from __future__ import annotations - -from dataclasses import dataclass -import json -from pathlib import Path -from typing import TYPE_CHECKING - -from python_pkg.token_audit.records import CACHE_1H, CACHE_5M - -if TYPE_CHECKING: - from python_pkg.token_audit.model import Turn - -PRICES_PATH = Path(__file__).with_name("prices.json") -PER_MILLION = 1_000_000 -WRITE_5M = 1.25 -WRITE_1H = 2.0 - - -@dataclass(frozen=True) -class Price: - """List prices for one model, USD per million tokens.""" - - input: float - output: float - cache_read: float - - -@dataclass(frozen=True) -class PriceBook: - """Every known model's price plus the fallback for unknown ids.""" - - models: dict[str, Price] - default: str - - def for_model(self, model: str) -> Price: - """Exact id, else the longest known id it starts with, else default. - - The prefix match covers dated or suffixed ids (``claude-haiku-4-5-2025...``). - """ - if model in self.models: - return self.models[model] - matches = [known for known in self.models if model.startswith(known)] - if matches: - return self.models[max(matches, key=len)] - return self.models[self.default] - - -def load(path: Path = PRICES_PATH) -> PriceBook: - """Read the price table shipped next to this module.""" - data = json.loads(path.read_text(encoding="utf-8")) - models = { - name: Price( - input=float(row["input"]), - output=float(row["output"]), - cache_read=float(row["cache_read"]), - ) - for name, row in data["models"].items() - } - return PriceBook(models=models, default=str(data["default"])) - - -def usd(turn: Turn, book: PriceBook) -> float: - """Return one turn's list-price cost in dollars. - - Synthetic turns (````) carry zero usage and so cost zero under - any price, which is why no special case is needed for them. When a turn - has no 1h/5m split, the whole cache write is priced as 5m (the cheaper, - older default) -- an underestimate, never an overestimate. - """ - usage = turn.usage - price = book.for_model(turn.model) - creation = usage.get("cache_creation_input_tokens", 0) - one_hour = usage.get(CACHE_1H, 0) - five_min = usage.get(CACHE_5M, creation - one_hour) - total = ( - usage.get("input_tokens", 0) * price.input - + five_min * WRITE_5M * price.input - + one_hour * WRITE_1H * price.input - + usage.get("cache_read_input_tokens", 0) * price.cache_read - + usage.get("output_tokens", 0) * price.output - ) - return total / PER_MILLION diff --git a/python_pkg/token_audit/probe.py b/python_pkg/token_audit/probe.py deleted file mode 100644 index 3c0576bd..00000000 --- a/python_pkg/token_audit/probe.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Measure the real per-turn prefix by starting Claude Code and reading usage. - -Disk sizes do not predict billed cost: deferred MCP schemas are absent while -their tool *names* are not, and skill bodies are summarised. The only honest -number comes from a real process start. - -Two lessons are encoded as defaults here, both learned by getting them wrong: - -* ``--strict-mcp-config`` suppresses project-scoped ``.mcp.json`` files, so an - A/B run with it understates the baseline (~2,000 tokens/turn in one audit). - Probes therefore run **non-strict** unless explicitly asked otherwise. -* A saving predicted from a synthetic config must be re-measured after the - change actually lands. :func:`check_prediction` exists to fail loudly when the - two disagree, because one audit predicted 6,122 and delivered 4,465. -""" - -from __future__ import annotations - -from dataclasses import dataclass -import json -from pathlib import Path -import subprocess - -CLAUDE_BIN = Path.home() / ".local" / "bin" / "claude" -PROBE_PROMPT = "say ok" -PROBE_TIMEOUT = 300 -# Fraction by which a predicted saving may miss the measured one before the -# prediction is treated as unreliable rather than merely imprecise. -PREDICTION_TOLERANCE = 0.2 -PREFIX_KINDS = ( - "input_tokens", - "cache_creation_input_tokens", - "cache_read_input_tokens", -) - - -@dataclass(frozen=True) -class Probe: - """One measured process start.""" - - label: str - prefix_tokens: int - - -def _prefix_from_usage(usage: dict) -> int: - """Sum the token classes that make up the request prefix. - - Cache creation and cache read are alternatives — a cold start reports the - prefix as creation, a warm one as read — so summing both is correct and - stable across repeat runs. - """ - return sum(usage.get(kind) or 0 for kind in PREFIX_KINDS) - - -def measure( - label: str, - *, - cwd: Path | None = None, - mcp_config: Path | None = None, - strict: bool = False, -) -> Probe: - """Start Claude Code once and report the prefix it billed. - - ``strict`` defaults to False on purpose; see the module docstring. - """ - cmd = [str(CLAUDE_BIN), "-p", PROBE_PROMPT, "--output-format", "json"] - if mcp_config is not None: - cmd += ["--mcp-config", str(mcp_config)] - if strict: - cmd.append("--strict-mcp-config") - result = subprocess.run( - cmd, - cwd=str(cwd or Path.home()), - capture_output=True, - text=True, - timeout=PROBE_TIMEOUT, - check=True, - ) - usage = json.loads(result.stdout).get("usage") or {} - return Probe(label, _prefix_from_usage(usage)) - - -def check_prediction( - predicted_saving: int, measured_saving: int, tolerance: float = PREDICTION_TOLERANCE -) -> str | None: - """Return a warning when a predicted saving missed the measured one. - - Returns ``None`` when the prediction held, so callers can treat a truthy - result as a gate failure. - """ - if predicted_saving <= 0: - return None - drift = abs(measured_saving - predicted_saving) / predicted_saving - if drift <= tolerance: - return None - return ( - f"prediction off by {drift:.0%}: predicted {predicted_saving:,} " - f"tokens/turn, measured {measured_saving:,}" - ) - - -def weighted_pct( - tokens_per_turn: int, - turns: int, - weighted_total: int, - cache_read_weight: float = 0.1, -) -> float: - """Convert a per-turn prefix saving into a share of weighted spend. - - Every lever must be quoted through this function. Prefix tokens are billed - as cache reads, which carry a 0.1 weight, so a raw per-turn figure compared - against a weighted headline overstates the saving by roughly 10x — a - mistake made, and caught, in an earlier audit. - """ - if weighted_total <= 0: - return 0.0 - return 100.0 * tokens_per_turn * turns * cache_read_weight / weighted_total diff --git a/python_pkg/token_audit/recommend.py b/python_pkg/token_audit/recommend.py deleted file mode 100644 index 621617e0..00000000 --- a/python_pkg/token_audit/recommend.py +++ /dev/null @@ -1,209 +0,0 @@ -"""Deterministic lever candidates: thresholds in code, not in a model's head. - -Every rule reads the snapshot ``metrics`` (and the previous snapshot's, for -growth checks) and either emits a :class:`Lever` or stays silent. The skill -layered on top adds judgment -- which levers suit the user, how to phrase a -prompt habit -- but never decides *whether* a number crossed a line; that is -an ``if`` statement and lives here. -""" - -from __future__ import annotations - -from dataclasses import dataclass, replace -from typing import Any - -# Rule thresholds. Each is the point past which the lever is worth a look, -# chosen from the 2026-10-04 audit's distributions, not tuned to fire. -SUBAGENT_SHARE = 0.15 -SUBAGENT_OPUS_SHARE = 0.5 -OVER_BUDGET_SHARE = 0.2 -HOME_CWD_SHARE = 0.2 -BATCHING_MIN = 1.15 -BASH_BIG_SHARE = 0.25 -PREFIX_GROWTH = 0.05 -IMAGE_SHARE = 0.03 - - -@dataclass(frozen=True) -class Lever: - """One candidate change, sized where the data allows.""" - - category: str - title: str - evidence: str - action: str - est_usd_per_day: float | None = None - # The snapshot metric the lever is meant to move: the join key to ledger - # lines, whose free-text titles never match these. - metric: str = "" - - -# Snapshot metric each rule's lever targets, by title (titles are fixed). -TITLE_METRIC: dict[str, str] = { - "Opus runs most subagent work": "subagent_opus_usd_share", - "Calls over the context budget": "over_budget_usd_share", - "Sessions started in ~": "home_cwd_usd_share", - "Independent tool calls not batched": "batching_calls_per_msg", - "Large Bash results": "bash_big_result_share", - "Always-loaded context grew": "standing_tokens_est", - "Images held in context": "image_usd_share", - "Unused MCP servers / skills": "standing_tokens_est", -} - - -def _num(metrics: dict[str, Any], key: str) -> float: - """Read a numeric metric, treating anything missing as zero.""" - value = metrics.get(key, 0) - return float(value) if isinstance(value, (int, float)) else 0.0 - - -def _subagents(m: dict[str, Any]) -> list[Lever]: - """Opus doing mechanical subagent work.""" - if _num(m, "subagent_usd_share") < SUBAGENT_SHARE: - return [] - if _num(m, "subagent_opus_usd_share") < SUBAGENT_OPUS_SHARE: - return [] - saving = _num(m, "subagent_sonnet_saving_usd") - days = _num(m, "usd_total") / max(_num(m, "usd_per_day"), 1e-9) - return [ - Lever( - "cheaper agents", - "Opus runs most subagent work", - f"subagents {_num(m, 'subagent_usd_share'):.0%} of spend, " - f"{_num(m, 'subagent_opus_usd_share'):.0%} of that on Opus; " - f"re-priced at Sonnet the same turns cost ${saving:,.2f} less", - "route coverage/test/search agents to the Sonnet-pinned agent " - "definitions; keep implementers on the default", - saving / days if days else None, - ), - ] - - -def _shape(m: dict[str, Any]) -> list[Lever]: - """Session shape: context budget and working directory.""" - out: list[Lever] = [] - per_day = _num(m, "usd_per_day") - if _num(m, "over_budget_usd_share") > OVER_BUDGET_SHARE: - out.append( - Lever( - "session length", - "Calls over the context budget", - f"{_num(m, 'over_budget_usd_share'):.0%} of spend ran with " - "context above the gate's budget", - "check the context gate fires (hook log) and that handoffs are " - "being written; lower the budget if they are", - # Calls over budget cost ~2.4x a fresh call (21k vs 51k). - per_day * _num(m, "over_budget_usd_share") * 0.58, - ), - ) - if _num(m, "home_cwd_usd_share") > HOME_CWD_SHARE: - out.append( - Lever( - "prompt habits", - "Sessions started in ~", - f"{_num(m, 'home_cwd_usd_share'):.0%} of spend had cwd=~ " - "(no repo CLAUDE.md, so agents explore to find commands)", - "start claude inside the repo; add exact run/test commands to " - "each repo's CLAUDE.md", - ), - ) - return out - - -def _mechanics(m: dict[str, Any], prev: dict[str, Any] | None) -> list[Lever]: - """Batching, output size, prefix growth, images.""" - out: list[Lever] = [] - ratio = _num(m, "batching_calls_per_msg") - if 0 < ratio < BATCHING_MIN: - out.append( - Lever( - "agent mechanics", - "Independent tool calls not batched", - f"{ratio:.2f} calls per tool message", - "batch-nudge hook is not moving this; prefer scripts that run " - "a whole read-only chain and report once", - ), - ) - big = _num(m, "bash_big_result_share") - if big > BASH_BIG_SHARE: - out.append( - Lever( - "agent mechanics", - "Large Bash results", - f"{big:.0%} of Bash result tokens come from results over 2k " - f"({_num(m, 'bash_tokens_per_day'):,.0f} tokens/day)", - "read files with ranges (sed -n 'a,bp', grep -n), never cat a " - "whole file; cap every pipeline stage, not just the last", - ), - ) - if prev is not None: - before = _num(prev, "standing_tokens_est") - now = _num(m, "standing_tokens_est") - if before and (now - before) / before > PREFIX_GROWTH: - out.append( - Lever( - "standing prefix", - "Always-loaded context grew", - f"{before:,.0f} -> {now:,.0f} est tokens per call", - "compact the surfaces that grew (see checkup); archive " - "finished memory entries", - ), - ) - if _num(m, "image_usd_share") > IMAGE_SHARE: - out.append( - Lever( - "agent mechanics", - "Images held in context", - f"{_num(m, 'image_usd_share'):.0%} of spend", - "extend the shrink hook to the tools returning screenshots", - ), - ) - return out - - -def _dead(dead_mcp: list[str], dead_skills: list[str]) -> list[Lever]: - """Registered surfaces nobody used in either window.""" - if not dead_mcp and not dead_skills: - return [] - return [ - Lever( - "standing prefix", - "Unused MCP servers / skills", - f"MCP: {', '.join(dead_mcp) or '-'}; " - f"skills: {', '.join(dead_skills) or '-'}", - "park them (mcp_park.sh / move skill dir to backups/skills-parked)", - ), - ] - - -def candidates( - metrics: dict[str, Any], - previous: dict[str, Any] | None, - dead_mcp: list[str], - dead_skills: list[str], -) -> list[Lever]: - """Every lever whose rule fired, sized ones first by daily saving.""" - levers = ( - _subagents(metrics) - + _shape(metrics) - + _mechanics(metrics, previous) - + _dead(dead_mcp, dead_skills) - ) - tagged = [replace(lv, metric=TITLE_METRIC.get(lv.title, "")) for lv in levers] - return sorted(tagged, key=lambda lv: -(lv.est_usd_per_day or 0.0)) - - -def render(levers: list[Lever]) -> list[str]: - """Markdown table of the candidates.""" - if not levers: - return ["_No rule fired._"] - out = [ - "| category | lever | evidence | est $/day | action |", - "|---|---|---|---|---|", - ] - for lv in levers: - est = f"${lv.est_usd_per_day:,.2f}" if lv.est_usd_per_day else "-" - out.append( - f"| {lv.category} | {lv.title} | {lv.evidence} | {est} | {lv.action} |" - ) - return out diff --git a/python_pkg/token_audit/records.py b/python_pkg/token_audit/records.py deleted file mode 100644 index 7a0c3c34..00000000 --- a/python_pkg/token_audit/records.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Record-level helpers: timestamps, the time window, and per-message dedupe. - -Claude Code writes ONE API message as SEVERAL transcript records -- one per -content block (thinking, text, each ``tool_use``) -- and every one of them -carries the same, complete ``usage`` block. Summing usage per record therefore -counts each API call once per block. Measured 2026-10-04 over 32 days: 74,937 -records for 40,388 real calls, a 1.9x overstatement that every report before -that date carried. :class:`TurnMerger` collapses the records back into one -:class:`Turn` per ``message.id``. - -The window is applied per record, by its own ``timestamp``, because a file's -mtime only says when the session was last active: a session started three weeks -ago and touched today would otherwise count three weeks of spend as "this week". -""" - -from __future__ import annotations - -from dataclasses import replace -from datetime import datetime -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from python_pkg.token_audit.model import Turn - -# Window bounds as unix seconds, inclusive at both ends. -Window = tuple[float, float] - -# Names under which the 1h / 5m cache-write split is stored on a Turn's usage. -# They are deliberately NOT in model.TOKEN_KINDS: the reconciled weighted total -# keeps its historical definition, while pricing reads the split when present. -CACHE_1H = "cache_creation_1h" -CACHE_5M = "cache_creation_5m" - - -def timestamp(record: dict[str, Any]) -> float | None: - """Return a record's ISO timestamp as unix seconds, or ``None``.""" - raw = record.get("timestamp") - if not isinstance(raw, str): - return None - try: - return datetime.fromisoformat(raw).timestamp() - except ValueError: - return None - - -def in_window(record: dict[str, Any], window: Window | None) -> bool: - """Whether a record falls inside the window; undated records are kept.""" - if window is None: - return True - stamp = timestamp(record) - return stamp is None or window[0] <= stamp <= window[1] - - -def cache_split(usage: dict[str, Any]) -> dict[str, int]: - """Extract the 1h / 5m cache-write split from a raw usage block. - - A 1h cache write bills at 2x input and a 5m one at 1.25x, so the split is - worth up to 60% of the cache-write cost. Claude Code currently writes 1h. - """ - nested = usage.get("cache_creation") - if not isinstance(nested, dict): - return {} - out: dict[str, int] = {} - for key, name in ( - ("ephemeral_1h_input_tokens", CACHE_1H), - ("ephemeral_5m_input_tokens", CACHE_5M), - ): - value = nested.get(key) - if isinstance(value, int): - out[name] = value - return out - - -def merge(held: Turn, extra: Turn) -> Turn: - """Fold a second record of the same API message into the first. - - Usage is the same block repeated, so the max per key is taken (robust to a - record written before ``output_tokens`` was final). Tool calls are spread - across the records -- one per ``tool_use`` block -- so those are summed. - """ - keys = held.usage.keys() | extra.usage.keys() - usage = {k: max(held.usage.get(k, 0), extra.usage.get(k, 0)) for k in keys} - return replace( - held, - usage=usage, - context=max(held.context, extra.context), - tool_calls=held.tool_calls + extra.tool_calls, - ) - - -class TurnMerger: - """Collapse consecutive same-id records into one turn, drop late repeats.""" - - def __init__(self) -> None: - """Start with nothing held and no ids seen.""" - self._held: Turn | None = None - self._seen: set[str] = set() - - def push(self, turn: Turn) -> Turn | None: - """Accept one record's turn; return a completed turn if one is ready.""" - held = self._held - if held is not None and turn.message_id and turn.message_id == held.message_id: - self._held = merge(held, turn) - return None - done = self.flush() - if turn.message_id and turn.message_id in self._seen: - return done - if turn.message_id: - self._seen.add(turn.message_id) - self._held = turn - return done - - def flush(self) -> Turn | None: - """Release the held turn, if any.""" - done, self._held = self._held, None - return done diff --git a/python_pkg/token_audit/report.py b/python_pkg/token_audit/report.py deleted file mode 100644 index 4f1c8a07..00000000 --- a/python_pkg/token_audit/report.py +++ /dev/null @@ -1,248 +0,0 @@ -"""Render the audit as markdown plus a machine-readable snapshot. - -Two outputs, two consumers. ``WEEKLY.md`` is for reading; ``weekly.json`` is -for the next run, which diffs against it to produce a week-over-week delta — -the part that makes running this repeatedly worth anything, since a single -week's absolute numbers cannot tell you whether a change helped. - -Both land outside the repo, in ``~/.claude/token-report/``. The report names -paths and tools but never contains prompt or file *content*, so it stays safe -to keep on disk; it is still never committed, because the code repo is public -and the working-directory list alone maps out private projects. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from datetime import UTC, datetime -import json -from pathlib import Path -from typing import TYPE_CHECKING - -from python_pkg.token_audit.attribute import LONG_SESSION_TURNS, Axes, Totals -from python_pkg.token_audit.model import TOKEN_KINDS - -if TYPE_CHECKING: - from collections.abc import Sequence - - from python_pkg.token_audit.model import Session - -REPORT_DIR = Path.home() / ".claude" / "token-report" -MARKDOWN_NAME = "WEEKLY.md" -JSON_NAME = "weekly.json" -TOP_N = 12 - - -@dataclass(frozen=True) -class Window: - """The time span a report covers.""" - - since: float - until: float - - def describe(self) -> str: - """Human-readable window bounds.""" - fmt = "%Y-%m-%d %H:%M" - start = datetime.fromtimestamp(self.since, tz=UTC).strftime(fmt) - end = datetime.fromtimestamp(self.until, tz=UTC).strftime(fmt) - return f"{start} - {end} UTC" - - -def snapshot(totals: Totals, axes: Axes, window: Window, sessions: int) -> dict: - """Build the JSON structure persisted for next week's delta.""" - return { - "generated": datetime.now(tz=UTC).isoformat(), - "window": {"since": window.since, "until": window.until}, - "sessions": sessions, - "turns": totals.turns, - "weighted_total": round(totals.weighted), - "raw": dict(totals.by_kind), - "image_cost": round(axes.image_cost), - "image_reads": axes.images.count, - "cold_start_tokens": int(axes.cold_start.total), - "cold_starts": axes.cold_start.count, - "long_sessions": axes.long_sessions.count, - "long_session_cost": round(axes.long_sessions.total), - "sidechain_cost": round(axes.sidechain.total), - "projects": dict(axes.project.most_common(TOP_N)), - "mcp_calls": dict(axes.mcp_calls), - "skills": dict(axes.skills), - "tool_messages": axes.batching.messages, - "batched_messages": axes.batching.batched, - "batched_calls": axes.batching.calls, - } - - -def _batching_lines(axes: Axes) -> list[str]: - """Render the share of API messages that carried more than one tool call. - - Counted per ``message_id``, not per transcript record: one API message is - written as several records, one per ``tool_use`` block. A per-record count - reports 1.00 calls/message no matter what the model actually did. - """ - batching = axes.batching - if not batching.messages: - return ["_No tool calls in this window._"] - return [ - f"**{batching.batched:,} of {batching.messages:,} tool messages " - f"carried more than one call ({batching.share * 100:.1f}%)** — " - f"{batching.calls:,} calls, {batching.per_message:.2f} per message.", - "", - "Every unbatched call re-sends the whole context to earn one result.", - ] - - -def _pct(part: float, whole: float) -> str: - """Format ``part`` as a percentage of ``whole``, guarding divide-by-zero.""" - return f"{part / whole * 100:.1f}%" if whole else "n/a" - - -def _delta_line(current: dict, previous: dict | None) -> str: - """Describe the change in weighted spend since the previous snapshot.""" - if previous is None: - return "_No previous report on disk - next run will show a delta._" - before = previous.get("weighted_total", 0) - now = current["weighted_total"] - if not before: - return "_Previous report had no spend recorded._" - change = (now - before) / before * 100 - arrow = "up" if change >= 0 else "down" - return f"**Week over week: {arrow} {abs(change):.1f}%** ({before:,} -> {now:,})" - - -def _table(rows: Sequence[tuple[str, str]], headers: tuple[str, str]) -> list[str]: - """Render a two-column markdown table.""" - out = [f"| {headers[0]} | {headers[1]} |", "|---|---|"] - out += [f"| {left} | {right} |" for left, right in rows] - return out - - -def render( - totals: Totals, - axes: Axes, - window: Window, - sessions: Sequence[Session], - current: dict, - previous: dict | None, -) -> str: - """Render the full markdown report.""" - total = totals.weighted - lines = [ - "# Claude Code token audit", - "", - f"Window: {window.describe()}", - f"Sessions: {len(sessions)} | assistant turns: {totals.turns:,}", - "", - _delta_line(current, previous), - "", - "## Totals", - "", - f"**Cost-weighted total: {total:,.0f}**", - "", - ] - lines += _table( - [(kind, f"{totals.by_kind.get(kind, 0):,}") for kind in TOKEN_KINDS], - ("raw token class", "count"), - ) - lines += ["", "## Ranked drivers", ""] - lines += _table(_driver_rows(axes, total), ("driver", "share of weighted cost")) - lines += ["", "## Projects", ""] - lines += _table( - [ - (path, f"{_pct(cost, total)} ({cost:,})") - for path, cost in axes.project.most_common(TOP_N) - ], - ("working directory", "weighted cost"), - ) - lines += ["", "## Tool result payloads (estimated tokens returned)", ""] - lines += _table( - [ - (name, f"{tok:,} over {axes.tools.calls[name]:,} calls") - for name, tok in axes.tools.tokens.most_common(TOP_N) - ], - ("tool", "result tokens"), - ) - lines += ["", "## Tool-call batching", ""] - lines += _batching_lines(axes) - lines += ["", "## MCP servers invoked", ""] - lines += _table( - [(name, f"{n:,}") for name, n in axes.mcp_calls.most_common()] - or [("(none invoked)", "0")], - ("server", "calls"), - ) - lines += ["", "## Skills invoked", ""] - lines += _table( - [(name, f"{n:,}") for name, n in axes.skills.most_common()] - or [("(none invoked)", "0")], - ("skill", "invocations"), - ) - lines += ["", "## Most expensive sessions", ""] - lines += _table(_session_rows(sessions, total), ("session", "weighted cost")) - lines += ["", _footer(axes)] - return "\n".join(lines) + "\n" - - -def _driver_rows(axes: Axes, total: float) -> list[tuple[str, str]]: - """Rows for the headline driver table.""" - long_cost = axes.long_sessions.total - return [ - ( - f"Images re-sent in context ({axes.images.count:,} reads)", - f"{_pct(axes.image_cost, total)} ({axes.image_cost:,.0f})", - ), - ( - f"Sessions over {LONG_SESSION_TURNS} turns ({axes.long_sessions.count})", - f"{_pct(long_cost, total)} ({long_cost:,.0f})", - ), - ( - f"Cold-start prefix ({axes.cold_start.count} starts)", - f"{axes.cold_start.total:,.0f} raw tokens cached", - ), - ( - f"Subagent sidechains ({axes.sidechain.count} turns)", - f"{_pct(axes.sidechain.total, total)} ({axes.sidechain.total:,.0f})", - ), - ] - - -def _session_rows(sessions: Sequence[Session], total: float) -> list[tuple[str, str]]: - """Rows for the per-session table, most expensive first.""" - ranked = sorted(sessions, key=lambda s: s.cost, reverse=True)[:TOP_N] - return [ - ( - f"`{s.session_id[:8]}` ({s.turn_count:,} turns, max ctx {s.max_context:,})", - f"{_pct(s.cost, total)} ({s.cost:,.0f})", - ) - for s in ranked - ] - - -def _footer(axes: Axes) -> str: - """Closing note on how the image figure should be read.""" - return ( - f"_Image cost is an estimate: {axes.images.total:,.0f} tokens were read once, " - "then re-sent as cache-read on every later turn until compaction. " - "It is excluded from the reconciled totals above, which come only from " - "the API's reported usage._" - ) - - -def write(markdown: str, data: dict, directory: Path = REPORT_DIR) -> Path: - """Persist both outputs, returning the markdown path.""" - directory.mkdir(parents=True, exist_ok=True) - md_path = directory / MARKDOWN_NAME - md_path.write_text(markdown, encoding="utf-8") - (directory / JSON_NAME).write_text(json.dumps(data, indent=2), encoding="utf-8") - return md_path - - -def load_previous(directory: Path = REPORT_DIR) -> dict | None: - """Load the last snapshot, or ``None`` if this is the first run.""" - path = directory / JSON_NAME - if not path.exists(): - return None - try: - loaded = json.loads(path.read_text(encoding="utf-8")) - except ValueError: - return None - return loaded if isinstance(loaded, dict) else None diff --git a/python_pkg/token_audit/review.py b/python_pkg/token_audit/review.py deleted file mode 100644 index 8c96fd01..00000000 --- a/python_pkg/token_audit/review.py +++ /dev/null @@ -1,138 +0,0 @@ -"""The "since last time" half of the report: comparison, ledger, candidates. - -Kept apart from :mod:`report` (the per-window tables) because everything here -depends on state outside the window -- the previous snapshot, the ledger, the -registered MCP servers and skills -- and so is exactly the part a re-run must -reproduce from durable files rather than from transcripts that expire. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -from python_pkg.token_audit import history, recommend, surfaces, unused - -if TYPE_CHECKING: - from python_pkg.token_audit.attribute import Axes, Totals - -SNAPSHOT_VERSION = 2 -# Metrics compared run-over-run; all are rates, never window totals, because -# windows differ in length (the "up 122%" of 2026-10-04 compared 37 days to 7). -COMPARED: tuple[str, ...] = ( - "usd_per_commit", - "usd_per_note", - "usd_per_day", - "usd_per_call", - "calls_per_day", - "subagent_usd_share", - "over_budget_usd_share", - "home_cwd_usd_share", - "batching_calls_per_msg", - # Per-day + big-result share, not per-call: a per-call mean rose 41% on - # 2026-10-10 with Bash tokens/day flat, only because cheap calls vanished. - "bash_tokens_per_day", - "bash_big_result_share", - "standing_tokens_est", -) - - -def extra_metrics(totals: Totals, axes: Axes, days: float = 0.0) -> dict[str, float]: - """Metrics that come from the token axes rather than from dollars.""" - bash_calls = axes.tools.calls.get("Bash", 0) - bash_tokens = axes.tools.tokens.get("Bash", 0) - standing = sum(s.est_tokens for s in surfaces.collect()) - return { - "batching_calls_per_msg": round(axes.batching.per_message, 3), - "bash_tokens_per_call": ( - round(axes.tools.tokens.get("Bash", 0) / bash_calls, 1) - if bash_calls - else 0.0 - ), - "bash_tokens_per_day": round(bash_tokens / days, 1) if days else 0.0, - "bash_big_result_share": ( - round(axes.tools.big_tokens["Bash"] / bash_tokens, 4) - if bash_tokens - else 0.0 - ), - "image_usd_share": ( - round(axes.image_cost / totals.weighted, 4) if totals.weighted else 0.0 - ), - "standing_tokens_est": standing, - } - - -def _comparison(now: dict[str, Any], prev: dict[str, Any] | None) -> list[str]: - """Rate-by-rate table against the previous comparable snapshot.""" - if prev is None or prev.get("v", 1) < SNAPSHOT_VERSION: - return [ - "_No comparable previous snapshot (pre-2026-10-04 reports counted " - "every API call ~1.9x and omitted subagents; do not compare against " - "them). The next run will show deltas._", - ] - before = prev.get("metrics", {}) - out = ["| metric | previous | now | change |", "|---|---|---|---|"] - for key in COMPARED: - old, new = before.get(key), now.get(key) - if not isinstance(old, (int, float)) or not isinstance(new, (int, float)): - continue - change = f"{(new - old) / old * 100:+.1f}%" if old else "n/a" - out.append(f"| {key} | {old:,} | {new:,} | {change} |") - return out - - -def _cell(value: float | None) -> str: - """A ledger table cell: the number, or a dash when there is none.""" - return "-" if value is None else str(value) - - -def _ledger(snap: dict[str, Any]) -> list[str]: - """Score every applied lever against the current metrics.""" - verdicts = history.score(history.read_ledger(), snap) - if not verdicts: - return ["_Ledger empty: nothing applied yet._"] - out = [ - "| date-applied lever | metric | baseline | predicted | now | outcome | note |", - "|---|---|---|---|---|---|---|", - ] - out += [ - f"| {v.lever} | {v.metric} | {v.baseline:,} | {_cell(v.predicted)} " - f"| {_cell(v.actual)} | **{v.outcome}** | {v.note} |" - for v in verdicts - ] - return out - - -def candidates( - snap: dict[str, Any], prev: dict[str, Any] | None -) -> list[recommend.Lever]: - """Every rule-based lever that fires for this snapshot.""" - dead_mcp = [u.name for u in unused.mcp_usage() if u.dead] - dead_skills = [u.name for u in unused.skill_usage() if u.dead] - prev_metrics = ( - prev.get("metrics") if prev and prev.get("v", 1) >= SNAPSHOT_VERSION else None - ) - return recommend.candidates(snap["metrics"], prev_metrics, dead_mcp, dead_skills) - - -def render( - snap: dict[str, Any], - prev: dict[str, Any] | None, - *, - levers: bool, - found: list[recommend.Lever] | None = None, -) -> list[str]: - """Markdown for comparison, ledger and (optionally) candidate levers. - - ``found`` is the already-computed candidate list when the caller also - persists it in the snapshot, so the unused-surface scan runs once. - """ - lines = ["## Since the previous analysis", ""] - lines += _comparison(snap["metrics"], prev) - lines += ["", "## What helped (ledger)", ""] - lines += _ledger(snap) - if levers: - if found is None: - found = candidates(snap, prev) - lines += ["", "## Candidate levers (rule-based)", ""] - lines += recommend.render(found) - return lines diff --git a/python_pkg/token_audit/surfaces.py b/python_pkg/token_audit/surfaces.py deleted file mode 100644 index 1c5273b2..00000000 --- a/python_pkg/token_audit/surfaces.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Enumerate every context surface that is billed on every turn. - -The weekly audit kept missing surfaces — plugins and ``autoMode.environment`` -were both found only because the user asked "nothing else?". The fix is that -the list of surfaces lives in code: a surface that is not measured shows up as -an explicit row with ``measured=False`` instead of silently not existing. - -Sizes here are *estimates from disk*. Anything that can be A/B'd against a real -process start should be, via :mod:`python_pkg.token_audit.probe`; disk size and -measured prefix cost disagree (skill descriptions are summarised, MCP servers -contribute only tool names), and the measured number is the one to trust. -""" - -from __future__ import annotations - -from dataclasses import dataclass -import json -from pathlib import Path - -CLAUDE_DIR = Path.home() / ".claude" -GLOBAL_CONFIG = Path.home() / ".claude.json" -SETTINGS = CLAUDE_DIR / "settings.json" -# Anthropic's rough byte-to-token ratio for English prose. -BYTES_PER_TOKEN = 4 - - -@dataclass(frozen=True) -class Surface: - """One always-loaded context component.""" - - name: str - detail: str - bytes_on_disk: int - measurable: bool = False - - @property - def est_tokens(self) -> int: - """Disk size converted to an approximate token count.""" - return self.bytes_on_disk // BYTES_PER_TOKEN - - -def _tree_bytes(root: Path, pattern: str = "*.md") -> int: - """Total size of every file under *root* matching *pattern*.""" - if not root.is_dir(): - return 0 - return sum(p.stat().st_size for p in root.rglob(pattern) if p.is_file()) - - -def _json_field_bytes(path: Path, field: str) -> int: - """Serialized size of one top-level field, or 0 when absent.""" - if not path.exists(): - return 0 - try: - data = json.loads(path.read_text()) - except json.JSONDecodeError, OSError: - return 0 - if field not in data: - return 0 - return len(json.dumps(data[field])) - - -def _skill_description_bytes() -> int: - """Approximate the frontmatter that every skill contributes.""" - total = 0 - for skill in (CLAUDE_DIR / "skills").glob("*/SKILL.md"): - head = skill.read_text(errors="replace")[:4000] - _, _, rest = head.partition("description:") - total += len(rest.split("\n---", 1)[0]) if rest else 0 - return total - - -def _mcp_names(path: Path, field: str = "mcpServers") -> list[str]: - """Server names registered in a config file.""" - if not path.exists(): - return [] - try: - return sorted(json.loads(path.read_text()).get(field, {})) - except json.JSONDecodeError, OSError: - return [] - - -def _enabled_plugins() -> list[str]: - """Plugins switched on in settings.json.""" - if not SETTINGS.exists(): - return [] - try: - plugins = json.loads(SETTINGS.read_text()).get("enabledPlugins", {}) - except json.JSONDecodeError, OSError: - return [] - return sorted(name for name, on in plugins.items() if on) - - -def _project_mcp_files() -> list[Path]: - """Project-scoped .mcp.json files under the home directory. - - These are invisible to ``--strict-mcp-config``, which is exactly how an - earlier audit under-measured the real baseline by ~2,000 tokens/turn. - """ - return sorted(p for p in Path.home().glob("*/.mcp.json") if p.is_file()) - - -def collect() -> list[Surface]: - """Enumerate every known billed surface. - - Adding a surface here is what makes it appear in the report; the weekly - check walks this list rather than a human's memory of it. - """ - memory_dir = CLAUDE_DIR / "projects" / "-home-kuhy" / "memory" - plugins = _enabled_plugins() - project_mcp = _project_mcp_files() - global_mcp = _mcp_names(GLOBAL_CONFIG) - parked = _mcp_names(CLAUDE_DIR / "mcp-parked.json") - return [ - Surface( - "rules/", - "always-on, loaded recursively", - _tree_bytes(CLAUDE_DIR / "rules"), - ), - Surface("memories/", "always-on", _tree_bytes(CLAUDE_DIR / "memories")), - Surface( - "CLAUDE.md", - "global instructions", - (CLAUDE_DIR / "CLAUDE.md").stat().st_size - if (CLAUDE_DIR / "CLAUDE.md").exists() - else 0, - ), - Surface( - "MEMORY.md", - "auto-memory index", - (memory_dir / "MEMORY.md").stat().st_size - if (memory_dir / "MEMORY.md").exists() - else 0, - ), - Surface( - "skill descriptions", - f"{len(list((CLAUDE_DIR / 'skills').glob('*/SKILL.md')))} personal skills", - _skill_description_bytes(), - ), - Surface("plugins", ", ".join(plugins) or "none", 0, measurable=True), - Surface( - "global MCP", - f"{len(global_mcp)} active, {len(parked)} parked", - 0, - measurable=True, - ), - Surface( - "project .mcp.json", - f"{len(project_mcp)} repos (per-project only)", - 0, - measurable=True, - ), - Surface( - "autoMode.environment", - "injected in auto mode", - _json_field_bytes(SETTINGS, "autoMode"), - ), - ] diff --git a/python_pkg/token_audit/tests/__init__.py b/python_pkg/token_audit/tests/__init__.py deleted file mode 100644 index fc6b53a1..00000000 --- a/python_pkg/token_audit/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Tests for the token-audit package.""" diff --git a/python_pkg/token_audit/tests/conftest.py b/python_pkg/token_audit/tests/conftest.py deleted file mode 100644 index 3d3c911f..00000000 --- a/python_pkg/token_audit/tests/conftest.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Keep every token_audit test away from the real ``~/.claude`` state.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -import pytest - -from python_pkg.token_audit import __main__ as cli -from python_pkg.token_audit import history, report, surfaces, unused - -if TYPE_CHECKING: - from pathlib import Path - - -@pytest.fixture(autouse=True) -def _isolated_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """Point HOME and every module-level ``~/.claude`` constant at tmp_path.""" - home = tmp_path / "home" - claude = home / ".claude" - claude.mkdir(parents=True) - monkeypatch.setenv("HOME", str(home)) - monkeypatch.setattr(history, "REPORT_DIR", claude / "token-report") - monkeypatch.setattr(report, "REPORT_DIR", claude / "token-report") - monkeypatch.setattr(cli, "TRANSCRIPT_ROOT", claude / "projects") - monkeypatch.setattr(unused, "TRANSCRIPT_ROOT", claude / "projects") - monkeypatch.setattr(surfaces, "CLAUDE_DIR", claude) - monkeypatch.setattr(surfaces, "GLOBAL_CONFIG", home / ".claude.json") - monkeypatch.setattr(surfaces, "SETTINGS", claude / "settings.json") - return home diff --git a/python_pkg/token_audit/tests/test_attribute.py b/python_pkg/token_audit/tests/test_attribute.py deleted file mode 100644 index c596b1f9..00000000 --- a/python_pkg/token_audit/tests/test_attribute.py +++ /dev/null @@ -1,239 +0,0 @@ -from __future__ import annotations - -import pytest - -from python_pkg.token_audit import attribute -from python_pkg.token_audit.imagecost import image_cost -from python_pkg.token_audit.model import Session, ToolCall, Turn - - -def _turn( - context: int = 0, - out: int = 0, - read: int = 0, - create: int = 0, - *, - sidechain: bool = False, - model: str = "claude-opus-5", -) -> Turn: - return Turn( - usage={ - "output_tokens": out, - "cache_read_input_tokens": read, - "cache_creation_input_tokens": create, - }, - context=context, - model=model, - is_sidechain=sidechain, - ) - - -def _tool_turn(message_id: str = "", tool_calls: int = 0) -> Turn: - """A turn that carried tool calls, for the batching axis. - - Separate from :func:`_turn` because batching cares only about the message - id and the call count -- none of the token fields are relevant to it. - """ - return Turn( - usage={}, - context=0, - model="claude-opus-5", - is_sidechain=False, - message_id=message_id, - tool_calls=tool_calls, - ) - - -def _session( - cwd: str | None = "/home/kuhy/p", - turns: list[Turn] | None = None, - tools: list[ToolCall] | None = None, - image_cost_value: float = 0.0, -) -> Session: - session = Session(session_id="s", path="p", cwd=cwd) - session.turns = turns or [] - session.tools = tools or [] - session.image_cost = image_cost_value - return session - - -def test_totals_sum_each_token_kind() -> None: - session = _session(turns=[_turn(out=2, read=100), _turn(out=1, read=50)]) - totals, _ = attribute.build([session]) - assert totals.by_kind["output_tokens"] == 3 - assert totals.by_kind["cache_read_input_tokens"] == 150 - assert totals.turns == 2 - - -def test_unknown_cwd_is_labelled() -> None: - _, axes = attribute.build([_session(cwd=None, turns=[_turn(out=1)])]) - assert "unknown" in axes.project - - -def test_sidechain_cost_isolated() -> None: - session = _session(turns=[_turn(out=1, sidechain=True), _turn(out=1)]) - _, axes = attribute.build([session]) - assert axes.sidechain.count == 1 - assert axes.sidechain.total == pytest.approx(5.0) - - -def test_tool_mcp_and_skill_axes() -> None: - tools = [ - ToolCall(name="Read", result_tokens=10, path="/a.png"), - ToolCall(name="Read", result_tokens=5, path="/a.py"), - ToolCall(name="mcp__aseprite__draw_line", result_tokens=1), - ToolCall(name="Skill", result_tokens=1, skill="finish"), - ] - _, axes = attribute.build([_session(tools=tools)]) - assert axes.tools.tokens["Read"] == 15 - assert axes.tools.calls["Read"] == 2 - assert axes.mcp_calls["aseprite"] == 1 - assert axes.skills["finish"] == 1 - assert axes.images.count == 1 - assert axes.images.total == 10 - - -def test_long_session_threshold() -> None: - short = _session(turns=[_turn(out=1)]) - long_turns = [_turn(out=1) for _ in range(attribute.LONG_SESSION_TURNS + 1)] - long = _session(turns=long_turns) - _, axes = attribute.build([short, long]) - assert axes.long_sessions.count == 1 - assert axes.long_sessions.total == pytest.approx(len(long_turns) * 5.0) - - -def test_cold_start_only_counted_when_nonzero() -> None: - cold = _session(turns=[_turn(create=1000)]) - warm = _session(turns=[_turn(read=10, create=5)]) - _, axes = attribute.build([cold, warm]) - assert axes.cold_start.count == 1 - assert axes.cold_start.total == 1000 - - -def test_image_cost_is_accumulated_from_sessions() -> None: - _, axes = attribute.build([_session(image_cost_value=7.5)]) - assert axes.image_cost == pytest.approx(7.5) - - -def test_models_axis_tracks_output() -> None: - session = _session(turns=[_turn(out=4, model="claude-sonnet-5")]) - _, axes = attribute.build([session]) - assert axes.models["claude-sonnet-5"] == 4 - - -def test_reconcile_zero_when_consistent() -> None: - session = _session(turns=[_turn(out=1)]) - totals, _ = attribute.build([session]) - assert attribute.reconcile(totals, [session]) == pytest.approx(0.0) - - -def test_reconcile_detects_drift() -> None: - session = _session(turns=[_turn(out=1)]) - totals, _ = attribute.build([session]) - totals.weighted *= 2 - assert attribute.reconcile(totals, [session]) == pytest.approx(0.5) - - -def test_reconcile_guards_zero_total() -> None: - totals = attribute.Totals() - assert attribute.reconcile(totals, []) == 0.0 - - -# --- imagecost --------------------------------------------------------------- - - -def _ev_turn(context: int) -> tuple[str, Turn]: - return ("turn", _turn(context=context)) - - -def _ev_image(tokens: int) -> tuple[str, ToolCall]: - return ("tool", ToolCall(name="Read", result_tokens=tokens, path="/s.png")) - - -def test_image_charged_once_per_following_turn() -> None: - events = [_ev_image(1000), _ev_turn(10), _ev_turn(20), _ev_turn(30)] - assert image_cost(events) == pytest.approx(300.0) - - -def test_image_not_charged_for_earlier_turns() -> None: - events = [_ev_turn(10), _ev_image(1000), _ev_turn(20)] - assert image_cost(events) == pytest.approx(100.0) - - -def test_compaction_clears_live_images() -> None: - # context collapses from 100 to 20 (below the 0.7 ratio) => history dropped - events = [_ev_image(1000), _ev_turn(100), _ev_turn(20), _ev_turn(30)] - assert image_cost(events) == pytest.approx(100.0) - - -def test_growth_below_ratio_does_not_trigger_compaction() -> None: - events = [_ev_image(1000), _ev_turn(100), _ev_turn(90)] - assert image_cost(events) == pytest.approx(200.0) - - -def test_non_image_tools_are_free() -> None: - events = [ - ("tool", ToolCall(name="Bash", result_tokens=9999, path=None)), - ("tool", ToolCall(name="Read", result_tokens=9999, path="/a.py")), - _ev_turn(10), - ] - assert image_cost(events) == 0.0 - - -def test_unknown_event_payloads_are_ignored() -> None: - assert image_cost([("tool", "not a toolcall"), ("turn", "not a turn")]) == 0.0 - - -def test_annotate_sets_session_image_cost() -> None: - from python_pkg.token_audit import imagecost - - session = _session() - imagecost.annotate(session, [_ev_image(100), _ev_turn(5)]) - assert session.image_cost == pytest.approx(10.0) - - -def test_batching_groups_records_sharing_a_message_id() -> None: - """One API message split across records counts once, as a batch. - - This is the whole point of the axis: the transcript writes one record per - ``tool_use`` block, so a per-record count reports 1.00 calls/message and - hides every batch that ever happened. - """ - session = _session( - turns=[ - _tool_turn(message_id="m1", tool_calls=1), - _tool_turn(message_id="m1", tool_calls=1), - _tool_turn(message_id="m1", tool_calls=1), - ], - ) - _, axes = attribute.build([session]) - assert axes.batching.messages == 1 - assert axes.batching.batched == 1 - assert axes.batching.calls == 3 - - -def test_batching_counts_single_call_messages_as_unbatched() -> None: - session = _session( - turns=[ - _tool_turn(message_id="m1", tool_calls=1), - _tool_turn(message_id="m2", tool_calls=1), - ], - ) - _, axes = attribute.build([session]) - assert axes.batching.messages == 2 - assert axes.batching.batched == 0 - assert axes.batching.calls == 2 - - -def test_batching_ignores_turns_with_no_tool_calls_or_no_id() -> None: - """Plain text turns and id-less records must not inflate the denominator.""" - session = _session( - turns=[ - _turn(), - _tool_turn(message_id="m1", tool_calls=0), - _tool_turn(message_id="", tool_calls=2), - ], - ) - _, axes = attribute.build([session]) - assert axes.batching.messages == 0 - assert axes.batching.calls == 0 diff --git a/python_pkg/token_audit/tests/test_breakdown.py b/python_pkg/token_audit/tests/test_breakdown.py deleted file mode 100644 index 4d0024cb..00000000 --- a/python_pkg/token_audit/tests/test_breakdown.py +++ /dev/null @@ -1,138 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from python_pkg.token_audit import breakdown -from python_pkg.token_audit.model import Session, ToolCall, Turn -from python_pkg.token_audit.pricing import Price, PriceBook - -BOOK = PriceBook( - models={ - "claude-opus-5": Price(input=10.0, output=0.0, cache_read=0.0), - "claude-sonnet-5-5": Price(input=2.0, output=0.0, cache_read=0.0), - }, - default="claude-opus-5", -) -MILLION = 1_000_000 - - -def _turn( - model: str = "claude-opus-5", - context: int = 0, - stamp: float = 0.0, - tokens: int = MILLION, -) -> Turn: - return Turn( - usage={"input_tokens": tokens}, - context=context, - model=model, - is_sidechain=False, - timestamp=stamp, - ) - - -def _sessions() -> list[Session]: - main = Session( - "m", - "m.jsonl", - cwd=str(Path.home()), - turns=[ - _turn(stamp=1000.0), - _turn(context=breakdown.CONTEXT_BUDGET + 1, stamp=500.0), - _turn(stamp=2000.0), - ], - tools=[ToolCall("Bash", 1, commit=True), ToolCall("Read", 1)], - ) - sub_opus = Session( - "a", - "a.jsonl", - cwd="/elsewhere", - turns=[_turn()], - parent_id="m", - agent_type="Explore", - description="scan", - ) - sub_sonnet = Session( - "b", - "b.jsonl", - turns=[_turn(model="claude-sonnet-5-5")], - parent_id="m", - ) - return [main, sub_opus, sub_sonnet] - - -def test_bucket_labels() -> None: - assert breakdown._bucket(1) == "1-100" - assert breakdown._bucket(100) == "1-100" - assert breakdown._bucket(101) == "101-200" - assert breakdown._bucket(1500) == "1001+" - - -def test_stamp_ignores_zero_and_tracks_span() -> None: - found = breakdown.Breakdown() - breakdown._stamp(found, 0.0) - assert (found.earliest, found.latest) == (0.0, 0.0) - breakdown._stamp(found, 100.0) - breakdown._stamp(found, 50.0) - assert (found.earliest, found.latest) == (50.0, 100.0) - - -def test_build_rolls_up_every_axis() -> None: - found = breakdown.build(_sessions(), BOOK) - assert found.usd_main == pytest.approx(30.0) - assert found.usd_sub == pytest.approx(10.0 + 2.0) - assert found.usd_total == pytest.approx(42.0) - assert (found.calls_main, found.calls_sub, found.calls) == (3, 2, 5) - assert found.over_budget_calls == 1 - assert found.over_budget_usd == pytest.approx(10.0) - assert found.home_usd == pytest.approx(30.0) - assert found.commits == 1 - assert found.by_agent_type == {"Explore": 10.0, "unknown": 2.0} - assert found.by_description == {"scan": 10.0, "?": 2.0} - assert found.sub_opus_usd == pytest.approx(10.0) - assert found.sub_opus_repriced_usd == pytest.approx(2.0) - assert found.buckets["1-100"].calls == 3 - assert (found.earliest, found.latest) == (500.0, 2000.0) - - -def test_days_has_one_minute_floor() -> None: - assert breakdown.Breakdown().days == 60.0 / breakdown.SECONDS_PER_DAY - - -def test_metrics_for_populated_breakdown() -> None: - m = breakdown.metrics(breakdown.build(_sessions(), BOOK)) - assert m["usd_total"] == 42.0 - assert m["calls"] == 5 - assert m["commits"] == 1 - assert m["usd_per_commit"] == 42.0 - assert m["usd_per_call"] == pytest.approx(8.4) - assert m["subagent_usd_share"] == round(12 / 42, 4) - assert m["subagent_opus_usd_share"] == round(10 / 12, 4) - assert m["opus_usd_share"] == round(40 / 42, 4) - assert m["home_cwd_usd_share"] == round(30 / 42, 4) - assert m["over_budget_call_share"] == 0.2 - assert m["subagent_sonnet_saving_usd"] == 8.0 - - -def test_metrics_for_empty_breakdown_is_all_zero() -> None: - m = breakdown.metrics(breakdown.Breakdown()) - assert m["usd_per_call"] == 0.0 - assert m["usd_per_commit"] == 0.0 - assert m["subagent_usd_share"] == 0.0 - assert m["calls"] == 0 - - -def test_render_has_every_section() -> None: - text = "\n".join(breakdown.render(breakdown.build(_sessions(), BOOK))) - for heading in ( - "## Dollar view", - "### By model", - "### Subagents by type", - "### Costliest subagent tasks", - "### Cost per call by position", - ): - assert heading in text - assert "| claude-opus-5 | $40.00 |" in text - assert "| 1-100 | $10.0000 | 3 |" in text diff --git a/python_pkg/token_audit/tests/test_checkup.py b/python_pkg/token_audit/tests/test_checkup.py deleted file mode 100644 index 7be31e04..00000000 --- a/python_pkg/token_audit/tests/test_checkup.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Tests for the standing-cost checkup report.""" - -from __future__ import annotations - -import pytest - -from python_pkg.token_audit import checkup, surfaces, unused - - -@pytest.fixture(autouse=True) -def stub_sources(monkeypatch: pytest.MonkeyPatch) -> None: - """Replace disk scans with fixed rows so the render is deterministic.""" - monkeypatch.setattr( - surfaces, - "collect", - lambda: [ - surfaces.Surface("rules/", "always-on", 4000), - surfaces.Surface("global MCP", "13 active", 0, measurable=True), - ], - ) - monkeypatch.setattr( - unused, - "mcp_usage", - lambda: [unused.Usage("reaper", 0, 0), unused.Usage("ck3", 0, 9)], - ) - monkeypatch.setattr(unused, "skill_usage", lambda: [unused.Usage("grilling", 0, 0)]) - - -def test_report_lists_every_surface() -> None: - """A surface present in the enumeration reaches the report.""" - text = checkup.build(44311, 1230346533) - assert "rules/" in text - assert "global MCP" in text - - -def test_disk_sized_surface_shows_weighted_share() -> None: - """Shares are weighted, not raw — the unit error an audit made.""" - text = checkup.build(44311, 1230346533) - assert "| 1,000 | 0.36% |" in text - - -def test_unmeasurable_surface_is_labelled() -> None: - """A surface that needs an A/B says so rather than reporting zero bytes.""" - assert "A/B only" in checkup.build(1000, 1000000) - - -def test_only_both_window_zero_is_parked() -> None: - """ck3 (unused 7d, used 30d) must not be recommended for parking.""" - text = checkup.build(44311, 1230346533) - assert "| reaper | 0 | 0 | **park** |" in text - assert "| ck3 | 0 | 9 | keep |" in text - - -def test_skills_section_present() -> None: - """Skills are audited alongside MCP servers.""" - assert "## Skills" in checkup.build(1000, 1000000) - - -def test_main_defaults_and_args(capsys: pytest.CaptureFixture[str]) -> None: - """The CLI accepts turns and total, and defaults when omitted.""" - assert checkup.main([]) == 0 - assert "Standing-cost checkup" in capsys.readouterr().out - assert checkup.main(["100", "1000000"]) == 0 - assert "100 turns" in capsys.readouterr().out diff --git a/python_pkg/token_audit/tests/test_cli.py b/python_pkg/token_audit/tests/test_cli.py deleted file mode 100644 index 770502d6..00000000 --- a/python_pkg/token_audit/tests/test_cli.py +++ /dev/null @@ -1,232 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from typing import TYPE_CHECKING - -import pytest - -from python_pkg.token_audit import __main__ as cli - -if TYPE_CHECKING: - from pathlib import Path - - -def _transcript( - root: Path, - name: str = "s1", - turns: int = 2, - *, - image: bool = False, - stamp: str | None = None, -) -> Path: - project = root / "proj" - project.mkdir(exist_ok=True) - records: list[dict[str, object]] = [] - if image: - records += [ - { - "cwd": "/home/kuhy/demo", - "message": { - "content": [ - { - "type": "tool_use", - "id": "i1", - "name": "Read", - "input": {"file_path": "/shot.png"}, - }, - ], - }, - }, - { - "message": { - "content": [ - { - "type": "tool_result", - "tool_use_id": "i1", - "content": "x" * 400, - }, - ], - }, - }, - ] - for index in range(turns): - records.append( - { - "cwd": "/home/kuhy/demo", - "message": { - "model": "claude-opus-5", - "usage": { - "output_tokens": 10, - "cache_read_input_tokens": 100 * index, - "cache_creation_input_tokens": 50, - }, - }, - }, - ) - if stamp is not None: - records = [r | {"timestamp": stamp} for r in records] - path = project / f"{name}.jsonl" - path.write_text("\n".join(json.dumps(r) for r in records), encoding="utf-8") - return path - - -def test_no_data_exits_three( - tmp_path: Path, capsys: pytest.CaptureFixture[str] -) -> None: - code = cli.main(["--root", str(tmp_path), "--no-write"]) - assert code == cli.EXIT_NO_DATA - assert "No transcripts" in capsys.readouterr().err - - -def test_transcript_without_usage_is_skipped(tmp_path: Path) -> None: - project = tmp_path / "proj" - project.mkdir() - (project / "empty.jsonl").write_text( - '{"message": {"content": []}}', encoding="utf-8" - ) - assert cli.main(["--root", str(tmp_path), "--no-write"]) == cli.EXIT_NO_DATA - - -def test_successful_run_writes_report( - tmp_path: Path, capsys: pytest.CaptureFixture[str] -) -> None: - _transcript(tmp_path, image=True) - out_dir = tmp_path / "out" - code = cli.main(["--root", str(tmp_path), "--out", str(out_dir), "--days", "36500"]) - assert code == 0 - captured = capsys.readouterr() - assert "# Claude Code token audit" in captured.out - assert (out_dir / "WEEKLY.md").exists() - assert (out_dir / "weekly.json").exists() - assert "Analysed 1 sessions" in captured.err - - -def test_no_write_leaves_no_files(tmp_path: Path) -> None: - _transcript(tmp_path) - out_dir = tmp_path / "out" - cli.main( - [ - "--root", - str(tmp_path), - "--out", - str(out_dir), - "--days", - "36500", - "--no-write", - ] - ) - assert not out_dir.exists() - - -def test_json_flag_emits_parsable_snapshot( - tmp_path: Path, capsys: pytest.CaptureFixture[str] -) -> None: - _transcript(tmp_path) - cli.main( - ["--root", str(tmp_path), "--days", "36500", "--no-write", "--json"], - ) - payload = json.loads(capsys.readouterr().out) - assert payload["sessions"] == 1 - assert payload["weighted_total"] > 0 - - -def test_explicit_since_and_until_window(tmp_path: Path) -> None: - _transcript(tmp_path) - code = cli.main( - [ - "--root", - str(tmp_path), - "--since", - "0", - "--until", - "99999999999", - "--no-write", - ], - ) - assert code == 0 - - -def test_until_before_files_yields_no_data(tmp_path: Path) -> None: - _transcript(tmp_path, stamp="2026-01-01T00:00:00+00:00") - code = cli.main( - ["--root", str(tmp_path), "--since", "0", "--until", "1", "--no-write"] - ) - assert code == cli.EXIT_NO_DATA - - -def test_drift_fails_closed( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - _transcript(tmp_path) - monkeypatch.setattr(cli.attribute, "reconcile", lambda *_: 0.5) - code = cli.main(["--root", str(tmp_path), "--days", "36500", "--no-write"]) - assert code == cli.EXIT_DRIFT - assert "RECONCILIATION FAILED" in capsys.readouterr().err - - -def test_second_run_reports_delta( - tmp_path: Path, capsys: pytest.CaptureFixture[str] -) -> None: - _transcript(tmp_path) - out_dir = tmp_path / "out" - args = ["--root", str(tmp_path), "--out", str(out_dir), "--days", "36500"] - cli.main(args) - capsys.readouterr() - # A bare rerun starts where the first ended (nothing new); pin the window. - cli.main([*args, "--since", "0"]) - out = capsys.readouterr().out - assert "## Since the previous analysis" in out - assert "| usd_per_day |" in out - - -def test_parser_defaults() -> None: - args = cli._build_parser().parse_args([]) - assert args.days == cli.DEFAULT_DAYS - assert args.since is None - assert args.json is False - - -def test_module_entrypoint_is_wired() -> None: - assert callable(cli.main) - - -@pytest.mark.parametrize("flag", ["--help"]) -def test_help_exits_zero(flag: str) -> None: - with pytest.raises(SystemExit) as excinfo: - cli.main([flag]) - assert excinfo.value.code == 0 - - -def test_rerun_starts_where_previous_window_ended(tmp_path: Path) -> None: - _transcript(tmp_path) - out_dir = tmp_path / "out" - args = ["--root", str(tmp_path), "--out", str(out_dir), "--days", "36500"] - assert cli.main(args) == 0 - # The first run's window end is now the default start: nothing newer. - assert cli.main(args) == cli.EXIT_NO_DATA - - -def test_recommend_persists_levers_as_structured_snapshot_data( - tmp_path: Path, -) -> None: - root = tmp_path / "projects" - root.mkdir() - _transcript(root, turns=3) - out = tmp_path / "out" - assert cli.main(["--root", str(root), "--out", str(out), "--recommend"]) == 0 - (snap_path,) = (out / "history").glob("*.json") - snap = json.loads(snap_path.read_text(encoding="utf-8")) - assert isinstance(snap["levers"], list) - keys = {"title", "metric", "evidence", "action"} - assert all(keys <= set(lever) for lever in snap["levers"]) - - -def test_without_recommend_no_levers_key(tmp_path: Path) -> None: - root = tmp_path / "projects" - root.mkdir() - _transcript(root, turns=3) - out = tmp_path / "out" - assert cli.main(["--root", str(root), "--out", str(out)]) == 0 - (snap_path,) = (out / "history").glob("*.json") - assert "levers" not in json.loads(snap_path.read_text(encoding="utf-8")) diff --git a/python_pkg/token_audit/tests/test_discover.py b/python_pkg/token_audit/tests/test_discover.py deleted file mode 100644 index 1976cf22..00000000 --- a/python_pkg/token_audit/tests/test_discover.py +++ /dev/null @@ -1,124 +0,0 @@ -from __future__ import annotations - -import json -import os -from pathlib import Path - -from python_pkg.token_audit import discover - - -def _touch(path: Path, mtime: float, text: str = "{}") -> Path: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(text, encoding="utf-8") - os.utime(path, (mtime, mtime)) - return path - - -def test_find_transcripts_prunes_by_mtime_newest_first(tmp_path: Path) -> None: - old = _touch(tmp_path / "p" / "old.jsonl", 1000) - mid = _touch(tmp_path / "p" / "mid.jsonl", 20_000) - new = _touch(tmp_path / "q" / "new.jsonl", 50_000) - assert discover.find_transcripts(tmp_path, since=10_000) == [new, mid] - assert discover.find_transcripts(tmp_path, since=0) == [new, mid, old] - - -def test_find_transcripts_includes_subagent_layout(tmp_path: Path) -> None: - main = _touch(tmp_path / "p" / "s.jsonl", 100) - sub = _touch(tmp_path / "p" / "s" / "subagents" / "agent-1.jsonl", 200) - assert discover.find_transcripts(tmp_path, since=0) == [sub, main] - - -def test_first_cwd_skips_junk_and_cwdless_records(tmp_path: Path) -> None: - lines = ["not json", "[1]", '{"a": 1}', '{"cwd": 5}', '{"cwd": "/x"}'] - path = _touch(tmp_path / "t.jsonl", 1, "\n".join(lines)) - assert discover.first_cwd(path) == "/x" - - -def test_first_cwd_none_when_absent(tmp_path: Path) -> None: - path = _touch(tmp_path / "t.jsonl", 1, '{"a": 1}') - assert discover.first_cwd(path) is None - - -def test_agent_meta_reads_sibling(tmp_path: Path) -> None: - path = _touch(tmp_path / "agent-1.jsonl", 1) - _touch(tmp_path / "agent-1.meta.json", 1, '{"agentType": "Explore"}') - assert discover.agent_meta(path) == {"agentType": "Explore"} - - -def test_agent_meta_empty_when_missing_or_bad(tmp_path: Path) -> None: - path = _touch(tmp_path / "agent-1.jsonl", 1) - assert discover.agent_meta(path) == {} - _touch(tmp_path / "agent-1.meta.json", 1, "garbage") - assert discover.agent_meta(path) == {} - - -def _meta(path: Path, meta: dict[str, object] | str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - text = meta if isinstance(meta, str) else json.dumps(meta) - path.write_text(text, encoding="utf-8") - - -def test_agent_meta_prefers_own_description(tmp_path: Path) -> None: - own = tmp_path / "p" / "s" / "subagents" / "agent-1.meta.json" - _meta(own, {"agentType": "Plan", "description": "mine"}) - _meta( - tmp_path / "p" / "o" / "subagents" / "agent-1.meta.json", {"description": "x"} - ) - path = own.parent / "agent-1.jsonl" - assert discover.agent_meta(path) == {"agentType": "Plan", "description": "mine"} - - -def test_agent_meta_borrows_missing_fields(tmp_path: Path) -> None: - sub = tmp_path / "p" / "new" / "subagents" - _meta(sub / "agent-2.meta.json", {"agentType": "fork", "spawnDepth": 0}) - _meta(tmp_path / "p" / "bad" / "subagents" / "agent-2.meta.json", "not json") - _meta(tmp_path / "p" / "nod" / "subagents" / "agent-2.meta.json", {"x": 1}) - _meta( - tmp_path / "q" / "old" / "subagents" / "agent-2.meta.json", - {"agentType": "general-purpose", "description": "spawned"}, - ) - assert discover.agent_meta(sub / "agent-2.jsonl") == { - "agentType": "fork", - "description": "spawned", - } - - -def test_agent_meta_empty_when_nothing_found(tmp_path: Path) -> None: - path = tmp_path / "p" / "s" / "subagents" / "agent-3.jsonl" - path.parent.mkdir(parents=True) - assert discover.agent_meta(path) == {} - - -def test_first_prompt_skips_non_prompts_and_truncates(tmp_path: Path) -> None: - rows = [ - "junk", - json.dumps({"type": "assistant", "message": {"content": "no"}}), - json.dumps({"type": "user", "message": "flat"}), - json.dumps({"type": "user", "message": {"content": 7}}), - json.dumps( - {"type": "user", "message": {"content": ["s", {"type": "tool_result"}]}} - ), - json.dumps({"type": "user", "message": {"content": " \n"}}), - json.dumps( - { - "type": "user", - "message": {"content": [{"type": "text", "text": "x" * 80}]}, - } - ), - ] - path = _touch(tmp_path / "t.jsonl", 1, "\n".join(rows)) - assert discover.first_prompt(path) == "x" * 59 + "…" - assert discover.first_prompt(path, limit=100) == "x" * 80 - - -def test_first_prompt_text_block_without_text(tmp_path: Path) -> None: - row = {"type": "user", "message": {"content": [{"type": "text"}]}} - path = _touch(tmp_path / "t.jsonl", 1, json.dumps(row)) - assert discover.first_prompt(path) == "" - - -def test_first_prompt_strips_harness_wrappers(tmp_path: Path) -> None: - text = "\nYou are a fork x\n\nFix CI" - row = {"type": "user", "message": {"content": text}} - path = _touch(tmp_path / "t.jsonl", 1, json.dumps(row)) - assert discover.first_prompt(path) == "Fix CI" diff --git a/python_pkg/token_audit/tests/test_history.py b/python_pkg/token_audit/tests/test_history.py deleted file mode 100644 index 674727f7..00000000 --- a/python_pkg/token_audit/tests/test_history.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -from python_pkg.token_audit import history - - -def test_save_snapshot_never_overwrites(tmp_path: Path) -> None: - first = history.save_snapshot({"n": 1}, tmp_path) - second = history.save_snapshot({"n": 2}, tmp_path) - assert first != second - assert json.loads(first.read_text(encoding="utf-8")) == {"n": 1} - assert json.loads(second.read_text(encoding="utf-8")) == {"n": 2} - assert first.parent == tmp_path / history.HISTORY_NAME - - -def test_default_directory_is_redirected_report_dir() -> None: - path = history.save_snapshot({"n": 1}) - assert path.is_relative_to(history.REPORT_DIR) - assert history.last_snapshot() == {"n": 1} - - -def test_last_snapshot_none_without_folder(tmp_path: Path) -> None: - assert history.last_snapshot(tmp_path) is None - assert history.default_since(tmp_path) is None - - -def test_last_snapshot_skips_unreadable_and_non_dict(tmp_path: Path) -> None: - folder = tmp_path / history.HISTORY_NAME - folder.mkdir() - (folder / "2026-01-01.json").write_text('{"ok": true}', encoding="utf-8") - (folder / "2026-01-02.json").write_text("[1]", encoding="utf-8") - (folder / "2026-01-03.json").write_text("garbage", encoding="utf-8") - assert history.last_snapshot(tmp_path) == {"ok": True} - - -def test_last_snapshot_none_when_nothing_usable(tmp_path: Path) -> None: - folder = tmp_path / history.HISTORY_NAME - folder.mkdir() - (folder / "2026-01-01.json").write_text("garbage", encoding="utf-8") - assert history.last_snapshot(tmp_path) is None - - -def test_default_since_reads_window_end(tmp_path: Path) -> None: - history.save_snapshot({"window": {"until": 123.5}}, tmp_path) - assert history.default_since(tmp_path) == 123.5 - - -def test_default_since_none_when_until_not_numeric(tmp_path: Path) -> None: - history.save_snapshot({"window": {"until": "soon"}}, tmp_path) - assert history.default_since(tmp_path) is None - - -def test_read_ledger_missing_file(tmp_path: Path) -> None: - assert history.read_ledger(tmp_path) == [] - - -def test_append_and_read_ledger_filters_bad_lines(tmp_path: Path) -> None: - target = tmp_path / "nested" - history.append_ledger({"metric": "m", "baseline": 1.0}, target) - history.append_ledger( - {"metric": "m", "baseline": 2.0, "date": "2020-01-01"}, target - ) - ledger = target / history.LEDGER_NAME - with ledger.open("a", encoding="utf-8") as handle: - handle.write("garbage\n[1]\n" + json.dumps({"metric": "only"}) + "\n") - entries = history.read_ledger(target) - assert [e["baseline"] for e in entries] == [1.0, 2.0] - assert entries[1]["date"] == "2020-01-01" - assert len(entries[0]["date"]) == len("2026-10-04") - - -def test_append_ledger_default_directory() -> None: - history.append_ledger({"metric": "m", "baseline": 1}) - assert len(history.read_ledger()) == 1 diff --git a/python_pkg/token_audit/tests/test_history_score.py b/python_pkg/token_audit/tests/test_history_score.py deleted file mode 100644 index 989019fe..00000000 --- a/python_pkg/token_audit/tests/test_history_score.py +++ /dev/null @@ -1,69 +0,0 @@ -from __future__ import annotations - -import pytest - -from python_pkg.token_audit import history - - -def _entry(**over: object) -> dict[str, object]: - return {"lever": "L", "metric": "m", "baseline": 100.0, **over} - - -@pytest.mark.parametrize( - ("baseline", "actual", "lower", "outcome"), - [ - (0.0, 5.0, True, "no baseline"), - (100.0, 102.0, True, "no effect"), - (100.0, 50.0, True, "helped"), - (100.0, 150.0, True, "regressed"), - (100.0, 150.0, False, "helped"), - (100.0, 50.0, False, "regressed"), - ], -) -def test_outcome_classes( - baseline: float, actual: float, outcome: str, *, lower: bool -) -> None: - assert history._outcome(baseline, actual, lower_is_better=lower) == outcome - - -def test_prediction_note_variants() -> None: - assert history._prediction_note(100, None, 50) == "" - assert history._prediction_note(100, 100, 50) == "" - assert history._prediction_note(100, 50, 50) == "prediction held" - assert history._prediction_note(100, 50, 100) == "prediction off by 100%" - - -def test_score_measured_entry() -> None: - verdicts = history.score( - [_entry(predicted=50.0)], - {"metrics": {"m": 50}}, - ) - (verdict,) = verdicts - assert verdict.outcome == "helped" - assert verdict.actual == 50.0 - assert verdict.predicted == 50.0 - assert verdict.note == "prediction held" - - -def test_score_higher_is_better_without_prediction() -> None: - (verdict,) = history.score( - [_entry(lower_is_better=False)], - {"metrics": {"m": 200}}, - ) - assert verdict.outcome == "helped" - assert verdict.predicted is None - assert verdict.note == "" - - -def test_score_unmeasured_metric() -> None: - (verdict,) = history.score([_entry(predicted=1)], {"metrics": {}}) - assert verdict.outcome == "unmeasured" - assert verdict.actual is None - assert verdict.predicted == 1.0 - - -def test_score_unmeasured_without_lever_or_metrics_key() -> None: - entry = {"metric": "m", "baseline": 1} - (verdict,) = history.score([entry], {}) - assert verdict.lever == "?" - assert verdict.outcome == "unmeasured" diff --git a/python_pkg/token_audit/tests/test_model.py b/python_pkg/token_audit/tests/test_model.py deleted file mode 100644 index 8608d20c..00000000 --- a/python_pkg/token_audit/tests/test_model.py +++ /dev/null @@ -1,94 +0,0 @@ -from __future__ import annotations - -import pytest - -from python_pkg.token_audit.model import ( - COMPACTION_DROP_RATIO, - WEIGHTS, - Session, - ToolCall, - Turn, - weighted, -) - - -def test_weights_price_output_highest() -> None: - assert WEIGHTS["output_tokens"] > WEIGHTS["input_tokens"] - assert WEIGHTS["cache_read_input_tokens"] < WEIGHTS["input_tokens"] - assert 0 < COMPACTION_DROP_RATIO < 1 - - -def test_weighted_applies_each_multiplier() -> None: - usage = { - "input_tokens": 10, - "cache_creation_input_tokens": 100, - "cache_read_input_tokens": 1000, - "output_tokens": 1, - } - assert weighted(usage) == pytest.approx(10 + 125 + 100 + 5) - - -def test_weighted_treats_missing_kinds_as_zero() -> None: - assert weighted({}) == 0.0 - - -def test_turn_cost_delegates_to_weighted() -> None: - turn = Turn(usage={"output_tokens": 2}, context=0, model="m", is_sidechain=False) - assert turn.cost == pytest.approx(10.0) - - -@pytest.mark.parametrize("path", ["/a/b.png", "/a/b.PNG", "/a/b.jpeg"]) -def test_tool_call_detects_images(path: str) -> None: - assert ToolCall(name="Read", result_tokens=1, path=path).is_image - - -@pytest.mark.parametrize("path", ["/a/b.py", "/a/noextension", None]) -def test_tool_call_rejects_non_images(path: str | None) -> None: - assert not ToolCall(name="Read", result_tokens=1, path=path).is_image - - -@pytest.mark.parametrize( - ("name", "expected"), - [ - ("mcp__aseprite__draw_line", "aseprite"), - ("mcp__i3wm__get_tree", "i3wm"), - ("Read", None), - ("mcp__broken", None), - ], -) -def test_tool_call_mcp_server(name: str, expected: str | None) -> None: - assert ToolCall(name=name, result_tokens=0).mcp_server == expected - - -def _turn(context: int, out: int = 0, read: int = 0, create: int = 0) -> Turn: - return Turn( - usage={ - "output_tokens": out, - "cache_read_input_tokens": read, - "cache_creation_input_tokens": create, - }, - context=context, - model="m", - is_sidechain=False, - ) - - -def test_session_aggregates() -> None: - session = Session(session_id="s", path="p") - session.turns = [_turn(100, out=1), _turn(500, out=2)] - assert session.turn_count == 2 - assert session.max_context == 500 - assert session.cost == pytest.approx(15.0) - - -def test_session_empty_has_zero_max_context() -> None: - assert Session(session_id="s", path="p").max_context == 0 - - -def test_cold_start_counts_only_turns_with_no_cache_read() -> None: - session = Session(session_id="s", path="p") - session.turns = [ - _turn(0, create=5000), # cold: nothing to read yet - _turn(9, read=9, create=100), # warm: excluded - ] - assert session.cold_start_tokens == 5000 diff --git a/python_pkg/token_audit/tests/test_notes_backlog.py b/python_pkg/token_audit/tests/test_notes_backlog.py deleted file mode 100644 index 8bdb7916..00000000 --- a/python_pkg/token_audit/tests/test_notes_backlog.py +++ /dev/null @@ -1,94 +0,0 @@ -"""BACKLOG parsing and done dates recovered from the Firebase dumps.""" - -from __future__ import annotations - -from datetime import UTC, datetime -from typing import TYPE_CHECKING - -from python_pkg.token_audit import notes_backlog -from python_pkg.token_audit.tests import test_notes_fixtures as fx - -if TYPE_CHECKING: - from pathlib import Path - - -def _ts(iso: str) -> float: - return datetime.fromisoformat(iso).replace(tzinfo=UTC).timestamp() - - -def test_parse_reads_ids_status_title(tmp_path: Path) -> None: - text = fx.backlog(tmp_path / "BACKLOG.md").read_text() - notes = notes_backlog.parse_backlog(text) - assert [n.id for n in notes] == [fx.NOTE_A, fx.NOTE_B, fx.NOTE_C, fx.NOTE_D] - assert notes[0].status == "done" - assert notes[0].title == "diet guard meal times picker" - assert notes[1].status == "abandoned" - assert notes[0].updated is not None - - -def test_unknown_status_falls_back_to_todo() -> None: - text = '\nbody' - assert notes_backlog.parse_backlog(text)[0].status == "todo" - - -def test_hlc_time_parses_prefix_and_rejects_junk() -> None: - assert notes_backlog.hlc_time("2026-10-01T10:00:00.000Z-0000-dev") == _ts( - "2026-10-01T10:00:00" - ) - assert notes_backlog.hlc_time("nonsense") is None - assert notes_backlog.hlc_time(None) is None - - -def test_done_at_is_status_hlc_of_first_dump_showing_done(tmp_path: Path) -> None: - dumps = tmp_path / "dumps" - fx.dump(dumps, "2026-10-01T000000Z", {fx.NOTE_A: (1, "2026-09-30T08:00:00.000Z")}) - fx.dump(dumps, "2026-10-02T000000Z", {fx.NOTE_A: (2, "2026-10-01T12:00:00.000Z")}) - # Re-stamped later by an unrelated upsert: must not move the done date. - fx.dump(dumps, "2026-10-03T000000Z", {fx.NOTE_A: (2, "2026-10-02T20:00:00.000Z")}) - notes = notes_backlog.load_notes(fx.backlog(tmp_path / "B.md"), dumps) - done = {n.id: n.done_at for n in notes} - assert done[fx.NOTE_A] == _ts("2026-10-01T12:00:00") - - -def test_device_with_newest_hlc_wins(tmp_path: Path) -> None: - dumps = tmp_path / "dumps" - path = fx.dump(dumps, "2026-10-02T000000Z", {}) - first = notes_backlog.merge_devices( - { - "d1": '{"n": {"fields": {"status": [2, "2026-10-01T00:00:00.000Z-0-d1"]}}}', - "d2": '{"n": {"fields": {"status": [0, "2026-09-01T00:00:00.000Z-0-d2"]}}}', - "d3": '{"gone": {"deleted": true, "fields": {"status": [2, "x"]}}}', - "d4": "not json", - } - ) - assert first == {"n": (2, "2026-10-01T00:00:00.000Z-0-d1")} - assert path.exists() - - -def test_abandoned_dated_from_dump_and_todo_has_no_date(tmp_path: Path) -> None: - dumps = tmp_path / "dumps" - fx.dump(dumps, "2026-10-05T000000Z", {fx.NOTE_B: (3, "2026-10-04T09:00:00.000Z")}) - notes = { - n.id: n for n in notes_backlog.load_notes(fx.backlog(tmp_path / "B.md"), dumps) - } - assert notes[fx.NOTE_B].done_at == _ts("2026-10-04T09:00:00") - assert notes[fx.NOTE_C].done_at is None - - -def test_fallback_to_backlog_updated_when_no_dump_has_it(tmp_path: Path) -> None: - notes = { - n.id: n - for n in notes_backlog.load_notes( - fx.backlog(tmp_path / "B.md"), tmp_path / "none" - ) - } - assert notes[fx.NOTE_D].done_at == notes[fx.NOTE_D].updated - assert notes[fx.NOTE_D].done_at is not None - - -def test_unreadable_dump_is_skipped(tmp_path: Path) -> None: - dumps = tmp_path / "dumps" - dumps.mkdir() - (dumps / "kuhy-syncs-2026-10-01T000000Z.json.gz").write_bytes(b"not gzip") - notes = notes_backlog.load_notes(fx.backlog(tmp_path / "B.md"), dumps) - assert len(notes) == 4 diff --git a/python_pkg/token_audit/tests/test_notes_cli.py b/python_pkg/token_audit/tests/test_notes_cli.py deleted file mode 100644 index 0a665d5f..00000000 --- a/python_pkg/token_audit/tests/test_notes_cli.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Notes metrics wired into the audit CLI, and lifetime session pricing.""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from python_pkg.token_audit import __main__ as cli -from python_pkg.token_audit import notes, notes_cost, pricing -from python_pkg.token_audit.tests import test_notes_fixtures as fx - - -def _session(root: Path) -> None: - fx.transcript( - root, - "s1", - [ - fx.prompt(fx.BODY_A), - fx.tool_use("Bash", {"command": "ls"}, msg="m1"), - ], - ) - fx.subagent(root, "s1", "a1", [fx.tool_use("Read", {"file_path": "/x"}, msg="m2")]) - - -def test_lifetime_spend_sums_main_and_subagents(tmp_path: Path) -> None: - root = tmp_path / "projects" - _session(root) - book = pricing.load() - got = notes_cost.lifetime({"s1": root / "proj" / "s1.jsonl"}, {"s1"}, book, {}) - assert got["s1"].tokens == 2000 - assert got["s1"].usd > 0 - reused = notes_cost.lifetime( - {"s1": root / "proj" / "s1.jsonl"}, - {"s1"}, - book, - {str(root / "proj" / "s1.jsonl"): notes_cost.Spend(1.0, 7)}, - ) - assert reused["s1"].tokens == 1007 - - -def test_audit_writes_notes_section_and_json( - tmp_path: Path, capsys: pytest.CaptureFixture[str] -) -> None: - root = tmp_path / "projects" - _session(root) - fx.backlog(notes.backlog_path_for(Path.home())) - out = tmp_path / "out" - code = cli.main(["--root", str(root), "--out", str(out), "--days", "36500"]) - assert code == 0 - assert "## Cost per todo note" in (out / "WEEKLY.md").read_text() - snap = json.loads((out / "weekly.json").read_text()) - assert snap["metrics"]["notes_done"] == 2 # A and D (2026-09-01) - assert snap["metrics"]["note_linked_usd_share"] == 1.0 - assert snap["metrics"]["usd_per_note"] > 0 - data = json.loads((out / "notes.json").read_text()) - assert data["notes"][0]["id"] == fx.NOTE_A - assert "notes:" in capsys.readouterr().err - - -def test_audit_skips_notes_without_backlog( - tmp_path: Path, capsys: pytest.CaptureFixture[str] -) -> None: - root = tmp_path / "projects" - _session(root) - out = tmp_path / "out" - assert cli.main(["--root", str(root), "--out", str(out), "--days", "36500"]) == 0 - assert "notes: skipped" in capsys.readouterr().err - assert not (out / "notes.json").exists() - - -def test_standalone_entry_writes_notes_json(tmp_path: Path) -> None: - root = tmp_path / "projects" - _session(root) - fx.backlog(notes.backlog_path_for(Path.home())) - out = tmp_path / "out" - assert notes.main(["--root", str(root), "--out", str(out)]) == 0 - assert (out / "notes.json").exists() - assert notes.main(["--root", str(tmp_path / "missing"), "--out", str(out)]) == 0 diff --git a/python_pkg/token_audit/tests/test_notes_edges.py b/python_pkg/token_audit/tests/test_notes_edges.py deleted file mode 100644 index 17d53fcc..00000000 --- a/python_pkg/token_audit/tests/test_notes_edges.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Edge cases of the note join that the rule tests do not reach.""" - -from __future__ import annotations - -from pathlib import Path - -from python_pkg.token_audit import ( - notes, - notes_backlog, - notes_collect, - notes_link, - notes_scan, - pricing, -) -from python_pkg.token_audit.model import Session -from python_pkg.token_audit.tests import test_notes_fixtures as fx - -HANDOFF = "/home/kuhy/.claude/handoff/e.md" - - -def test_prompt_text_rejects_tool_results_and_non_text() -> None: - result = fx.tool_result("x") - assert notes_scan.prompt_text(result) is None - empty = fx.prompt("x") - empty["message"]["content"] = None - assert notes_scan.prompt_text(empty) is None - - -def test_bad_backlog_stamp_and_malformed_status_field() -> None: - text = '\nbody' - assert notes_backlog.parse_backlog(text)[0].updated is None - odd = '{"n": {"fields": {"status": [2]}}}' - assert notes_backlog.merge_devices({"d": odd}) == {} - - -def test_non_handoff_writes_and_repeat_writes_are_ignored(tmp_path: Path) -> None: - root = tmp_path / "projects" - fx.transcript( - root, - "w1", - [ - fx.tool_use("Write", {"file_path": "/home/kuhy/notes/x.md"}, msg="m1"), - fx.tool_use("Write", {"file_path": HANDOFF}, "2026-10-20T09:00:00Z", "m2"), - fx.tool_use("Write", {"file_path": HANDOFF}, "2026-10-20T10:00:00Z", "m3"), - ], - ) - fx.subagent(root, "orphan", "z", [fx.tool_use("Write", {"file_path": HANDOFF})]) - facts = notes_scan.scan_root(root) - assert set(facts) == {"w1"} - assert list(facts["w1"].creates) == ["e.md"] - - -def test_unlinked_writer_passes_nothing_on(tmp_path: Path) -> None: - root = tmp_path / "projects" - fx.transcript( - root, - "w1", - [ - fx.prompt("nothing matching at all here", "2026-10-20T08:00:00Z"), - fx.tool_use("Write", {"file_path": HANDOFF}, "2026-10-20T08:10:00Z"), - ], - ) - fx.transcript( - root, "r1", [fx.prompt(f"Implement {HANDOFF}", "2026-10-20T09:00:00Z")] - ) - notes_list = notes_backlog.parse_backlog(fx.backlog(tmp_path / "B.md").read_text()) - assert notes_link.link_all(notes_scan.scan_root(root), notes_list) == {} - - -def test_sessions_started_before_window_are_not_reused(tmp_path: Path) -> None: - root = tmp_path / "projects" - path = fx.transcript(root, "s1", [fx.prompt("x", "2026-10-01T00:00:00Z")]) - facts = notes_scan.scan_root(root) - sessions = [Session("s1", str(path)), Session("ghost", "/nowhere")] - reuse = notes_collect._reusable(sessions, facts, 2e9, pricing.load()) - assert reuse == {} - - -def test_standalone_entry_without_backlog_writes_nothing(tmp_path: Path) -> None: - out = tmp_path / "out" - assert notes.main(["--root", str(tmp_path), "--out", str(out)]) == 0 - assert not out.exists() diff --git a/python_pkg/token_audit/tests/test_notes_fixtures.py b/python_pkg/token_audit/tests/test_notes_fixtures.py deleted file mode 100644 index 71361ebe..00000000 --- a/python_pkg/token_audit/tests/test_notes_fixtures.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Builders for fixture transcripts, BACKLOG files and Firebase dumps.""" - -from __future__ import annotations - -import gzip -import json -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from pathlib import Path - -NOTE_A = "aaaaaaaa-1111-4111-8111-000000000001" -NOTE_B = "bbbbbbbb-2222-4222-8222-000000000002" -NOTE_C = "cccccccc-3333-4333-8333-000000000003" -NOTE_D = "dddddddd-4444-4444-8444-000000000004" - -BODY_A = ( - "diet guard meal times picker\n\n" - "allow user to pick meal hours in intervals of fifteen minutes " - "instead of typing them, remember the last chosen catering option" -) -BODY_B = ( - "spellcode audio from first frame\n\n" - "play hub music and ambience immediately when the godot game opens, " - "no silent splash screen, crossfade between rooms" -) -BODY_C = "book guard volume buttons\n\nvolume keys turn reader pages forward backward" -BODY_D = "horror game" - - -def backlog(path: Path, notes: list[tuple[str, str, str, str]] | None = None) -> Path: - """Write a BACKLOG.md with ``(id, status, updated, body)`` notes.""" - notes = notes or [ - (NOTE_A, "done", "2026-10-20T10:00:00.000", BODY_A), - (NOTE_B, "abandoned", "2026-10-21T10:00:00.000", BODY_B), - (NOTE_C, "todo", "2026-10-22T10:00:00.000", BODY_C), - (NOTE_D, "done", "2026-09-01T10:00:00.000", BODY_D), - ] - path.parent.mkdir(parents=True, exist_ok=True) - parts = ["", ""] - for note_id, status, updated, body in notes: - parts += [ - f'', - body, - "", - ] - path.write_text("\n".join(parts), encoding="utf-8") - return path - - -def dump(directory: Path, stamp: str, statuses: dict[str, tuple[int, str]]) -> Path: - """Write one ``kuhy-syncs-.json.gz`` with ``id -> (status, hlc)``.""" - directory.mkdir(parents=True, exist_ok=True) - device = { - note_id: { - "id": note_id, - "fields": {"status": [status, f"{hlc}-0000-dev1"]}, - "deleted": False, - "deleted_hlc": None, - } - for note_id, (status, hlc) in statuses.items() - } - data = {"todo-sync": {"notes": {"dev1~2Ejson": json.dumps(device)}}} - path = directory / f"kuhy-syncs-{stamp}.json.gz" - path.write_bytes(gzip.compress(json.dumps(data).encode())) - return path - - -def prompt(text: str, stamp: str = "2026-10-20T09:00:00Z") -> dict[str, Any]: - """A typed user prompt record.""" - return { - "type": "user", - "timestamp": stamp, - "cwd": "/home/kuhy/demo", - "message": {"role": "user", "content": text}, - } - - -def tool_use( - name: str, - tool_input: dict[str, Any], - stamp: str = "2026-10-20T09:01:00Z", - msg: str = "m1", - usd_tokens: int = 1000, -) -> dict[str, Any]: - """An assistant record carrying one tool_use and a usage block.""" - return { - "type": "assistant", - "timestamp": stamp, - "message": { - "id": msg, - "model": "claude-opus-5-5", - "usage": {"input_tokens": 0, "output_tokens": usd_tokens}, - "content": [ - { - "type": "tool_use", - "id": f"t-{msg}", - "name": name, - "input": tool_input, - } - ], - }, - } - - -def tool_result(text: str, msg: str = "m1") -> dict[str, Any]: - """A tool_result record (not a prompt).""" - return { - "type": "user", - "timestamp": "2026-10-20T09:02:00Z", - "message": { - "content": [ - {"type": "tool_result", "tool_use_id": f"t-{msg}", "content": text} - ] - }, - } - - -def _write(path: Path, records: list[dict[str, Any]]) -> None: - lines = (json.dumps(r, separators=(",", ":")) for r in records) - path.write_text("\n".join(lines), encoding="utf-8") - - -def transcript( - root: Path, session: str, records: list[dict[str, Any]], slug: str = "proj" -) -> Path: - """Write ``//.jsonl``.""" - project = root / slug - project.mkdir(parents=True, exist_ok=True) - path = project / f"{session}.jsonl" - path.write_text( - "\n".join(json.dumps(r, separators=(",", ":")) for r in records), - encoding="utf-8", - ) - return path - - -def subagent( - root: Path, session: str, agent: str, records: list[dict[str, Any]] -) -> Path: - """Write ``/proj//subagents/agent-.jsonl``.""" - directory = root / "proj" / session / "subagents" - directory.mkdir(parents=True, exist_ok=True) - path = directory / f"agent-{agent}.jsonl" - path.write_text( - "\n".join(json.dumps(r, separators=(",", ":")) for r in records), - encoding="utf-8", - ) - return path diff --git a/python_pkg/token_audit/tests/test_notes_link_explicit.py b/python_pkg/token_audit/tests/test_notes_link_explicit.py deleted file mode 100644 index 49981596..00000000 --- a/python_pkg/token_audit/tests/test_notes_link_explicit.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Rule (a): a Bash tool_use running note_claim.sh links the session.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from python_pkg.token_audit import notes_backlog, notes_link, notes_scan -from python_pkg.token_audit.tests import test_notes_fixtures as fx - -if TYPE_CHECKING: - from pathlib import Path - - -def _notes(tmp_path: Path) -> list[notes_backlog.Note]: - return notes_backlog.parse_backlog(fx.backlog(tmp_path / "B.md").read_text()) - - -def test_claim_with_prefix_links_note(tmp_path: Path) -> None: - root = tmp_path / "projects" - fx.transcript( - root, - "s1", - [ - fx.prompt("do the thing"), - fx.tool_use( - "Bash", {"command": "~/.claude/scripts/note_claim.sh bbbbbbbb"} - ), - ], - ) - links = notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)) - assert links["s1"] == notes_link.Link((fx.NOTE_B,), "a") - - -def test_claim_beats_image_and_fuzzy(tmp_path: Path) -> None: - root = tmp_path / "projects" - fx.transcript( - root, - "s1", - [ - fx.prompt(f"{fx.BODY_A}\n![x](todo-images/{fx.NOTE_C}/a.png)"), - fx.tool_use("Bash", {"command": f"note_claim.sh {fx.NOTE_B}"}), - ], - ) - links = notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)) - assert links["s1"].rule == "a" - assert links["s1"].notes == (fx.NOTE_B,) - - -def test_unknown_or_ambiguous_claim_is_ignored(tmp_path: Path) -> None: - root = tmp_path / "projects" - fx.transcript( - root, - "s1", - [ - fx.prompt("hello there general"), - fx.tool_use("Bash", {"command": "note_claim.sh ffffffff"}), - ], - ) - notes = [ - *_notes(tmp_path), - notes_backlog.Note("aaaaaaaa-9999", "dup", "dup", "low", "todo", None), - ] - facts = notes_scan.scan_root(root) - assert "s1" not in notes_link.link_all(facts, notes) - assert notes_link.resolve("aaaaaaaa", [n.id for n in notes]) is None - - -def test_claim_in_tool_result_is_not_a_link(tmp_path: Path) -> None: - root = tmp_path / "projects" - ids = f"{fx.NOTE_A} {fx.NOTE_B} {fx.NOTE_C}" - fx.transcript( - root, - "s1", - [ - fx.prompt("read my backlog please now"), - fx.tool_use("Read", {"file_path": "/home/kuhy/src/todo/BACKLOG.md"}), - fx.tool_result(f"note_claim.sh {fx.NOTE_A} {ids}"), - ], - ) - assert notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)) == {} - - -def test_claim_inside_subagent_does_not_count(tmp_path: Path) -> None: - root = tmp_path / "projects" - fx.transcript(root, "s1", [fx.prompt("hello there general")]) - fx.subagent( - root, - "s1", - "x", - [fx.tool_use("Bash", {"command": f"note_claim.sh {fx.NOTE_A}"})], - ) - assert notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)) == {} diff --git a/python_pkg/token_audit/tests/test_notes_link_fuzzy.py b/python_pkg/token_audit/tests/test_notes_link_fuzzy.py deleted file mode 100644 index 0a6988a6..00000000 --- a/python_pkg/token_audit/tests/test_notes_link_fuzzy.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Rule (c): the first real prompt fuzzy-matched to one note's text.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from python_pkg.token_audit import notes_backlog, notes_link, notes_scan -from python_pkg.token_audit.tests import test_notes_fixtures as fx - -if TYPE_CHECKING: - from pathlib import Path - - -def _notes(tmp_path: Path) -> list[notes_backlog.Note]: - return notes_backlog.parse_backlog(fx.backlog(tmp_path / "B.md").read_text()) - - -def _link(tmp_path: Path, *texts: str) -> notes_link.Link | None: - root = tmp_path / "projects" - fx.transcript(root, "s1", [fx.prompt(t) for t in texts]) - return notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)).get("s1") - - -def test_pasted_note_body_links(tmp_path: Path) -> None: - pasted = f'\n{fx.BODY_B}\n\nplease' - assert _link(tmp_path, pasted) == notes_link.Link((fx.NOTE_B,), "c") - - -def test_title_only_prompt_links_by_first_line(tmp_path: Path) -> None: - link = _link(tmp_path, "diet guard meal times picker") - assert link == notes_link.Link((fx.NOTE_A,), "c") - - -def test_unrelated_prompt_does_not_link(tmp_path: Path) -> None: - assert _link(tmp_path, "restart the steam backlog enforcer while game runs") is None - - -def test_only_first_prompt_is_matched(tmp_path: Path) -> None: - assert _link(tmp_path, "unrelated opening words here now", fx.BODY_A) is None - - -def test_short_prompt_is_never_matched(tmp_path: Path) -> None: - assert _link(tmp_path, "horror game") is None - - -def test_wrappers_and_caveats_are_skipped(tmp_path: Path) -> None: - link = _link( - tmp_path, - "/clear", - "Caveat: the messages below were generated by the user", - f"noise{fx.BODY_A}", - ) - assert link == notes_link.Link((fx.NOTE_A,), "c") - - -def test_list_content_prompt_is_read(tmp_path: Path) -> None: - root = tmp_path / "projects" - record = fx.prompt("") - record["message"]["content"] = [{"type": "text", "text": fx.BODY_B}] - fx.transcript(root, "s1", [record]) - links = notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)) - assert links["s1"].notes == (fx.NOTE_B,) - - -def test_score_is_symmetric_dice_or_title() -> None: - note = notes_backlog.Note("n", "alpha beta gamma delta", "x", "low", "todo", None) - assert notes_link.score("alpha beta gamma delta", note) == 1.0 - assert notes_link.score("zeta eta theta iota", note) == 0.0 diff --git a/python_pkg/token_audit/tests/test_notes_link_handoff.py b/python_pkg/token_audit/tests/test_notes_link_handoff.py deleted file mode 100644 index 7a1bf352..00000000 --- a/python_pkg/token_audit/tests/test_notes_link_handoff.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Rule (d): an Implement/Continue handoff session inherits its writer's notes.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from python_pkg.token_audit import notes_backlog, notes_link, notes_scan -from python_pkg.token_audit.tests import test_notes_fixtures as fx - -if TYPE_CHECKING: - from pathlib import Path - -HANDOFF = "/home/kuhy/.claude/handoff/sub/20261020-x.md" -HANDOFF2 = "/home/kuhy/.claude/handoff/20261021-y.md" - - -def _notes(tmp_path: Path) -> list[notes_backlog.Note]: - return notes_backlog.parse_backlog(fx.backlog(tmp_path / "B.md").read_text()) - - -def _writer(root: Path) -> None: - fx.transcript( - root, - "w1", - [ - fx.prompt(fx.BODY_A, "2026-10-20T08:00:00Z"), - fx.tool_use( - "Write", - {"file_path": HANDOFF, "content": "x"}, - "2026-10-20T08:30:00Z", - ), - ], - ) - - -def test_continuation_inherits_writer_notes(tmp_path: Path) -> None: - root = tmp_path / "projects" - _writer(root) - fx.transcript( - root, "r1", [fx.prompt(f"Implement {HANDOFF}", "2026-10-20T09:00:00Z")] - ) - links = notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)) - assert links["w1"].rule == "c" - assert links["r1"] == notes_link.Link((fx.NOTE_A,), "d") - - -def test_chain_follows_through_bash_heredoc_writer(tmp_path: Path) -> None: - root = tmp_path / "projects" - _writer(root) - fx.transcript( - root, - "r1", - [ - fx.prompt(f"Implement {HANDOFF}", "2026-10-20T09:00:00Z"), - fx.tool_use( - "Bash", - {"command": "cat > ~/.claude/handoff/20261021-y.md <<'EOF'\nx\nEOF"}, - "2026-10-20T10:00:00Z", - ), - ], - ) - fx.transcript( - root, "r2", [fx.prompt(f"continue {HANDOFF2}", "2026-10-20T11:00:00Z")] - ) - links = notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)) - assert links["r2"] == notes_link.Link((fx.NOTE_A,), "d") - - -def test_subagent_writer_maps_to_parent(tmp_path: Path) -> None: - root = tmp_path / "projects" - fx.transcript(root, "w1", [fx.prompt(fx.BODY_B, "2026-10-20T08:00:00Z")]) - fx.subagent( - root, - "w1", - "a1", - [fx.tool_use("Write", {"file_path": HANDOFF}, "2026-10-20T08:10:00Z")], - ) - fx.transcript( - root, "r1", [fx.prompt(f"Implement {HANDOFF}", "2026-10-20T09:00:00Z")] - ) - links = notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)) - assert links["r1"] == notes_link.Link((fx.NOTE_B,), "d") - - -def test_appender_is_not_the_writer_and_unknown_handoff_unlinked( - tmp_path: Path, -) -> None: - root = tmp_path / "projects" - fx.transcript( - root, - "x1", - [ - fx.prompt(fx.BODY_B, "2026-10-20T07:00:00Z"), - fx.tool_use( - "Bash", {"command": f"cat >> {HANDOFF} <<'EOF'"}, "2026-10-20T07:10:00Z" - ), - ], - ) - fx.transcript( - root, "r1", [fx.prompt(f"Implement {HANDOFF}", "2026-10-20T09:00:00Z")] - ) - assert "r1" not in notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)) - - -def test_write_after_reader_started_is_ignored(tmp_path: Path) -> None: - root = tmp_path / "projects" - fx.transcript( - root, "r1", [fx.prompt(f"Implement {HANDOFF}", "2026-10-20T07:00:00Z")] - ) - _writer(root) - assert "r1" not in notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)) - - -def test_handoff_prompt_is_not_fuzzy_matched(tmp_path: Path) -> None: - root = tmp_path / "projects" - fx.transcript( - root, - "r1", - [fx.prompt(f"Implement /home/kuhy/.claude/handoff/{fx.BODY_A.split()[0]}.md")], - ) - assert notes_scan.scan_root(root)["r1"].handoff == f"{fx.BODY_A.split()[0]}.md" - assert "r1" not in notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)) diff --git a/python_pkg/token_audit/tests/test_notes_link_images.py b/python_pkg/token_audit/tests/test_notes_link_images.py deleted file mode 100644 index ea8f3ac0..00000000 --- a/python_pkg/token_audit/tests/test_notes_link_images.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Rule (b): a typed user prompt naming todo-images// links the note.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from python_pkg.token_audit import notes_backlog, notes_link, notes_scan -from python_pkg.token_audit.tests import test_notes_fixtures as fx - -if TYPE_CHECKING: - from pathlib import Path - - -def _notes(tmp_path: Path) -> list[notes_backlog.Note]: - return notes_backlog.parse_backlog(fx.backlog(tmp_path / "B.md").read_text()) - - -def test_image_path_in_later_prompt_links(tmp_path: Path) -> None: - root = tmp_path / "projects" - fx.transcript( - root, - "s1", - [ - fx.prompt("something unrelated entirely here"), - fx.prompt(f"see ~/data/todo-images/{fx.NOTE_C}/shot.png"), - ], - ) - links = notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)) - assert links["s1"] == notes_link.Link((fx.NOTE_C,), "b") - - -def test_image_only_in_tool_result_does_not_link(tmp_path: Path) -> None: - root = tmp_path / "projects" - fx.transcript( - root, - "s1", - [ - fx.prompt("something unrelated entirely here"), - fx.tool_use("Bash", {"command": "ls"}), - fx.tool_result(f"todo-images/{fx.NOTE_C}/shot.png"), - ], - ) - assert notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)) == {} - - -def test_three_distinct_images_look_like_an_export(tmp_path: Path) -> None: - root = tmp_path / "projects" - refs = " ".join(f"todo-images/{n}/x.png" for n in (fx.NOTE_A, fx.NOTE_B, fx.NOTE_C)) - fx.transcript(root, "s1", [fx.prompt(f"zzz qqq www {refs}")]) - assert notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)) == {} - - -def test_two_images_split_and_unknown_uuid_dropped(tmp_path: Path) -> None: - root = tmp_path / "projects" - unknown = "eeeeeeee-0000-4000-8000-000000000000" - fx.transcript( - root, - "s1", - [ - fx.prompt( - f"zzz todo-images/{fx.NOTE_A}/x.png todo-images/{fx.NOTE_C}/y.png " - f"todo-images/{unknown}/z.png" - ) - ], - ) - links = notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)) - assert links["s1"] == notes_link.Link((fx.NOTE_A, fx.NOTE_C), "b") - - -def test_meta_and_sidechain_prompts_are_ignored(tmp_path: Path) -> None: - root = tmp_path / "projects" - meta = fx.prompt(f"todo-images/{fx.NOTE_C}/x.png") | {"isMeta": True} - side = fx.prompt(f"todo-images/{fx.NOTE_C}/x.png") | {"isSidechain": True} - fx.transcript(root, "s1", [meta, side, fx.prompt("plain words only here")]) - assert notes_link.link_all(notes_scan.scan_root(root), _notes(tmp_path)) == {} diff --git a/python_pkg/token_audit/tests/test_notes_metrics.py b/python_pkg/token_audit/tests/test_notes_metrics.py deleted file mode 100644 index d3d5b97e..00000000 --- a/python_pkg/token_audit/tests/test_notes_metrics.py +++ /dev/null @@ -1,112 +0,0 @@ -"""USD per completed note: the split, the window rules and the outputs.""" - -from __future__ import annotations - -from datetime import UTC, datetime -import json -from typing import TYPE_CHECKING - -from python_pkg.token_audit import notes, notes_backlog, notes_cost -from python_pkg.token_audit.notes_link import Link -from python_pkg.token_audit.tests import test_notes_fixtures as fx - -if TYPE_CHECKING: - from pathlib import Path - -NOTE_E = "eeeeeeee-5555-4555-8555-000000000005" - - -def _ts(iso: str) -> float: - return datetime.fromisoformat(iso).replace(tzinfo=UTC).timestamp() - - -def _note(note_id: str, status: str, done: str | None) -> notes_backlog.Note: - stamp = _ts(done) if done else None - title = f"title of {note_id[:4]} " + "x" * 80 - return notes_backlog.Note(note_id, title, title, "high", status, stamp, stamp) - - -NOTES = [ - _note(fx.NOTE_A, "done", "2026-10-20T00:00:00"), - _note(fx.NOTE_B, "abandoned", "2026-10-21T00:00:00"), - _note(fx.NOTE_C, "todo", None), - _note(fx.NOTE_D, "done", "2026-09-01T00:00:00"), - _note(NOTE_E, "done", "2026-10-22T00:00:00"), -] -LINKS = { - "s1": Link((fx.NOTE_A,), "a"), - "s2": Link((fx.NOTE_A, fx.NOTE_B), "b"), - "s3": Link((fx.NOTE_C,), "c"), -} -SPEND = { - "s1": notes_cost.Spend(10.0, 1000), - "s2": notes_cost.Spend(4.0, 400), - "s3": notes_cost.Spend(5.0, 500), -} -WINDOW = (_ts("2026-10-15T00:00:00"), _ts("2026-10-25T00:00:00")) - - -def test_rows_split_session_spend_equally() -> None: - rows = {r.id: r for r in notes.note_rows(NOTES, LINKS, SPEND)} - assert rows[fx.NOTE_A].usd == 12.0 - assert rows[fx.NOTE_A].tokens == 1200 - assert rows[fx.NOTE_A].sessions == ["s1", "s2"] - assert rows[fx.NOTE_B].usd == 2.0 - assert NOTE_E not in rows - - -def test_metrics_window_math() -> None: - rows = notes.note_rows(NOTES, LINKS, SPEND) - window_usd = {"s1": 10.0, "s2": 4.0, "s3": 5.0, "s4": 1.0} - got = notes.metrics(NOTES, rows, WINDOW, window_usd, set(LINKS)) - # (A 12 + abandoned B 2) / done-and-linked {A}; E is done but unlinked. - assert got == { - "usd_per_note": 14.0, - "note_linked_usd_share": 0.95, - "notes_done": 2, - "notes_done_linked": 1, - } - - -def test_no_linked_done_note_leaves_usd_per_note_unmeasured() -> None: - got = notes.metrics(NOTES[2:], [], WINDOW, {}, set()) - assert "usd_per_note" not in got - assert got["note_linked_usd_share"] == 0.0 - assert got["notes_done"] == 1 - - -def test_render_has_metrics_and_truncated_table() -> None: - rows = notes.note_rows(NOTES, LINKS, SPEND) - got = notes.metrics(NOTES, rows, WINDOW, {"s1": 1.0}, set(LINKS)) - text = "\n".join(notes.render(rows, got, {"a": 1, "c": 2})) - assert "## Cost per todo note" in text - assert "usd_per_note" in text - assert "a=1" in text - assert f"title of {fx.NOTE_A[:4]} " + "x" * 45 + "…" in text - assert "| abandoned |" in text - - -def test_render_without_measurement_says_so() -> None: - text = "\n".join( - notes.render([], {"notes_done": 0, "note_linked_usd_share": 0.0}, {}) - ) - assert "usd_per_note: n/a" in text - - -def test_write_json_lists_every_linked_note(tmp_path: Path) -> None: - rows = notes.note_rows(NOTES, LINKS, SPEND) - path = notes.write_json(rows, tmp_path) - data = json.loads(path.read_text()) - assert {r["id"] for r in data["notes"]} == {fx.NOTE_A, fx.NOTE_B, fx.NOTE_C} - first = data["notes"][0] - assert set(first) == { - "id", - "title", - "status", - "priority", - "done_at", - "sessions", - "usd", - "tokens", - } - assert first["usd"] == 12.0 diff --git a/python_pkg/token_audit/tests/test_parse.py b/python_pkg/token_audit/tests/test_parse.py deleted file mode 100644 index d5a3c395..00000000 --- a/python_pkg/token_audit/tests/test_parse.py +++ /dev/null @@ -1,208 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from typing import TYPE_CHECKING - -from python_pkg.token_audit import parse - -if TYPE_CHECKING: - from pathlib import Path - - import pytest - - -def _assistant( - usage: dict[str, int] | None = None, - content: list[dict[str, object]] | None = None, - cwd: str = "/home/kuhy/demo", - *, - sidechain: bool = False, -) -> dict[str, object]: - message: dict[str, object] = {"model": "claude-opus-5"} - if usage is not None: - message["usage"] = usage - if content is not None: - message["content"] = content - return { - "type": "assistant", - "cwd": cwd, - "isSidechain": sidechain, - "message": message, - } - - -def _write(tmp_path: Path, records: list[object], name: str = "sess.jsonl") -> Path: - path = tmp_path / name - path.write_text( - "\n".join(json.dumps(r) if isinstance(r, dict) else r for r in records), - encoding="utf-8", - ) - return path - - -def test_load_session_reads_usage_and_cwd(tmp_path: Path) -> None: - path = _write(tmp_path, [_assistant(usage={"output_tokens": 3})]) - session = parse.load_session(path) - assert session.cwd == "/home/kuhy/demo" - assert session.turn_count == 1 - assert session.turns[0].model == "claude-opus-5" - - -def test_context_is_cache_read_plus_creation(tmp_path: Path) -> None: - usage = {"cache_read_input_tokens": 40, "cache_creation_input_tokens": 2} - path = _write(tmp_path, [_assistant(usage=usage)]) - assert parse.load_session(path).turns[0].context == 42 - - -def test_malformed_lines_are_skipped_not_fatal(tmp_path: Path) -> None: - path = _write( - tmp_path, - [ - "{not json", - "[1,2,3]", - "null", - json.dumps(_assistant(usage={"output_tokens": 1})), - ], - ) - assert parse.load_session(path).turn_count == 1 - - -def test_tool_result_is_joined_to_its_tool_use(tmp_path: Path) -> None: - records = [ - _assistant( - content=[ - { - "type": "tool_use", - "id": "t1", - "name": "Read", - "input": {"file_path": "/x/shot.png"}, - }, - ], - ), - { - "message": { - "content": [ - {"type": "tool_result", "tool_use_id": "t1", "content": "abcdefgh"} - ] - } - }, - ] - session = parse.load_session(_write(tmp_path, records)) - assert len(session.tools) == 1 - call = session.tools[0] - assert call.name == "Read" - assert call.path == "/x/shot.png" - assert call.is_image - - -def test_orphan_tool_result_is_ignored(tmp_path: Path) -> None: - records = [ - {"message": {"content": [{"type": "tool_result", "tool_use_id": "nope"}]}} - ] - assert parse.load_session(_write(tmp_path, records)).tools == [] - - -def test_skill_input_is_captured(tmp_path: Path) -> None: - records = [ - _assistant( - content=[ - { - "type": "tool_use", - "id": "s1", - "name": "Skill", - "input": {"skill": "finish"}, - }, - ], - ), - { - "message": { - "content": [ - {"type": "tool_result", "tool_use_id": "s1", "content": "ok"} - ] - } - }, - ] - assert parse.load_session(_write(tmp_path, records)).tools[0].skill == "finish" - - -def test_tool_use_without_id_or_dict_input_is_tolerated(tmp_path: Path) -> None: - records = [ - _assistant(content=[{"type": "tool_use", "name": "Read"}]), - _assistant( - content=[ - {"type": "tool_use", "id": "t2", "name": "Bash", "input": "notadict"} - ], - ), - {"message": {"content": [{"type": "tool_result", "tool_use_id": "t2"}]}}, - ] - session = parse.load_session(_write(tmp_path, records)) - assert session.tools[0].name == "Bash" - assert session.tools[0].path is None - assert session.tools[0].result_tokens == 0 - - -def test_non_dict_messages_and_content_are_ignored(tmp_path: Path) -> None: - records = [ - {"message": "a string"}, - {"message": {"content": "not a list"}}, - {"message": {"usage": "not a dict"}}, - {"no_message": True}, - {"message": {"content": ["not a dict block"]}}, - ] - session = parse.load_session(_write(tmp_path, records)) - assert session.turn_count == 0 - assert session.tools == [] - - -def test_model_falls_back_to_unknown(tmp_path: Path) -> None: - records = [{"message": {"usage": {"output_tokens": 1}}}] - assert parse.load_session(_write(tmp_path, records)).turns[0].model == "unknown" - - -def test_non_string_model_falls_back(tmp_path: Path) -> None: - records = [{"message": {"usage": {"output_tokens": 1}, "model": 42}}] - assert parse.load_session(_write(tmp_path, records)).turns[0].model == "unknown" - - -def test_tool_use_with_non_string_id_is_dropped(tmp_path: Path) -> None: - records = [ - _assistant( - content=[{"type": "tool_use", "id": 99, "name": "Read", "input": {}}] - ), - {"message": {"content": [{"type": "tool_result", "tool_use_id": 99}]}}, - ] - assert parse.load_session(_write(tmp_path, records)).tools == [] - - -def test_unrecognised_block_types_are_ignored(tmp_path: Path) -> None: - records = [_assistant(content=[{"type": "text", "text": "hello"}])] - session = parse.load_session(_write(tmp_path, records)) - assert session.tools == [] - assert session.turn_count == 0 - - -def test_load_session_ignores_events_of_other_types( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - path = _write(tmp_path, [_assistant(usage={"output_tokens": 1})]) - monkeypatch.setattr(parse, "iter_events", lambda *_: iter([("turn", "not a Turn")])) - session = parse.load_session(path) - assert session.turns == [] - assert session.tools == [] - - -def test_cwd_none_when_absent(tmp_path: Path) -> None: - records = [{"message": {"usage": {"output_tokens": 1}}}] - assert parse.load_session(_write(tmp_path, records)).cwd is None - - -def test_usage_keeps_only_integer_fields(tmp_path: Path) -> None: - usage = {"output_tokens": 5, "service_tier": "standard", "iterations": [{}]} - session = parse.load_session(_write(tmp_path, [_assistant(usage=usage)])) - assert session.turns[0].usage == {"output_tokens": 5} - - -def test_sidechain_flag_is_preserved(tmp_path: Path) -> None: - path = _write(tmp_path, [_assistant(usage={"output_tokens": 1}, sidechain=True)]) - assert parse.load_session(path).turns[0].is_sidechain diff --git a/python_pkg/token_audit/tests/test_parse_events.py b/python_pkg/token_audit/tests/test_parse_events.py deleted file mode 100644 index e603dfef..00000000 --- a/python_pkg/token_audit/tests/test_parse_events.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Tests for token_audit event iteration and transcript discovery.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import TYPE_CHECKING - -import pytest - -from python_pkg.token_audit import parse -from python_pkg.token_audit.model import ToolCall, Turn - -if TYPE_CHECKING: - from pathlib import Path - - -def _assistant( - usage: dict[str, int] | None = None, - content: list[dict[str, object]] | None = None, - cwd: str = "/home/kuhy/demo", - *, - sidechain: bool = False, -) -> dict[str, object]: - message: dict[str, object] = {"model": "claude-opus-5"} - if usage is not None: - message["usage"] = usage - if content is not None: - message["content"] = content - return { - "type": "assistant", - "cwd": cwd, - "isSidechain": sidechain, - "message": message, - } - - -def _write(tmp_path: Path, records: list[object], name: str = "sess.jsonl") -> Path: - path = tmp_path / name - path.write_text( - "\n".join(json.dumps(r) if isinstance(r, dict) else r for r in records), - encoding="utf-8", - ) - return path - - -def test_iter_events_preserves_order(tmp_path: Path) -> None: - records = [ - _assistant( - content=[ - { - "type": "tool_use", - "id": "a", - "name": "Read", - "input": {"file_path": "/i.png"}, - }, - ], - ), - { - "message": { - "content": [ - {"type": "tool_result", "tool_use_id": "a", "content": "xxxx"} - ] - } - }, - _assistant(usage={"output_tokens": 1}), - ] - kinds = [k for k, _ in parse.iter_events(_write(tmp_path, records))] - assert kinds == ["tool", "turn"] - - -def test_events_carry_expected_types(tmp_path: Path) -> None: - records = [ - _assistant( - content=[ - {"type": "tool_use", "id": "a", "name": "Read", "input": {}}, - ], - ), - {"message": {"content": [{"type": "tool_result", "tool_use_id": "a"}]}}, - _assistant(usage={"output_tokens": 1}), - ] - events = dict(parse.iter_events(_write(tmp_path, records))) - assert isinstance(events["tool"], ToolCall) - assert isinstance(events["turn"], Turn) - - -@pytest.mark.parametrize("payload", [None, "abcdefgh"]) -def test_result_tokens_handles_missing_content(payload: str | None) -> None: - assert parse._result_tokens({"content": payload}) >= 0 diff --git a/python_pkg/token_audit/tests/test_parse_subagent.py b/python_pkg/token_audit/tests/test_parse_subagent.py deleted file mode 100644 index 43c4102d..00000000 --- a/python_pkg/token_audit/tests/test_parse_subagent.py +++ /dev/null @@ -1,103 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -from python_pkg.token_audit import parse -from python_pkg.token_audit.model import Turn - - -def _line(record: dict[str, object]) -> str: - return json.dumps(record) - - -def test_tool_result_flushes_held_turn_in_order(tmp_path: Path) -> None: - use = {"type": "tool_use", "id": "t1", "name": "Bash", "input": {}} - records = [ - { - "message": { - "id": "m1", - "model": "claude-opus-5", - "usage": {"output_tokens": 2}, - "content": [use], - }, - }, - {"message": {"content": [{"type": "tool_result", "tool_use_id": "t1"}]}}, - ] - path = tmp_path / "s.jsonl" - path.write_text("\n".join(_line(r) for r in records), encoding="utf-8") - kinds = [kind for kind, _ in parse.iter_events(path)] - assert kinds == ["turn", "tool"] - - -def test_load_session_tags_subagent(tmp_path: Path) -> None: - sub = tmp_path / "proj" / "sess-1" / "subagents" - sub.mkdir(parents=True) - path = sub / "agent-a.jsonl" - record = {"message": {"model": "claude-opus-5", "usage": {"output_tokens": 1}}} - path.write_text(_line(record), encoding="utf-8") - (sub / "agent-a.meta.json").write_text( - json.dumps({"agentType": "Explore", "description": "find things"}), - encoding="utf-8", - ) - session = parse.load_session(path) - assert session.parent_id == "sess-1" - assert session.agent_type == "Explore" - assert session.description == "find things" - assert isinstance(session.turns[0], Turn) - - -def test_load_session_subagent_without_meta_defaults(tmp_path: Path) -> None: - sub = tmp_path / "proj" / "sess-2" / "subagents" - sub.mkdir(parents=True) - path = sub / "agent-b.jsonl" - path.write_text("{}", encoding="utf-8") - session = parse.load_session(path) - assert session.agent_type == "unknown" - assert session.description == "(no label) agent-b" - - -def test_load_session_borrows_label_across_clear(tmp_path: Path) -> None: - """An agent that outlived ``/clear`` keeps its spawning session's label.""" - old = tmp_path / "proj" / "old" / "subagents" - new = tmp_path / "proj" / "new" / "subagents" - old.mkdir(parents=True) - new.mkdir(parents=True) - (old / "agent-c.meta.json").write_text( - json.dumps({"agentType": "Explore", "description": "map the repo"}), - encoding="utf-8", - ) - path = new / "agent-c.jsonl" - path.write_text("{}", encoding="utf-8") - session = parse.load_session(path) - assert session.agent_type == "Explore" - assert session.description == "map the repo" - - -def test_load_session_falls_back_to_first_prompt(tmp_path: Path) -> None: - sub = tmp_path / "proj" / "sess-3" / "subagents" - sub.mkdir(parents=True) - path = sub / "agent-d.jsonl" - prompt: dict[str, object] = { - "type": "user", - "message": {"content": "\nAudit the hooks\nmore"}, - } - path.write_text(_line(prompt), encoding="utf-8") - assert parse.load_session(path).description == "Audit the hooks" - - -def test_window_filters_records_by_timestamp(tmp_path: Path) -> None: - def rec(stamp: str, mid: str) -> dict[str, object]: - return { - "timestamp": stamp, - "message": {"id": mid, "usage": {"output_tokens": 1}}, - } - - path = tmp_path / "s.jsonl" - rows = [ - rec("1970-01-01T00:00:10+00:00", "a"), - rec("1970-01-01T00:01:40+00:00", "b"), - ] - path.write_text("\n".join(_line(r) for r in rows), encoding="utf-8") - session = parse.load_session(path, (50.0, 200.0)) - assert [t.message_id for t in session.turns] == ["b"] diff --git a/python_pkg/token_audit/tests/test_pricing.py b/python_pkg/token_audit/tests/test_pricing.py deleted file mode 100644 index 929153f2..00000000 --- a/python_pkg/token_audit/tests/test_pricing.py +++ /dev/null @@ -1,70 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from python_pkg.token_audit import pricing -from python_pkg.token_audit.model import Turn - -BOOK = pricing.PriceBook( - models={ - "claude-opus-5": pricing.Price(input=5.0, output=25.0, cache_read=0.5), - "claude-opus-5-5": pricing.Price(input=4.0, output=20.0, cache_read=0.2), - }, - default="claude-opus-5", -) - - -def _turn(usage: dict[str, int], model: str = "claude-opus-5") -> Turn: - return Turn(usage=usage, context=0, model=model, is_sidechain=False) - - -def test_for_model_exact_prefix_and_default() -> None: - assert BOOK.for_model("claude-opus-5").input == 5.0 - # Longest matching known id wins over a shorter prefix. - assert BOOK.for_model("claude-opus-5-5-2026").input == 4.0 - assert BOOK.for_model("mystery").input == 5.0 - - -def test_load_reads_shipped_table() -> None: - book = pricing.load() - assert book.default in book.models - assert book.for_model("claude-sonnet-5-5").cache_read == 0.2 - - -def test_load_custom_path(tmp_path: Path) -> None: - path = tmp_path / "p.json" - path.write_text( - json.dumps( - { - "default": "m", - "models": {"m": {"input": 1, "output": 2, "cache_read": 3}}, - }, - ), - encoding="utf-8", - ) - book = pricing.load(path) - assert book.models["m"] == pricing.Price(1.0, 2.0, 3.0) - - -def test_usd_without_split_prices_creation_as_5m() -> None: - usage = {"input_tokens": 1_000_000, "cache_creation_input_tokens": 1_000_000} - assert pricing.usd(_turn(usage), BOOK) == pytest.approx(5.0 + 1.25 * 5.0) - - -def test_usd_with_split_prices_1h_at_double() -> None: - usage = { - "cache_creation_input_tokens": 1_000_000, - "cache_creation_1h": 400_000, - "cache_creation_5m": 600_000, - "cache_read_input_tokens": 1_000_000, - "output_tokens": 1_000_000, - } - expected = 0.4 * 2.0 * 5.0 + 0.6 * 1.25 * 5.0 + 0.5 + 25.0 - assert pricing.usd(_turn(usage), BOOK) == pytest.approx(expected) - - -def test_usd_zero_for_empty_usage() -> None: - assert pricing.usd(_turn({}), BOOK) == 0.0 diff --git a/python_pkg/token_audit/tests/test_probe.py b/python_pkg/token_audit/tests/test_probe.py deleted file mode 100644 index 4d29e25c..00000000 --- a/python_pkg/token_audit/tests/test_probe.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Tests for the prefix probe and its prediction gate.""" - -from __future__ import annotations - -import json -import subprocess -from typing import TYPE_CHECKING - -import pytest - -from python_pkg.token_audit import probe - -if TYPE_CHECKING: - from pathlib import Path - - -def test_prefix_sums_cold_start() -> None: - """A cold start reports the prefix as cache creation.""" - usage = { - "input_tokens": 2, - "cache_creation_input_tokens": 46434, - "cache_read_input_tokens": 0, - } - assert probe._prefix_from_usage(usage) == 46436 - - -def test_prefix_sums_warm_start() -> None: - """A warm start reports the same prefix as cache read.""" - usage = {"input_tokens": 2, "cache_read_input_tokens": 46434} - assert probe._prefix_from_usage(usage) == 46436 - - -def test_prefix_handles_missing_and_null() -> None: - """Absent or null token classes count as zero.""" - assert probe._prefix_from_usage({}) == 0 - assert probe._prefix_from_usage({"input_tokens": None}) == 0 - - -def test_prediction_within_tolerance_passes() -> None: - """A prediction close to the measurement is not flagged.""" - assert probe.check_prediction(1000, 900) is None - - -def test_prediction_beyond_tolerance_is_flagged() -> None: - """The 6,122-vs-4,465 miss from a real audit must fail the gate.""" - warning = probe.check_prediction(6122, 4465) - assert warning is not None - assert "27%" in warning - - -def test_prediction_ignores_nonpositive() -> None: - """No prediction means nothing to check.""" - assert probe.check_prediction(0, 500) is None - - -def test_weighted_pct_applies_cache_read_weight() -> None: - """Prefix savings are billed as cache reads, at weight 0.1.""" - assert probe.weighted_pct(4465, 44311, 1230346533) == pytest.approx(1.608, abs=0.01) - - -def test_weighted_pct_guards_zero_total() -> None: - """An empty week cannot divide by zero.""" - assert probe.weighted_pct(100, 10, 0) == 0.0 - - -def test_measure_runs_non_strict_by_default( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """Strict mode hides project .mcp.json, so it must be opt-in.""" - seen: dict[str, list[str]] = {} - - def fake_run(cmd: list[str], **_: object) -> subprocess.CompletedProcess[str]: - seen["cmd"] = cmd - return subprocess.CompletedProcess( - cmd, 0, json.dumps({"usage": {"input_tokens": 5}}), "" - ) - - monkeypatch.setattr(subprocess, "run", fake_run) - result = probe.measure("base", mcp_config=tmp_path / "x.json") - assert "--strict-mcp-config" not in seen["cmd"] - assert result.prefix_tokens == 5 - - -def test_measure_can_opt_into_strict( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """Strict mode is still reachable when deliberately requested.""" - seen: dict[str, list[str]] = {} - - def fake_run(cmd: list[str], **_: object) -> subprocess.CompletedProcess[str]: - seen["cmd"] = cmd - return subprocess.CompletedProcess(cmd, 0, json.dumps({"usage": {}}), "") - - monkeypatch.setattr(subprocess, "run", fake_run) - probe.measure("strict", mcp_config=tmp_path / "x.json", strict=True) - assert "--strict-mcp-config" in seen["cmd"] - - -def test_measure_without_mcp_config_omits_flags( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """The plain probe passes no MCP flags at all.""" - seen: dict[str, list[str]] = {} - - def fake_run(cmd: list[str], **_: object) -> subprocess.CompletedProcess[str]: - seen["cmd"] = cmd - return subprocess.CompletedProcess(cmd, 0, json.dumps({"usage": {}}), "") - - monkeypatch.setattr(subprocess, "run", fake_run) - probe.measure("plain") - assert "--mcp-config" not in seen["cmd"] diff --git a/python_pkg/token_audit/tests/test_recommend.py b/python_pkg/token_audit/tests/test_recommend.py deleted file mode 100644 index df0ba856..00000000 --- a/python_pkg/token_audit/tests/test_recommend.py +++ /dev/null @@ -1,108 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from python_pkg.token_audit import recommend - - -def _fire_all() -> dict[str, Any]: - return { - "usd_total": 70.0, - "usd_per_day": 10.0, - "subagent_usd_share": 0.4, - "subagent_opus_usd_share": 0.9, - "subagent_sonnet_saving_usd": 14.0, - "over_budget_usd_share": 0.5, - "home_cwd_usd_share": 0.4, - "batching_calls_per_msg": 1.05, - "bash_big_result_share": 0.4, - "bash_tokens_per_day": 900_000, - "standing_tokens_est": 120, - "image_usd_share": 0.1, - } - - -def test_num_treats_missing_and_non_numeric_as_zero() -> None: - assert recommend._num({"a": 3}, "a") == 3.0 - assert recommend._num({"a": "x"}, "a") == 0.0 - assert recommend._num({}, "a") == 0.0 - - -def test_nothing_fires_on_quiet_metrics() -> None: - assert recommend.candidates({}, None, [], []) == [] - assert recommend.render([]) == ["_No rule fired._"] - - -def test_subagent_rule_needs_both_thresholds() -> None: - low_share = {"subagent_usd_share": 0.01, "subagent_opus_usd_share": 0.9} - low_opus = {"subagent_usd_share": 0.5, "subagent_opus_usd_share": 0.1} - assert recommend._subagents(low_share) == [] - assert recommend._subagents(low_opus) == [] - - -def test_subagent_lever_sizes_saving_per_day() -> None: - (lever,) = recommend._subagents(_fire_all()) - assert lever.category == "cheaper agents" - # 70 / 10 = 7 days; 14 saved over 7 days = 2 per day. - assert lever.est_usd_per_day == 2.0 - - -def test_every_rule_fires_and_sized_levers_sort_first() -> None: - previous = {"standing_tokens_est": 100} - levers = recommend.candidates(_fire_all(), previous, ["mcp1"], ["skill1"]) - titles = [lv.title for lv in levers] - assert titles[0] == "Calls over the context budget" - assert set(titles) == { - "Opus runs most subagent work", - "Calls over the context budget", - "Sessions started in ~", - "Independent tool calls not batched", - "Large Bash results", - "Always-loaded context grew", - "Images held in context", - "Unused MCP servers / skills", - } - sized = [lv.est_usd_per_day for lv in levers if lv.est_usd_per_day] - assert sized == sorted(sized, reverse=True) - - -def test_prefix_rule_silent_without_growth_or_baseline() -> None: - flat = {"standing_tokens_est": 100} - assert recommend._mechanics(flat, {"standing_tokens_est": 100}) == [] - assert recommend._mechanics(flat, {"standing_tokens_est": 0}) == [] - assert recommend._mechanics(flat, None) == [] - - -def test_dead_lists_render_dash_for_empty_side() -> None: - assert recommend._dead([], []) == [] - (only_mcp,) = recommend._dead(["m"], []) - assert "MCP: m; skills: -" in only_mcp.evidence - (only_skill,) = recommend._dead([], ["s"]) - assert "MCP: -; skills: s" in only_skill.evidence - - -def test_render_table_formats_estimates() -> None: - sized = recommend.Lever("c", "t", "e", "a", 1.5) - unsized = recommend.Lever("c", "t2", "e", "a") - lines = recommend.render([sized, unsized]) - assert lines[0].startswith("| category") - assert "| $1.50 |" in lines[2] - assert "| - |" in lines[3] - - -def test_every_rule_title_maps_to_a_snapshot_metric() -> None: - metrics = { - "subagent_usd_share": 0.9, - "subagent_opus_usd_share": 0.9, - "over_budget_usd_share": 0.9, - "home_cwd_usd_share": 0.9, - "batching_calls_per_msg": 1.0, - "bash_big_result_share": 0.9, - "image_usd_share": 0.9, - "standing_tokens_est": 200, - "usd_total": 10, - "usd_per_day": 1, - } - found = recommend.candidates(metrics, {"standing_tokens_est": 100}, ["m"], []) - assert len(found) == len(recommend.TITLE_METRIC) - assert all(lv.metric == recommend.TITLE_METRIC[lv.title] for lv in found) diff --git a/python_pkg/token_audit/tests/test_records.py b/python_pkg/token_audit/tests/test_records.py deleted file mode 100644 index b58956d9..00000000 --- a/python_pkg/token_audit/tests/test_records.py +++ /dev/null @@ -1,113 +0,0 @@ -from __future__ import annotations - -import pytest - -from python_pkg.token_audit import records -from python_pkg.token_audit.model import Turn - - -def _turn( - message_id: str = "m1", - usage: dict[str, int] | None = None, - context: int = 0, - tool_calls: int = 0, -) -> Turn: - return Turn( - usage=usage or {}, - context=context, - model="claude-opus-5", - is_sidechain=False, - message_id=message_id, - tool_calls=tool_calls, - ) - - -@pytest.mark.parametrize( - "record", - [{}, {"timestamp": 5}, {"timestamp": "not a date"}], -) -def test_timestamp_none_for_missing_or_bad(record: dict[str, object]) -> None: - assert records.timestamp(record) is None - - -def test_timestamp_parses_iso() -> None: - stamp = records.timestamp({"timestamp": "1970-01-02T00:00:00+00:00"}) - assert stamp == 86400.0 - - -def test_in_window_rules() -> None: - dated = {"timestamp": "1970-01-01T00:01:40+00:00"} # 100s - assert records.in_window(dated, None) - assert records.in_window({}, (0.0, 1.0)) - assert records.in_window(dated, (100.0, 100.0)) - assert not records.in_window(dated, (101.0, 200.0)) - assert not records.in_window(dated, (0.0, 99.0)) - - -def test_cache_split_reads_both_classes() -> None: - usage = { - "cache_creation": { - "ephemeral_1h_input_tokens": 7, - "ephemeral_5m_input_tokens": 3, - }, - } - assert records.cache_split(usage) == { - records.CACHE_1H: 7, - records.CACHE_5M: 3, - } - - -def test_cache_split_ignores_non_dict_and_non_int() -> None: - assert records.cache_split({"cache_creation": 4}) == {} - nested = {"cache_creation": {"ephemeral_1h_input_tokens": "x"}} - assert records.cache_split(nested) == {} - - -def test_merge_takes_max_usage_and_sums_tools() -> None: - held = _turn(usage={"output_tokens": 1, "input_tokens": 5}, context=3, tool_calls=1) - extra = _turn( - usage={"output_tokens": 9, "cache_read_input_tokens": 2}, - context=1, - tool_calls=2, - ) - merged = records.merge(held, extra) - assert merged.usage == { - "output_tokens": 9, - "input_tokens": 5, - "cache_read_input_tokens": 2, - } - assert merged.context == 3 - assert merged.tool_calls == 3 - - -def test_merger_collapses_consecutive_same_id() -> None: - merger = records.TurnMerger() - assert merger.push(_turn("a", {"output_tokens": 1})) is None - assert merger.push(_turn("a", {"output_tokens": 4})) is None - done = merger.push(_turn("b")) - assert done is not None - assert done.usage == {"output_tokens": 4} - last = merger.flush() - assert last is not None - assert last.message_id == "b" - assert merger.flush() is None - - -def test_merger_drops_late_repeat_of_seen_id() -> None: - merger = records.TurnMerger() - merger.push(_turn("a")) - merger.push(_turn("b")) - # "a" already seen and not the held turn: dropped, "b" is released. - done = merger.push(_turn("a")) - assert done is not None - assert done.message_id == "b" - assert merger.flush() is None - - -def test_merger_keeps_every_idless_turn() -> None: - merger = records.TurnMerger() - assert merger.push(_turn("")) is None - first = merger.push(_turn("")) - assert first is not None - last = merger.flush() - assert last is not None diff --git a/python_pkg/token_audit/tests/test_report.py b/python_pkg/token_audit/tests/test_report.py deleted file mode 100644 index 761b4b95..00000000 --- a/python_pkg/token_audit/tests/test_report.py +++ /dev/null @@ -1,177 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from typing import TYPE_CHECKING - -import pytest - -from python_pkg.token_audit import attribute, report -from python_pkg.token_audit.model import Session, ToolCall, Turn - -if TYPE_CHECKING: - from pathlib import Path - - -def _turn( - out: int = 1, - read: int = 0, - create: int = 0, - context: int = 0, - *, - sidechain: bool = False, -) -> Turn: - return Turn( - usage={ - "output_tokens": out, - "cache_read_input_tokens": read, - "cache_creation_input_tokens": create, - }, - context=context, - model="claude-opus-5", - is_sidechain=sidechain, - ) - - -def _fixture() -> tuple[attribute.Totals, attribute.Axes, report.Window, list[Session]]: - session = Session(session_id="abcdef123456", path="p", cwd="/home/kuhy/demo") - session.turns = [_turn(out=10, create=500), _turn(out=5, read=400, context=400)] - session.tools = [ - ToolCall(name="Read", result_tokens=900, path="/s.png"), - ToolCall(name="mcp__aseprite__draw_line", result_tokens=3), - ToolCall(name="Skill", result_tokens=1, skill="finish"), - ] - session.image_cost = 90.0 - totals, axes = attribute.build([session]) - window = report.Window(since=0.0, until=86400.0) - return totals, axes, window, [session] - - -def test_window_describe_is_readable() -> None: - assert "UTC" in report.Window(since=0.0, until=86400.0).describe() - - -def test_snapshot_has_required_keys() -> None: - totals, axes, window, sessions = _fixture() - snap = report.snapshot(totals, axes, window, len(sessions)) - for key in ( - "generated", - "window", - "weighted_total", - "raw", - "image_cost", - "projects", - ): - assert key in snap - assert snap["sessions"] == 1 - - -def test_render_contains_each_section() -> None: - totals, axes, window, sessions = _fixture() - snap = report.snapshot(totals, axes, window, len(sessions)) - text = report.render(totals, axes, window, sessions, snap, None) - for heading in ( - "# Claude Code token audit", - "## Totals", - "## Ranked drivers", - "## Projects", - "## MCP servers invoked", - "## Skills invoked", - "## Most expensive sessions", - ): - assert heading in text - assert "aseprite" in text - assert "finish" in text - assert "abcdef12" in text - - -def test_render_without_previous_says_so() -> None: - totals, axes, window, sessions = _fixture() - snap = report.snapshot(totals, axes, window, len(sessions)) - assert "No previous report" in report.render( - totals, axes, window, sessions, snap, None - ) - - -@pytest.mark.parametrize( - ("before", "expected"), - [(100, "down"), (1, "up")], -) -def test_delta_direction(before: int, expected: str) -> None: - assert expected in report._delta_line( - {"weighted_total": 50}, {"weighted_total": before} - ) - - -def test_delta_handles_zero_previous() -> None: - assert "no spend" in report._delta_line( - {"weighted_total": 5}, {"weighted_total": 0} - ) - - -def test_empty_axes_render_placeholders() -> None: - session = Session(session_id="s", path="p", cwd="/x") - session.turns = [_turn()] - totals, axes = attribute.build([session]) - window = report.Window(since=0.0, until=1.0) - snap = report.snapshot(totals, axes, window, 1) - text = report.render(totals, axes, window, [session], snap, None) - assert "(none invoked)" in text - - -def test_pct_guards_zero_denominator() -> None: - assert report._pct(1.0, 0.0) == "n/a" - - -def test_write_creates_both_files(tmp_path: Path) -> None: - totals, axes, window, sessions = _fixture() - snap = report.snapshot(totals, axes, window, len(sessions)) - text = report.render(totals, axes, window, sessions, snap, None) - md_path = report.write(text, snap, tmp_path) - assert md_path.exists() - assert (tmp_path / report.JSON_NAME).exists() - assert json.loads((tmp_path / report.JSON_NAME).read_text())["sessions"] == 1 - - -def test_load_previous_roundtrip(tmp_path: Path) -> None: - assert report.load_previous(tmp_path) is None - (tmp_path / report.JSON_NAME).write_text('{"weighted_total": 7}', encoding="utf-8") - assert report.load_previous(tmp_path) == {"weighted_total": 7} - - -def test_load_previous_tolerates_corrupt_json(tmp_path: Path) -> None: - (tmp_path / report.JSON_NAME).write_text("{not json", encoding="utf-8") - assert report.load_previous(tmp_path) is None - - -def test_load_previous_rejects_non_object(tmp_path: Path) -> None: - (tmp_path / report.JSON_NAME).write_text("[1,2]", encoding="utf-8") - assert report.load_previous(tmp_path) is None - - -def test_batching_section_reports_share_and_rate() -> None: - axes = attribute.Axes() - axes.batching.messages = 4 - axes.batching.batched = 1 - axes.batching.calls = 6 - lines = report._batching_lines(axes) - assert "1 of 4" in lines[0] - assert "25.0%" in lines[0] - assert "1.50 per message" in lines[0] - - -def test_batching_section_handles_a_window_with_no_tool_calls() -> None: - """A window with no tool calls must not divide by zero.""" - lines = report._batching_lines(attribute.Axes()) - assert lines == ["_No tool calls in this window._"] - - -def test_snapshot_carries_the_batching_counters() -> None: - axes = attribute.Axes() - axes.batching.messages = 9 - axes.batching.batched = 2 - axes.batching.calls = 12 - snap = report.snapshot(attribute.Totals(), axes, report.Window(0.0, 1.0), 1) - assert snap["tool_messages"] == 9 - assert snap["batched_messages"] == 2 - assert snap["batched_calls"] == 12 diff --git a/python_pkg/token_audit/tests/test_review.py b/python_pkg/token_audit/tests/test_review.py deleted file mode 100644 index 6aad5d58..00000000 --- a/python_pkg/token_audit/tests/test_review.py +++ /dev/null @@ -1,127 +0,0 @@ -from __future__ import annotations - -from typing import Any - -import pytest - -from python_pkg.token_audit import attribute, history, review, surfaces, unused -from python_pkg.token_audit.model import Session, ToolCall, Turn - - -def _usage(name: str, short: int, long: int) -> unused.Usage: - return unused.Usage(name, short, long) - - -def _axes(*, bash: bool, spend: bool) -> tuple[attribute.Totals, attribute.Axes]: - usage = {"output_tokens": 10} if spend else {} - turn = Turn(usage=usage, context=0, model="claude-opus-5", is_sidechain=False) - tools = [ToolCall("Bash", 400)] if bash else [] - return attribute.build([Session("s", "s.jsonl", turns=[turn], tools=tools)]) - - -def test_extra_metrics_with_bash_and_spend(monkeypatch: pytest.MonkeyPatch) -> None: - fake = [surfaces.Surface("a", "d", 40), surfaces.Surface("b", "d", 80)] - monkeypatch.setattr(review.surfaces, "collect", lambda: fake) - totals, axes = _axes(bash=True, spend=True) - axes.image_cost = totals.weighted / 2 - m = review.extra_metrics(totals, axes) - assert m["bash_tokens_per_call"] > 0 - assert m["image_usd_share"] == 0.5 - assert m["standing_tokens_est"] == 30 - assert "batching_calls_per_msg" in m - - -def test_extra_metrics_without_bash_or_spend() -> None: - totals, axes = _axes(bash=False, spend=False) - m = review.extra_metrics(totals, axes) - assert m["bash_tokens_per_call"] == 0.0 - assert m["bash_tokens_per_day"] == 0.0 - assert m["bash_big_result_share"] == 0.0 - assert m["image_usd_share"] == 0.0 - - -def test_comparison_without_usable_previous() -> None: - assert "No comparable previous snapshot" in review._comparison({}, None)[0] - assert "No comparable previous snapshot" in review._comparison({}, {"v": 1})[0] - assert "No comparable previous snapshot" in review._comparison({}, {})[0] - - -def test_comparison_table_rows() -> None: - prev: dict[str, Any] = { - "v": review.SNAPSHOT_VERSION, - "metrics": { - "usd_per_day": 100.0, - "usd_per_call": 0, - "calls_per_day": "bad", - "usd_per_commit": 2.0, - }, - } - now = {"usd_per_day": 50.0, "usd_per_call": 1.0, "calls_per_day": 3.0} - rows = review._comparison(now, prev) - assert rows[0].startswith("| metric") - assert "| usd_per_day | 100.0 | 50.0 | -50.0% |" in rows - assert "| usd_per_call | 0 | 1.0 | n/a |" in rows - assert len(rows) == 4 # header, rule, two rendered metrics - - -def test_ledger_empty_and_populated() -> None: - assert review._ledger({"metrics": {}}) == ["_Ledger empty: nothing applied yet._"] - history.append_ledger( - {"lever": "L1", "metric": "m", "baseline": 10, "predicted": 5} - ) - history.append_ledger({"lever": "L2", "metric": "gone", "baseline": 1}) - lines = review._ledger({"metrics": {"m": 4}}) - text = "\n".join(lines) - assert "| L1 | m | 10.0 | 5.0 | 4.0 | **helped** |" in text - assert "| L2 | gone | 1.0 | - | - | **unmeasured** |" in text - - -def _patch_dead(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - review.unused, - "mcp_usage", - lambda: [_usage("deadmcp", 0, 0), _usage("live", 1, 0)], - ) - monkeypatch.setattr( - review.unused, "skill_usage", lambda: [_usage("deadskill", 0, 0)] - ) - - -def test_render_without_levers_has_no_candidates() -> None: - snap = {"metrics": {"usd_per_day": 1.0}} - text = "\n".join(review.render(snap, None, levers=False)) - assert "## Since the previous analysis" in text - assert "## What helped (ledger)" in text - assert "Candidate levers" not in text - - -def test_render_with_levers_uses_previous_snapshot( - monkeypatch: pytest.MonkeyPatch, -) -> None: - _patch_dead(monkeypatch) - snap = {"metrics": {"standing_tokens_est": 200}} - prev = {"v": review.SNAPSHOT_VERSION, "metrics": {"standing_tokens_est": 100}} - text = "\n".join(review.render(snap, prev, levers=True)) - assert "## Candidate levers (rule-based)" in text - assert "Always-loaded context grew" in text - assert "MCP: deadmcp; skills: deadskill" in text - - -def test_render_with_levers_ignores_old_or_missing_previous( - monkeypatch: pytest.MonkeyPatch, -) -> None: - _patch_dead(monkeypatch) - snap = {"metrics": {"standing_tokens_est": 200}} - for prev in (None, {"v": 1, "metrics": {"standing_tokens_est": 100}}): - text = "\n".join(review.render(snap, prev, levers=True)) - assert "Always-loaded context grew" not in text - - -def test_extra_metrics_bash_per_day_and_big_share() -> None: - usage = {"output_tokens": 10} - turn = Turn(usage=usage, context=0, model="claude-opus-5", is_sidechain=False) - tools = [ToolCall("Bash", 3000), ToolCall("Bash", 1000), ToolCall("Read", 5000)] - totals, axes = attribute.build([Session("s", "s.jsonl", turns=[turn], tools=tools)]) - m = review.extra_metrics(totals, axes, days=2.0) - assert m["bash_tokens_per_day"] == 2000.0 - assert m["bash_big_result_share"] == 0.75 diff --git a/python_pkg/token_audit/tests/test_surfaces.py b/python_pkg/token_audit/tests/test_surfaces.py deleted file mode 100644 index 26300454..00000000 --- a/python_pkg/token_audit/tests/test_surfaces.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Tests for the billed-surface enumerator.""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from python_pkg.token_audit import surfaces - - -@pytest.fixture -def fake_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """Build a minimal ~/.claude tree and point the module at it.""" - claude = tmp_path / ".claude" - (claude / "rules").mkdir(parents=True) - (claude / "rules" / "a.instructions.md").write_text("x" * 400) - (claude / "memories").mkdir() - (claude / "memories" / "m.md").write_text("y" * 200) - (claude / "CLAUDE.md").write_text("z" * 100) - memory = claude / "projects" / "-home-kuhy" / "memory" - memory.mkdir(parents=True) - (memory / "MEMORY.md").write_text("w" * 80) - skill = claude / "skills" / "demo" - skill.mkdir(parents=True) - (skill / "SKILL.md").write_text("---\nname: demo\ndescription: hello there\n---\n") - (claude / "settings.json").write_text( - json.dumps( - { - "enabledPlugins": {"on@x": True, "off@y": False}, - "autoMode": {"environment": ["a", "b"]}, - } - ) - ) - (tmp_path / ".claude.json").write_text(json.dumps({"mcpServers": {"aseprite": {}}})) - (claude / "mcp-parked.json").write_text(json.dumps({"mcpServers": {"reaper": {}}})) - repo = tmp_path / "repo" - repo.mkdir() - (repo / ".mcp.json").write_text(json.dumps({"mcpServers": {"local": {}}})) - monkeypatch.setattr(Path, "home", staticmethod(lambda: tmp_path)) - for name, value in [ - ("CLAUDE_DIR", claude), - ("GLOBAL_CONFIG", tmp_path / ".claude.json"), - ("SETTINGS", claude / "settings.json"), - ]: - monkeypatch.setattr(surfaces, name, value) - return tmp_path - - -def test_every_surface_is_reported(fake_home: Path) -> None: - """The enumeration is the contract: a missing row means a missed surface.""" - names = [s.name for s in surfaces.collect()] - assert names == [ - "rules/", - "memories/", - "CLAUDE.md", - "MEMORY.md", - "skill descriptions", - "plugins", - "global MCP", - "project .mcp.json", - "autoMode.environment", - ] - - -def test_sizes_come_from_disk(fake_home: Path) -> None: - """Byte counts reflect real files.""" - found = {s.name: s for s in surfaces.collect()} - assert found["rules/"].bytes_on_disk == 400 - assert found["rules/"].est_tokens == 100 - assert found["CLAUDE.md"].bytes_on_disk == 100 - assert found["MEMORY.md"].bytes_on_disk == 80 - - -def test_enabled_plugins_only(fake_home: Path) -> None: - """Disabled plugins cost nothing and must not be listed.""" - found = {s.name: s for s in surfaces.collect()} - assert found["plugins"].detail == "on@x" - assert found["plugins"].measurable - - -def test_mcp_split_between_active_and_parked(fake_home: Path) -> None: - """Parked servers are reported separately from active ones.""" - found = {s.name: s for s in surfaces.collect()} - assert found["global MCP"].detail == "1 active, 1 parked" - - -def test_project_mcp_files_counted(fake_home: Path) -> None: - """Project-scoped configs are surfaced, not hidden.""" - found = {s.name: s for s in surfaces.collect()} - assert "1 repos" in found["project .mcp.json"].detail - - -def test_skill_description_measured(fake_home: Path) -> None: - """Skill frontmatter contributes bytes.""" - found = {s.name: s for s in surfaces.collect()} - assert found["skill descriptions"].bytes_on_disk > 0 - assert "1 personal skills" in found["skill descriptions"].detail - - -def test_missing_files_are_zero( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """An absent tree or config contributes zero rather than raising.""" - monkeypatch.setattr(Path, "home", staticmethod(lambda: tmp_path)) - for name in ("CLAUDE_DIR", "GLOBAL_CONFIG", "SETTINGS"): - monkeypatch.setattr(surfaces, name, tmp_path / "nope") - result = surfaces.collect() - assert all(s.bytes_on_disk == 0 for s in result) - assert surfaces._mcp_names(tmp_path / "nope.json") == [] - - -def test_corrupt_json_is_tolerated( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Malformed config files degrade to zero, never to a crash.""" - bad = tmp_path / "bad.json" - bad.write_text("{oops") - assert surfaces._json_field_bytes(bad, "autoMode") == 0 - assert surfaces._mcp_names(bad) == [] - monkeypatch.setattr(surfaces, "SETTINGS", bad) - assert surfaces._enabled_plugins() == [] - - -def test_absent_field_is_zero(tmp_path: Path) -> None: - """A config without the field contributes nothing.""" - cfg = tmp_path / "s.json" - cfg.write_text(json.dumps({"other": 1})) - assert surfaces._json_field_bytes(cfg, "autoMode") == 0 - - -def test_enabled_plugins_without_settings( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """No settings file means no plugins.""" - monkeypatch.setattr(surfaces, "SETTINGS", tmp_path / "gone.json") - assert surfaces._enabled_plugins() == [] - - -def test_tree_bytes_ignores_missing(tmp_path: Path) -> None: - """A non-directory root totals zero.""" - assert surfaces._tree_bytes(tmp_path / "absent") == 0 diff --git a/python_pkg/token_audit/tests/test_unused.py b/python_pkg/token_audit/tests/test_unused.py deleted file mode 100644 index ea66824a..00000000 --- a/python_pkg/token_audit/tests/test_unused.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Tests for the dual-window unused-surface detector.""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from python_pkg.token_audit import unused - - -@pytest.fixture -def transcripts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """Redirect the transcript scan at a temporary tree.""" - root = tmp_path / "projects" - (root / "proj").mkdir(parents=True) - monkeypatch.setattr(unused, "TRANSCRIPT_ROOT", root) - return root / "proj" - - -def test_dead_requires_both_windows_empty() -> None: - """Used at 30 days but not 7 is not dead — the rule that prevents overreach.""" - assert unused.Usage("reaper", 0, 0).dead - assert not unused.Usage("ck3", 0, 9).dead - assert not unused.Usage("aseprite", 232, 748).dead - - -def test_usage_counts_across_windows(transcripts: Path) -> None: - """A recent call lands in both windows; an old one only in the long window.""" - now = 1_000_000_000.0 - recent = transcripts / "recent.jsonl" - recent.write_text('{"name":"mcp__aseprite__draw_line"}\n') - import os - - os.utime(recent, (now - 3 * 86400, now - 3 * 86400)) - old = transcripts / "old.jsonl" - old.write_text('{"name":"mcp__ck3__ck3_search"}\n') - os.utime(old, (now - 20 * 86400, now - 20 * 86400)) - - result = { - u.name: u - for u in unused.usage_for( - ["aseprite", "ck3", "reaper"], unused._MCP_CALL, now=now - ) - } - assert result["aseprite"].short_calls == 1 - assert result["aseprite"].long_calls == 1 - assert result["ck3"].short_calls == 0 - assert result["ck3"].long_calls == 1 - assert result["reaper"].dead - - -def test_scan_skips_unreadable( - transcripts: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """An unreadable transcript is skipped rather than aborting the audit.""" - bad = transcripts / "bad.jsonl" - bad.write_text("{}") - - def boom(*_: object, **__: object) -> str: - msg = "nope" - raise OSError(msg) - - monkeypatch.setattr(Path, "read_text", boom) - assert unused._scan(unused._MCP_CALL, 0) == {} - - -def test_mcp_usage_reads_config(tmp_path: Path, transcripts: Path) -> None: - """Server names come from the global config.""" - cfg = tmp_path / "claude.json" - cfg.write_text(json.dumps({"mcpServers": {"reaper": {}, "aseprite": {}}})) - names = {u.name for u in unused.mcp_usage(cfg)} - assert names == {"reaper", "aseprite"} - - -def test_mcp_usage_survives_bad_config(tmp_path: Path) -> None: - """A corrupt config yields no rows instead of raising.""" - cfg = tmp_path / "broken.json" - cfg.write_text("{not json") - assert unused.mcp_usage(cfg) == [] - assert unused.mcp_usage(tmp_path / "missing.json") == [] - - -def test_skill_usage_lists_installed( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, transcripts: Path -) -> None: - """Skill names come from the skills directory.""" - home = tmp_path / "home" - skill = home / ".claude" / "skills" / "grilling" - skill.mkdir(parents=True) - (skill / "SKILL.md").write_text("---\nname: grilling\n---\n") - monkeypatch.setattr(Path, "home", staticmethod(lambda: home)) - result = unused.skill_usage() - assert [u.name for u in result] == ["grilling"] - - -def test_skill_pattern_matches_invocation() -> None: - """The skill regex matches the transcript form.""" - assert unused._SKILL_CALL.findall('{"skill":"phone-deploy"}') == ["phone-deploy"] diff --git a/python_pkg/token_audit/turns.py b/python_pkg/token_audit/turns.py deleted file mode 100644 index 4651ec48..00000000 --- a/python_pkg/token_audit/turns.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Build a :class:`Turn` from one assistant transcript record. - -Split out of :mod:`parse` to keep both under the file-length cap; this half is -the pure record -> turn conversion, with no I/O and no ordering concerns. -""" - -from __future__ import annotations - -from typing import Any - -from python_pkg.token_audit import records -from python_pkg.token_audit.model import Turn - - -def usage_of(record: dict[str, Any]) -> tuple[dict[str, Any], dict[str, int]] | None: - """Return an assistant record's message and its integer usage fields. - - The message is returned alongside the usage block so callers that need both - (the model id lives on the message) never have to re-validate its type. - """ - message = record.get("message") - if not isinstance(message, dict): - return None - usage = message.get("usage") - if not isinstance(usage, dict): - return None - flat = {k: v for k, v in usage.items() if isinstance(v, int)} - return message, flat | records.cache_split(usage) - - -def _model(message: dict[str, Any]) -> str: - """Return the model id that served a record, or a placeholder. - - Only ever called after :func:`_usage` has confirmed the message is a dict, - so it takes the message itself rather than re-checking the record. - """ - model = message.get("model") - return model if isinstance(model, str) else "unknown" - - -def build_turn( - record: dict[str, Any], - message: dict[str, Any], - usage: dict[str, int], -) -> Turn: - """Build a :class:`Turn` from an assistant record's usage block.""" - context = usage.get("cache_read_input_tokens", 0) + usage.get( - "cache_creation_input_tokens", - 0, - ) - message_id = message.get("id") - content = message.get("content") - tool_calls = ( - sum( - 1 - for block in content - if isinstance(block, dict) and block.get("type") == "tool_use" - ) - if isinstance(content, list) - else 0 - ) - return Turn( - usage=usage, - context=context, - model=_model(message), - is_sidechain=bool(record.get("isSidechain")), - message_id=message_id if isinstance(message_id, str) else "", - tool_calls=tool_calls, - timestamp=records.timestamp(record) or 0.0, - ) diff --git a/python_pkg/token_audit/unused.py b/python_pkg/token_audit/unused.py deleted file mode 100644 index 1984216b..00000000 --- a/python_pkg/token_audit/unused.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Find registered surfaces that nothing has called, using two time windows. - -A single seven-day window is not enough evidence to unregister anything: one -audit's seven-day "dead" list included six servers that had been used within -thirty days. A surface therefore qualifies as dead only when it is unused in -the short window *and* the long one. -""" - -from __future__ import annotations - -from collections import Counter -from dataclasses import dataclass -import json -from pathlib import Path -import re -import time - -TRANSCRIPT_ROOT = Path.home() / ".claude" / "projects" -SECONDS_PER_DAY = 86400 -SHORT_WINDOW_DAYS = 7 -LONG_WINDOW_DAYS = 30 -_MCP_CALL = re.compile(r'"name":"mcp__([a-zA-Z0-9_-]+)__') -_SKILL_CALL = re.compile(r'"skill":"([^"]+)"') - - -@dataclass(frozen=True) -class Usage: - """Call counts for one surface across both windows.""" - - name: str - short_calls: int - long_calls: int - - @property - def dead(self) -> bool: - """True when unused in both windows, so it is safe to park.""" - return self.short_calls == 0 and self.long_calls == 0 - - -def _scan(pattern: re.Pattern[str], since: float) -> Counter[str]: - """Count regex matches across transcripts modified after *since*.""" - found: Counter[str] = Counter() - for path in TRANSCRIPT_ROOT.glob("*/*.jsonl"): - try: - if path.stat().st_mtime < since: - continue - text = path.read_text(errors="replace") - except OSError: - continue - found.update(pattern.findall(text)) - return found - - -def usage_for( - names: list[str], pattern: re.Pattern[str], now: float | None = None -) -> list[Usage]: - """Report short- and long-window call counts for each of *names*.""" - now = time.time() if now is None else now - short = _scan(pattern, now - SHORT_WINDOW_DAYS * SECONDS_PER_DAY) - long_ = _scan(pattern, now - LONG_WINDOW_DAYS * SECONDS_PER_DAY) - return [Usage(n, short.get(n, 0), long_.get(n, 0)) for n in sorted(names)] - - -def mcp_usage(config: Path | None = None) -> list[Usage]: - """Call counts for every globally registered MCP server.""" - config = config or Path.home() / ".claude.json" - try: - names = list(json.loads(config.read_text()).get("mcpServers", {})) - except json.JSONDecodeError, OSError: - return [] - return usage_for(names, _MCP_CALL) - - -def skill_usage() -> list[Usage]: - """Invocation counts for every personal skill.""" - root = Path.home() / ".claude" / "skills" - names = [p.parent.name for p in root.glob("*/SKILL.md")] - return usage_for(names, _SKILL_CALL) diff --git a/python_pkg/token_dashboard/__init__.py b/python_pkg/token_dashboard/__init__.py deleted file mode 100644 index 74014bf8..00000000 --- a/python_pkg/token_dashboard/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Durable, zero-token store of Claude Code spend for the control-panel /tokens page. - -``ingest`` walks transcripts (reusing token_audit's parser, dedup and prices) -into SQLite under ``~/data/token-dashboard/``; the dashboard only reads it. -""" diff --git a/python_pkg/token_dashboard/ab.py b/python_pkg/token_dashboard/ab.py deleted file mode 100644 index c009a689..00000000 --- a/python_pkg/token_dashboard/ab.py +++ /dev/null @@ -1,249 +0,0 @@ -"""A/B test of subagents: Sonnet arm vs Opus arm, one verdict per log. - -Logs: ``agent-ab.jsonl`` (judgment, closed), ``agent-ab-build.jsonl`` (build). -Assignments join on ``tool_use_id`` to the stored ``Agent`` call: **redo** (a -later Agent shares >= :data:`REDO_JACCARD` of its words), **follow-up** (a -``SendMessage`` to it), **error**, **correction** (noisy: next prompt matches -:data:`CORRECTION`), end tag. Wilson / Newcombe 95% intervals; the banner is -RED only when, with >= :data:`MIN_N` per arm, Sonnet's redo or correction rate -is higher and the two Wilson intervals do not overlap. -""" - -from __future__ import annotations - -from dataclasses import dataclass -import json -import math -from pathlib import Path -import re -from typing import TYPE_CHECKING, Any - -from python_pkg.token_dashboard import ab_rows - -if TYPE_CHECKING: - import sqlite3 - -LOG = Path.home() / ".claude" / "logs" / "agent-ab.jsonl" -BUILD_LOG = LOG.with_name("agent-ab-build.jsonl") # build-class arm, own verdict -Z = 1.96 -MIN_N = 20 -REDO_JACCARD = 0.6 -ERRORS = frozenset({"error", "aborted", "killed", "failed", "cancelled"}) -CORRECTION = re.compile( - r"^\W*no\b|\bwrong\b|\bthat'?s not\b|\bthat is not\b|\bincorrect\b" - r"|\brevert\b|\byou missed\b", - re.IGNORECASE, -) -WORD = re.compile(r"[a-z0-9]+") -RATES = ("redo", "followup", "error", "correction") -_UPSERT = "INSERT OR REPLACE INTO meta VALUES (?, ?)" - - -def wilson(k: int, n: int, z: float = Z) -> tuple[float, float]: - """95% Wilson score interval for ``k`` successes in ``n`` trials.""" - if n == 0: - return 0.0, 1.0 - p = k / n - centre = p + z * z / (2 * n) - spread = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) - denom = 1 + z * z / n - return max(0.0, (centre - spread) / denom), min(1.0, (centre + spread) / denom) - - -def diff_ci(k1: int, n1: int, k2: int, n2: int) -> tuple[float, float, float]: - """``p1 - p2`` with Newcombe's hybrid-score 95% interval.""" - p1, p2 = (k1 / n1 if n1 else 0.0), (k2 / n2 if n2 else 0.0) - l1, u1 = wilson(k1, n1) - l2, u2 = wilson(k2, n2) - diff = p1 - p2 - low = diff - math.sqrt((p1 - l1) ** 2 + (u2 - p2) ** 2) - high = diff + math.sqrt((u1 - p1) ** 2 + (p2 - l2) ** 2) - return diff, low, high - - -def jaccard(a: str, b: str) -> float: - """Word-set Jaccard similarity of two descriptions.""" - left, right = set(WORD.findall(a.lower())), set(WORD.findall(b.lower())) - union = left | right - return len(left & right) / len(union) if union else 0.0 - - -def is_correction(text: str) -> bool: - """Whether a user prompt reads as correcting the previous answer.""" - return CORRECTION.search(text) is not None - - -@dataclass(frozen=True) -class ArmCounts: - """What the banner needs from one arm.""" - - n: int - redo: int - correction: int - - -def banner( - sonnet: ArmCounts, opus: ArmCounts, label: str = "judgment" -) -> dict[str, str]: - """RED / GREEN / GREY verdict for the top of every /tokens view.""" - n = min(sonnet.n, opus.n) - if n < MIN_N: - return {"level": "grey", "text": f"A/B {label} agents: collecting (n={n})"} - for name in ("redo", "correction"): - ks, ko = getattr(sonnet, name), getattr(opus, name) - low_s, _ = wilson(ks, sonnet.n) - _, high_o = wilson(ko, opus.n) - if ks / sonnet.n > ko / opus.n and low_s > high_o: - return { - "level": "red", - "text": f"QUALITY DROP: Sonnet {name} rate {ks}/{sonnet.n} vs Opus " - f"{ko}/{opus.n} (95% CIs do not overlap)", - } - return { - "level": "green", - "text": f"A/B {label} agents: no detectable drop (n={n})", - } - - -def _assignments(log: Path) -> list[dict[str, Any]]: - """Well-formed assignment lines.""" - if not log.is_file(): - return [] - rows = [] - for line in log.read_text(encoding="utf-8", errors="replace").splitlines(): - try: - row = json.loads(line) - except ValueError: - continue - if isinstance(row, dict) and row.get("tool_use_id") and row.get("arm"): - rows.append(row) - return rows - - -def _outcome(conn: sqlite3.Connection, tool_use_id: str) -> dict[str, Any] | None: - """The joined call: cost, duration and the four flags, or ``None``.""" - call = conn.execute( - "SELECT key, agent_id, ts, description, result_ts, status FROM agent_calls" - " WHERE tool_use_id = ?", - (tool_use_id,), - ).fetchone() - if call is None: - return None - key, agent_id, ts, desc, result_ts, status = call - main = key.split("/", 1)[0] - sub = conn.execute( - "SELECT usd, wall_s, last_ts FROM subagent_link WHERE sub_key = ?", - (f"{main}/agent-{agent_id}",), - ).fetchone() or (0.0, (result_ts or ts) - ts, result_ts or ts) - returned = max(result_ts or ts, sub[2] or 0) - later = conn.execute( - "SELECT description FROM agent_calls WHERE key = ? AND ts > ?", - (key, returned), - ).fetchall() - followup = conn.execute( - "SELECT COUNT(*) FROM sent_messages WHERE key = ? AND to_agent = ? AND ts > ?", - (key, agent_id, returned), - ).fetchone()[0] - nxt = conn.execute( - "SELECT text FROM prompts WHERE key = ? AND ts > ? ORDER BY ts LIMIT 1", - (main, returned), - ).fetchone() - tag = conn.execute( - "SELECT final_tag FROM session_summary WHERE key = ?", (main,) - ).fetchone() - return { - "key": key, - "ts": ts, - "usd": sub[0] or 0.0, - "duration_s": sub[1] or 0.0, - "redo": any(jaccard(desc, d) >= REDO_JACCARD for (d,) in later), - "followup": followup > 0, - "error": (status or "").lower() in ERRORS, - "correction": nxt is not None and is_correction(nxt[0]), - "tag": (tag[0] if tag else None) or "untagged", - } - - -def _mean_ci(values: list[float]) -> dict[str, float]: - """Mean with a normal-approximation 95% interval.""" - n = len(values) - if n == 0: - return {"mean": 0.0, "low": 0.0, "high": 0.0} - mean = sum(values) / n - var = sum((v - mean) ** 2 for v in values) / (n - 1) if n > 1 else 0.0 - half = Z * math.sqrt(var / n) - return {"mean": mean, "low": mean - half, "high": mean + half} - - -def _arm(rows: list[dict[str, Any]]) -> dict[str, Any]: - """Per-arm summary.""" - n = len(rows) - out: dict[str, Any] = {"n": n} - for name in RATES: - k = sum(1 for r in rows if r[name]) - low, high = wilson(k, n) - out[name] = {"k": k, "rate": k / n if n else None, "low": low, "high": high} - out["usd"] = _mean_ci([r["usd"] for r in rows]) - out["duration_s"] = _mean_ci([r["duration_s"] for r in rows]) - tags: dict[str, int] = {} - for r in rows: - tags[r["tag"]] = tags.get(r["tag"], 0) + 1 - out["tags"] = tags - return out - - -def _summary( - conn: sqlite3.Connection, log: Path, label: str -) -> tuple[dict[str, Any], list[tuple[str, str, dict[str, Any]]]]: - """Join one log's assignments; the per-arm payload and the joined rows.""" - joined: dict[str, list[dict[str, Any]]] = {"sonnet": [], "opus": []} - kept: list[tuple[str, str, dict[str, Any]]] = [] - unjoined = 0 - for row in _assignments(log): - found = _outcome(conn, str(row["tool_use_id"])) - if found is None or row["arm"] not in joined: - unjoined += 1 - continue - joined[row["arm"]].append(found) - kept.append((row["arm"], str(row["tool_use_id"]), found)) - arms = {name: _arm(rows) for name, rows in joined.items()} - diffs = { - name: dict( - zip( - ("diff", "low", "high"), - diff_ci( - arms["sonnet"][name]["k"], - arms["sonnet"]["n"], - arms["opus"][name]["k"], - arms["opus"]["n"], - ), - strict=True, - ) - ) - for name in RATES - } - counts = { - name: ArmCounts( - arms[name]["n"], arms[name]["redo"]["k"], arms[name]["correction"]["k"] - ) - for name in joined - } - payload = { - "arms": arms, - "diffs": diffs, - "unjoined": unjoined, - "banner": banner(counts["sonnet"], counts["opus"], label), - } - return payload, kept - - -def rebuild( - conn: sqlite3.Connection, log: Path | None = None, build_log: Path | None = None -) -> dict[str, Any]: - """Store the judgment A/B in ``meta['ab']``, the build A/B in ``ab_build``.""" - payload, kept = _summary(conn, log or LOG, "judgment") - ab_rows.store(conn, kept) - build, _ = _summary(conn, build_log or BUILD_LOG, "build") - for key, value in (("ab", payload), ("ab_build", build)): - conn.execute(_UPSERT, (key, json.dumps(value))) - return payload diff --git a/python_pkg/token_dashboard/ab_rows.py b/python_pkg/token_dashboard/ab_rows.py deleted file mode 100644 index 452260c5..00000000 --- a/python_pkg/token_dashboard/ab_rows.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Per-assignment A/B outcomes, so the dashboard can filter the A/B panel. - -:func:`ab.rebuild` summarises all assignments into ``meta['ab']`` (the -all-time verdict). This module also keeps one row per joined assignment, with -the parent call's session, cwd, model and scope, so a cwd / model / scope -filter can recount the arms. Derived: replaced wholesale on every rebuild. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - import sqlite3 - -SCHEMA = """ -CREATE TABLE IF NOT EXISTS ab_rows( - tool_use_id TEXT PRIMARY KEY, arm TEXT, key TEXT, main_key TEXT, is_sub INT, - cwd TEXT, model TEXT, ts REAL, usd REAL, duration_s REAL, redo INT, - followup INT, error INT, correction INT, tag TEXT); -""" - - -def store( - conn: sqlite3.Connection, joined: list[tuple[str, str, dict[str, Any]]] -) -> None: - """Replace every row with ``(arm, tool_use_id, outcome)`` triples.""" - conn.executescript(SCHEMA) - conn.execute("DELETE FROM ab_rows") - for arm, tool_use_id, found in joined: - key = str(found["key"]) - where = conn.execute( - "SELECT cwd, model FROM turns WHERE key = ? AND ts <= ?" - " ORDER BY ts DESC LIMIT 1", - (key, found["ts"]), - ).fetchone() or (None, None) - conn.execute( - "INSERT OR REPLACE INTO ab_rows VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", - ( - tool_use_id, - arm, - key, - key.split("/", 1)[0], - int("/" in key), - where[0], - where[1], - found["ts"], - found["usd"], - found["duration_s"], - int(found["redo"]), - int(found["followup"]), - int(found["error"]), - int(found["correction"]), - found["tag"], - ), - ) diff --git a/python_pkg/token_dashboard/active.py b/python_pkg/token_dashboard/active.py deleted file mode 100644 index a9ff996e..00000000 --- a/python_pkg/token_dashboard/active.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Active time: the sum of gaps between consecutive records, ignoring idle gaps. - -A gap longer than :data:`IDLE_GAP` contributes **zero** (not the cap): it means -nobody and nothing was working. Storing each file as maximal runs of records -spaced at most ``IDLE_GAP`` apart is lossless for that rule -- the active time -of several files' merged timeline equals the measure of the union of their runs -with gaps of up to ``IDLE_GAP`` between runs bridged. That is what lets a main -session and the subagents it spawned, ingested in different runs, combine -exactly afterwards. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from collections.abc import Iterable - -IDLE_GAP = 300.0 -Span = tuple[float, float] - - -def intervals(stamps: Iterable[float], gap: float = IDLE_GAP) -> list[Span]: - """Collapse sorted-or-not timestamps into maximal runs.""" - out: list[Span] = [] - for stamp in sorted(stamps): - if out and stamp - out[-1][1] <= gap: - out[-1] = (out[-1][0], stamp) - else: - out.append((stamp, stamp)) - return out - - -def measure( - spans: Iterable[Span], - low: float = float("-inf"), - high: float = float("inf"), - gap: float = IDLE_GAP, -) -> float: - """Active seconds of the union of runs, clipped to ``[low, high)``.""" - merged = intervals_union(spans, gap) - total = 0.0 - for start, end in merged: - total += max(0.0, min(end, high) - max(start, low)) - return total - - -def intervals_union(spans: Iterable[Span], gap: float = IDLE_GAP) -> list[Span]: - """Union of runs, bridging gaps no longer than ``gap``.""" - out: list[Span] = [] - for start, end in sorted(spans): - if out and start - out[-1][1] <= gap: - out[-1] = (out[-1][0], max(out[-1][1], end)) - else: - out.append((start, end)) - return out diff --git a/python_pkg/token_dashboard/aux.py b/python_pkg/token_dashboard/aux.py deleted file mode 100644 index 0cb992da..00000000 --- a/python_pkg/token_dashboard/aux.py +++ /dev/null @@ -1,200 +0,0 @@ -"""Non-transcript sources: snapshots, levers, autopsy aggregates, activity logs. - -Each loader is idempotent (it replaces its own rows) and cheap, so every -complete ingest pass reruns all of them. - -The session-autopsy cache (``~/.claude/autopsy/sessions.jsonl``) is the only -source of per-session numbers for transcripts already deleted. It is -**aggregate only**: tokens are summed per transcript record, not per API -message (about 2x), there is no model split and no subagents. It is therefore -scaled by ``api_turns / assistant_msgs``, priced as Opus 5.5, and then -calibrated against sessions present in both sources (``meta`` key -``autopsy_calibration``). It is never mixed into the per-session tables. -""" - -from __future__ import annotations - -from collections import Counter -from datetime import datetime -import json -from pathlib import Path -from typing import TYPE_CHECKING, Any - -from python_pkg.token_audit import pricing -from python_pkg.token_dashboard import levers, store - -if TYPE_CHECKING: - from collections.abc import Iterator - import sqlite3 - -CLAUDE = Path.home() / ".claude" -AUTOPSY = CLAUDE / "autopsy" / "sessions.jsonl" -PROMPT_HISTORY = CLAUDE / "history.jsonl" -TIME_CAP = CLAUDE / "logs" / "time-cap.jsonl" -_MS = 1000.0 -_MS_EPOCH = 1e11 # anything larger is milliseconds, not seconds - - -def _jsonl(path: Path) -> Iterator[dict[str, Any]]: - """Well-formed object lines of a JSONL file (none if it is missing).""" - if not path.is_file(): - return - with path.open(encoding="utf-8", errors="replace") as handle: - for line in handle: - try: - row = json.loads(line) - except ValueError: - continue - if isinstance(row, dict): - yield row - - -def _stamp(raw: object) -> float | None: - """ISO string or epoch (s or ms) to unix seconds.""" - if isinstance(raw, (int, float)): - return raw / _MS if raw > _MS_EPOCH else float(raw) - if isinstance(raw, str): - try: - return datetime.fromisoformat(raw).timestamp() - except ValueError: - return None - return None - - -def snapshots(conn: sqlite3.Connection, report: Path) -> None: - """Copy every history snapshot's window and metrics.""" - for path in sorted((report / "history").glob("*.json")): - try: - snap = json.loads(path.read_text(encoding="utf-8")) - except ValueError: - continue - window = snap.get("window", {}) - conn.execute( - "INSERT OR REPLACE INTO snapshots VALUES (?,?,?,?,?)", - ( - path.stem, - snap.get("generated"), - window.get("since"), - window.get("until"), - json.dumps(snap.get("metrics", {})), - ), - ) - - -def _nearest(generated: list[tuple[float, str]], stamp: float) -> str: - """The snapshot day generated closest to ``stamp``.""" - return min(generated, key=lambda g: abs(g[0] - stamp))[1] - - -def lever_rows(conn: sqlite3.Connection, report: Path) -> None: - """Rebuild the ``levers`` table from every source.""" - generated = [ - (_stamp(g) or 0.0, name[:10]) - for name, g in conn.execute("SELECT name, generated FROM snapshots") - ] - tables: dict[str, str] = {} - found = conn.execute("SELECT ts, markdown FROM lever_tables ORDER BY ts") - weekly = report / "WEEKLY.md" - extra = [] - if weekly.is_file(): - extra = [(weekly.stat().st_mtime, weekly.read_text(encoding="utf-8"))] - for ts, markdown in [*found, *extra]: - # A table belongs to the snapshot generated closest to it. - if generated: - tables.setdefault(_nearest(generated, ts), markdown) - conn.execute("DELETE FROM levers") - conn.executemany( - "INSERT INTO levers VALUES (?,?,?,?,?,?,?,?,?,?,?)", - [tuple(row.values()) for row in levers.build(report, tables)], - ) - - -def autopsy(conn: sqlite3.Connection) -> None: - """Load autopsy aggregates and calibrate their dollar estimate.""" - price = pricing.load().for_model("claude-opus-5-5") - for row in _jsonl(AUTOPSY): - meta, tok, turns = ( - row.get("meta", {}), - row.get("tokens", {}), - row.get("turns", {}), - ) - started = _stamp(meta.get("started_at")) - records = row.get("counts", {}).get("assistant_msgs") or 0 - api = turns.get("api_turns") or records - if started is None or not records: - continue - scale = api / records - raw = ( - tok.get("input", 0) * price.input - + tok.get("cache_write", 0) * price.input * pricing.WRITE_1H - + tok.get("cache_read", 0) * price.cache_read - + tok.get("output", 0) * price.output - ) / pricing.PER_MILLION - conn.execute( - "INSERT OR REPLACE INTO aggregate_sessions VALUES" - " (?,?,?,?,?,?,?,?,?,?,?,?,?)", - ( - row.get("session_id"), - "autopsy", - store.day_of(started), - started, - meta.get("cwd"), - meta.get("title"), - tok.get("input", 0), - tok.get("cache_write", 0), - tok.get("cache_read", 0), - tok.get("output", 0), - api, - records, - raw * scale, - ), - ) - est, real = conn.execute( - "SELECT SUM(a.usd_est), SUM(s.usd_main) FROM aggregate_sessions a" - " JOIN session_summary s ON s.key = a.session_id" - ).fetchone() - factor = (real / est) if est and real else 1.0 - conn.execute( - "INSERT OR REPLACE INTO meta VALUES ('autopsy_calibration', ?)", (str(factor),) - ) - - -def activity_logs(conn: sqlite3.Connection, report: Path) -> None: - """Prompt counts per day, time-cap events, task-size misses and estimates.""" - days: Counter[str] = Counter() - for row in _jsonl(PROMPT_HISTORY): - stamp = _stamp(row.get("timestamp")) - if stamp is not None: - days[store.day_of(stamp)] += 1 - # Upserts, never a wipe: if history.jsonl is trimmed, the days it lost stay. - conn.executemany( - "INSERT INTO prompt_activity VALUES (?,?) ON CONFLICT(day) DO UPDATE" - " SET prompts = MAX(prompts, excluded.prompts)", - days.items(), - ) - sources = ( - ("time-cap", TIME_CAP, "event"), - ("task-size-miss", report / "task-size" / "misses.jsonl", "kind"), - ) - for name, path, kind_key in sources: - conn.executemany( - "INSERT OR IGNORE INTO events VALUES (?,?,?,?)", - [ - ( - name, - _stamp(r.get("ts")), - str(r.get(kind_key)), - str(r.get("command") or r.get("first_prompt") or "")[:200], - ) - for r in _jsonl(path) - ], - ) - for row in _jsonl(report / "task-size" / "verdicts.jsonl"): - verdict = row.get("verdict") - calls = verdict.get("est_calls") if isinstance(verdict, dict) else None - if row.get("status") == "ok" and isinstance(calls, int): - # The first sized prompt of a session is its pre-work estimate. - conn.execute( - "INSERT OR IGNORE INTO estimates VALUES (?,?)", - (row.get("session"), calls), - ) diff --git a/python_pkg/token_dashboard/blocks.py b/python_pkg/token_dashboard/blocks.py deleted file mode 100644 index 9fcf1f8e..00000000 --- a/python_pkg/token_dashboard/blocks.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Pure readers for individual transcript records and content blocks. - -Split from :mod:`walk` so the record-level rules (what counts as a typed -prompt, how a status tag or a commit hash is recognised) are testable without a -file and the walker stays under the file-length cap. -""" - -from __future__ import annotations - -import os -from pathlib import Path -import re -from typing import Any - -from python_pkg.token_audit.discover import WRAPPER - -PROMPT_CHARS = 120 -LEVER_HEADING = "Candidate levers (rule-based)" -# kuhy's convention: every final reply ends on one of these status tags. -TAG = re.compile(r"^\W*(DONE|RUNNING|QUESTION|NEEDS ACTION)\W*:", re.MULTILINE) -COMMAND = re.compile(r"\s*(\S+?)\s*") -# `git commit` prints `[branch sha] subject`, also through finish_auto.sh. -COMMIT_LINE = re.compile( - r"^\[[^\]\s]+(?: \(root-commit\))? ([0-9a-f]{7,40})\]", re.MULTILINE -) -REPO_HINT = re.compile(r"(?:git\s+-C|\bcd)\s+(\"[^\"]+\"|'[^']+'|[^\s;&|]+)") -_SKIP_PREFIXES = ("[Request interrupted", "Caveat:") - - -def text_of(content: object) -> str: - """Flatten a message/tool-result ``content`` into plain text.""" - if isinstance(content, str): - return content - if not isinstance(content, list): - return "" - parts = [ - str(block.get("text") or "") - for block in content - if isinstance(block, dict) and block.get("type") == "text" - ] - return "\n".join(parts) - - -def prompt_text(record: dict[str, Any]) -> str | None: - """The typed text of a real user prompt, or ``None`` for anything else. - - Excludes tool results, ``isMeta`` / compact-summary records, interrupts, - and harness-injected ```` blocks. A slash command keeps its name. - """ - if record.get("type") != "user" or record.get("isMeta"): - return None - if record.get("isCompactSummary") or record.get("isVisibleInTranscriptOnly"): - return None - message = record.get("message") - content = message.get("content") if isinstance(message, dict) else None - if isinstance(content, list) and any( - isinstance(b, dict) and b.get("type") == "tool_result" for b in content - ): - return None - raw = text_of(content) - command = COMMAND.search(raw) - text = WRAPPER.sub("", raw).strip() - if command is not None and not text: - text = command.group(1) - if not text or text.startswith(_SKIP_PREFIXES): - return None - return " ".join(text.split())[:PROMPT_CHARS] - - -def final_tag(text: str) -> str | None: - """The last status tag in a reply, or ``None`` when it carries none.""" - found = TAG.findall(text) - return found[-1] if found else None - - -def commit_shas(text: str) -> list[str]: - """Commit hashes printed by ``git commit`` in a tool result.""" - return COMMIT_LINE.findall(text) - - -def repo_hint(command: str, cwd: str | None) -> str: - """The repository a commit command ran in: ``-C``/``cd`` target, else cwd.""" - match = REPO_HINT.search(command) - target = match.group(1).strip("\"'") if match else (cwd or "") - if not target: - return "" - path = Path(target).expanduser() - if cwd and not path.is_absolute(): - path = Path(cwd) / path - return os.path.normpath(path) - - -def lever_table(text: str) -> str | None: - """The markdown table following the candidate-levers heading, if any.""" - at = text.find(LEVER_HEADING) - if at < 0: - return None - rows: list[str] = [] - for line in text[at:].splitlines()[1:]: - if line.startswith("|"): - rows.append(line) - elif rows: - break - return "\n".join(rows) if rows else None diff --git a/python_pkg/token_dashboard/carry.py b/python_pkg/token_dashboard/carry.py deleted file mode 100644 index 2459836f..00000000 --- a/python_pkg/token_dashboard/carry.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Carry cost: what a tool result costs by staying in context afterwards. - -A result of N tokens is cache-read again on every later API call of the same -file until a compaction drops it (or the session ends), so its real price is -N x (cache-read price of each later call) -- usually far more than the call -that fetched it. A compaction shows up as the context shrinking below -``COMPACTION_DROP_RATIO`` of the previous call's, the same rule token_audit's -image-cost model uses. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Protocol - -from python_pkg.token_audit.model import COMPACTION_DROP_RATIO -from python_pkg.token_audit.pricing import PER_MILLION, WRITE_5M - -if TYPE_CHECKING: - from python_pkg.token_audit.pricing import PriceBook - - -class _Row(Protocol): - """Anything with creation / carry accumulators.""" - - create_usd: float - carry_usd: float - carry_calls: int - - -class Tracker: - """Collects calls and results in order, settles carry at the end.""" - - def __init__(self) -> None: - """Start with no calls and no results.""" - self.calls: list[tuple[str, int]] = [] - self.writes: list[float] = [] - self.results: list[tuple[int, int, _Row]] = [] - - def turn(self, model: str, context: int, write: float = WRITE_5M) -> None: - """Note one API call, in order, with its cache-write price multiplier.""" - self.calls.append((model, context)) - self.writes.append(write) - - def result(self, row: _Row, tokens: int) -> None: - """Note a result arriving before call number ``len(self.calls)``.""" - self.results.append((len(self.calls), tokens, row)) - - def spans(self) -> list[tuple[int, int, int, _Row]]: - """``(first, end, tokens, row)``: the calls ``first..end-1`` carry it.""" - # cut[i]: index of the first call at or after i that starts a new - # context (after a compaction), else len(calls). - size = len(self.calls) - cut = [size] * (size + 1) - for index in range(size - 1, -1, -1): - dropped = index > 0 and self.calls[index][1] < ( - COMPACTION_DROP_RATIO * self.calls[index - 1][1] - ) - cut[index] = index if dropped else cut[index + 1] - return [ - (start, cut[start + 1], tokens, row) - for start, tokens, row in self.results - if start < size - ] - - def settle(self, book: PriceBook) -> None: - """Add each result's creation and carry to its row. - - Creation: the first call after the result writes it to cache, at that - call's write multiplier of input price. Carry: every call from there - to the next compaction re-reads it at the cache-read price. - """ - prefix = [0.0] - for model, _ in self.calls: - prefix.append(prefix[-1] + book.for_model(model).cache_read) - for start, end, tokens, row in self.spans(): - price = book.for_model(self.calls[start][0]).input - row.create_usd += tokens * self.writes[start] * price / PER_MILLION - row.carry_usd += tokens * (prefix[end] - prefix[start]) / PER_MILLION - row.carry_calls += end - start diff --git a/python_pkg/token_dashboard/check_window.py b/python_pkg/token_dashboard/check_window.py deleted file mode 100644 index 6170374b..00000000 --- a/python_pkg/token_dashboard/check_window.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Reconcile the store against a token_audit snapshot: within 1% or exit 1. - -The snapshot's ``usd_per_day`` is ``usd_total`` over the span between the -earliest and latest priced record inside its window (not the window length), -so the same definition is applied to the store's turns here. - - python3 -m python_pkg.token_dashboard.check_window # newest - python3 -m python_pkg.token_dashboard.check_window --snapshot 2026-10-17 -""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path -import sys -from typing import TYPE_CHECKING - -from python_pkg.token_audit import history -from python_pkg.token_dashboard import schema - -if TYPE_CHECKING: - import sqlite3 - -TOLERANCE = 0.01 -DAY_S = 86400.0 -MIN_SPAN_S = 60.0 - - -def measure(conn: sqlite3.Connection, since: float, until: float) -> dict[str, float]: - """``usd_total``, ``usd_per_day`` and ``calls`` from the store's turns.""" - usd, calls, first, last = conn.execute( - "SELECT SUM(usd), COUNT(*), MIN(ts), MAX(ts) FROM turns" - " WHERE ts BETWEEN ? AND ?", - (since, until), - ).fetchone() - usd = usd or 0.0 - days = max((last or 0) - (first or 0), MIN_SPAN_S) / DAY_S - return {"usd_total": usd, "usd_per_day": usd / days, "calls": float(calls)} - - -def main(argv: list[str] | None = None) -> int: - """Compare and print one line per metric; 0 when every one is within 1%.""" - parser = argparse.ArgumentParser(prog="token_dashboard.check_window") - parser.add_argument("--db", type=Path, default=schema.DEFAULT_DB) - parser.add_argument("--report-dir", type=Path, default=history.REPORT_DIR) - parser.add_argument("--snapshot", default=None, help="history file stem") - args = parser.parse_args(argv) - folder = args.report_dir / history.HISTORY_NAME - path = ( - folder / f"{args.snapshot}.json" - if args.snapshot - else max(folder.glob("*.json"), default=None) - ) - if path is None or not path.is_file() or not args.db.is_file(): - sys.stderr.write(f"missing snapshot or store ({path}, {args.db})\n") - return 1 - snap = json.loads(path.read_text(encoding="utf-8")) - window = snap["window"] - conn = schema.connect(args.db, readonly=True) - try: - got = measure(conn, float(window["since"]), float(window["until"])) - finally: - conn.close() - ok = True - for key, value in got.items(): - want = float(snap["metrics"][key]) - drift = abs(value - want) / want if want else 0.0 - ok &= drift <= TOLERANCE - sys.stdout.write( - f"{key}: store {value:,.2f} vs snapshot {want:,.2f}" - f" ({drift:.2%}) {'OK' if drift <= TOLERANCE else 'FAIL'}\n" - ) - return 0 if ok else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_pkg/token_dashboard/classify.py b/python_pkg/token_dashboard/classify.py deleted file mode 100644 index 8878b1ac..00000000 --- a/python_pkg/token_dashboard/classify.py +++ /dev/null @@ -1,216 +0,0 @@ -"""Name what a tool call was, for the carry-cost and tool panels. - -``family`` is the bucket a panel groups by (``git status``, ``mcp:``, -``Read:memory``); ``target`` is the one thing it touched (the command, path, -host or query), capped so the store stays small; ``signature`` identifies -identical repeated calls (loops) without storing the full input. -""" - -from __future__ import annotations - -import hashlib -import json -import re -from urllib.parse import urlsplit - -from python_pkg.token_audit.model import IMAGE_SUFFIXES - -TARGET_CHARS = 160 -SIG_CHARS = 16 -EMPTY = "(empty)" - -# Words that only wrap the real command (and the flags/numbers they take). -_WRAPPERS = frozenset( - { - "sudo", - "env", - "nice", - "ionice", - "timeout", - "time", - "nohup", - "exec", - "command", - "capped.sh", - "stdbuf", - "xargs", - } -) -# Tools whose first non-flag argument is what the call really did. -_SUBCOMMANDS = frozenset( - { - "git", - "pnpm", - "npm", - "npx", - "flutter", - "dart", - "cargo", - "go", - "docker", - "systemctl", - "gh", - "pre-commit", - "uv", - "pip", - "yay", - "pacman", - "adb", - "make", - "fvm", - "journalctl", - } -) -_TAKES_VALUE = frozenset({"-C", "-c", "-n", "--dir", "--prefix"}) -_SHELLS = frozenset({"bash", "sh", "zsh"}) -_ASSIGN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=") -_SEGMENTS = re.compile(r"\s*(?:&&|\|\||;|\||\n)\s*") -_NUMBERISH = re.compile(r"^\d+[smhd]?$") -_ERRORS = ( - ("Exit code", "exit_code"), - ("", "tool_error"), - ("doesn't want to proceed", "rejected"), - ("Permission for this action was denied", "denied"), - ("hook error", "hook"), - ("TIME CAP", "time_cap"), - ("[Request interrupted", "interrupted"), -) - - -def _base(word: str) -> str: - """Last path component of a command word.""" - return word.rsplit("/", 1)[-1] - - -def _command_words(words: list[str]) -> list[str]: - """Drop leading assignments, wrappers and the flags/values they take.""" - index = 0 - while index < len(words): - word = words[index] - if not ( - _ASSIGN.match(word) - or _base(word) in _WRAPPERS - or word.startswith("-") - or _NUMBERISH.match(word) - ): - break - index += 1 - return words[index:] - - -def _first_argument(words: list[str]) -> str: - """The first non-flag word, skipping flags that take a value.""" - skip = False - for word in words: - if skip: - skip = False - elif word in _TAKES_VALUE: - skip = True - elif not word.startswith(("-", "<", ">", "2>")): - return word - return "" - - -def _name_command(words: list[str]) -> str: - """``git status``, ``python3 -m pytest``, ``bash x.sh`` or the bare name.""" - head, rest = _base(words[0]), words[1:] - if head.startswith("python"): - if "-m" in rest: - after = rest[rest.index("-m") + 1 :] - return f"{head} -m {after[0]}" if after else f"{head} -m" - script = _first_argument(rest) - return f"{head} {_base(script)}" if script else head - if head in _SHELLS and "-c" in rest: - return f"{head} -c" - if head in _SHELLS or head in _SUBCOMMANDS: - arg = _first_argument(rest) - return f"{head} {_base(arg) if head in _SHELLS else arg}".strip() - return head - - -def bash_family(command: str) -> str: - """The command a Bash call really ran, ignoring wrappers and ``cd``.""" - last = "" - for segment in _SEGMENTS.split(command.strip()): - words = _command_words(segment.split()) - if not words: - continue - last = _name_command(words) - if last != "cd": - return last - return last or EMPTY - - -_PATH_KINDS = ( - ("memory", ("/.claude/", "/memory/"), ("/.claude/", "/memories/")), - ("rules", ("/.claude/rules",), ("rules-ondemand",)), - ("skill", ("/skills/",)), - ("claude", ("/.claude/",)), -) - - -def _path_kind(path: str) -> str: - """What a file path is, as far as standing context goes.""" - name = _base(path) - if name.startswith("CLAUDE") and name.endswith(".md"): - return "claude-md" - for kind, *needle_sets in _PATH_KINDS: - if any(all(n in path for n in needles) for needles in needle_sets): - return kind - lowered = path.lower() - return "image" if lowered.endswith(tuple(IMAGE_SUFFIXES)) else "file" - - -def _text(tool_input: dict[str, object], key: str) -> str: - """One string input field, empty when absent.""" - value = tool_input.get(key) - return value if isinstance(value, str) else "" - - -def family(name: str, tool_input: dict[str, object]) -> str: - """The bucket a call is grouped under.""" - if name == "Bash": - return bash_family(_text(tool_input, "command")) - if name.startswith("mcp__"): - return f"mcp:{name.split('__')[1]}" - if name == "Skill": - return "skill" - if name in {"Read", "Write", "Edit", "MultiEdit", "NotebookEdit"}: - return f"{name}:{_path_kind(_text(tool_input, 'file_path'))}" - return name - - -def target(name: str, tool_input: dict[str, object]) -> str: - """The one thing a call touched, whitespace-collapsed and capped.""" - if name == "Bash": - found = _text(tool_input, "command") - elif name.startswith("mcp__"): - found = name.split("__", 2)[-1] - elif name == "Skill": - found = _text(tool_input, "skill") - elif name == "WebFetch": - found = urlsplit(_text(tool_input, "url")).netloc - elif name in {"WebSearch", "Grep", "Glob"}: - found = _text(tool_input, "query") or _text(tool_input, "pattern") - elif name in {"Agent", "Task"}: - kind = _text(tool_input, "subagent_type") or "general-purpose" - worktree = _text(tool_input, "isolation") == "worktree" - found = f"{kind} [worktree]" if worktree else kind - else: - found = _text(tool_input, "file_path") - return " ".join(found.split())[:TARGET_CHARS] - - -def signature(name: str, tool_input: dict[str, object]) -> str: - """A short stable hash of the full call, for spotting identical repeats.""" - raw = json.dumps([name, tool_input], sort_keys=True, default=str) - return hashlib.sha1(raw.encode(), usedforsecurity=False).hexdigest()[:SIG_CHARS] - - -def error_kind(text: str) -> str: - """Why a tool call failed, from its result text.""" - head = text[:400] - for needle, kind in _ERRORS: - if needle in head: - return kind - return "error" diff --git a/python_pkg/token_dashboard/commits.py b/python_pkg/token_dashboard/commits.py deleted file mode 100644 index 13d0ecdd..00000000 --- a/python_pkg/token_dashboard/commits.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Find the commits each session made, from git rather than from tool output. - -Measured 2026-10-17: of 601 recent commit commands only 35 printed a parseable -``[branch sha]`` line -- most run through ``finish_auto.sh`` with a tailed or -backgrounded output. So every commit command is stored with its time window -and repository, and the hashes come from ``git log`` instead: - -* a commit whose hash a command printed belongs to that command's session; -* otherwise a commit carrying a ``Co-Authored-By: Claude`` trailer belongs to - the commit command in the same repository whose window covers its commit - time (from :data:`LEAD_S` before the call to :data:`TAIL_S` after its - result -- a backgrounded finish_auto.sh commits after the call returns), - the nearest one when several do. - -Rows are only ever added, so a repository deleted later keeps its history. -""" - -from __future__ import annotations - -from collections import defaultdict -import re -from typing import TYPE_CHECKING - -from python_pkg.token_dashboard import gitcmd - -if TYPE_CHECKING: - import sqlite3 - -LEAD_S = 60.0 -TAIL_S = 1800.0 -TRAILER = re.compile(r"^Co-Authored-By:\s*Claude", re.IGNORECASE | re.MULTILINE) -_FIELD, _RECORD = "\x1f", "\x1e" -_LOG_FIELDS = 3 - - -def _git(repo: str, *args: str) -> str | None: - """Run git in ``repo``; ``None`` on any failure.""" - return gitcmd.run(repo, *args) - - -def _toplevel(hint: str, cache: dict[str, str | None]) -> str | None: - """The work tree containing ``hint``, cached per hint.""" - if hint not in cache: - out = _git(hint, "rev-parse", "--show-toplevel") if hint else None - cache[hint] = out.strip() if out else None - return cache[hint] - - -def _log(repo: str, since: float) -> list[tuple[str, float, bool]]: - """``(sha, commit time, has Claude trailer)`` on HEAD since ``since``.""" - out = _git( - repo, - "log", - "HEAD", - f"--since=@{int(since)}", - f"--format=%H{_FIELD}%ct{_FIELD}%B{_RECORD}", - ) - rows = [] - for record in (out or "").split(_RECORD): - parts = record.strip("\n").split(_FIELD, 2) - if len(parts) == _LOG_FIELDS: - rows.append((parts[0], float(parts[1]), bool(TRAILER.search(parts[2])))) - return rows - - -Command = tuple[str, int, float, float, str] # key, prompt, start, end, sha - - -def _match( - sha: str, ts: float, cmds: list[Command], *, trailer: bool -) -> Command | None: - """The command that made this commit, if any.""" - for cmd in cmds: - if cmd[4] and sha.startswith(cmd[4]): - return cmd - if not trailer: - return None - covering = [c for c in cmds if c[2] - LEAD_S <= ts <= max(c[2], c[3]) + TAIL_S] - if not covering: - return None - return min(covering, key=lambda c: abs(ts - c[2])) - - -def discover(conn: sqlite3.Connection) -> int: - """Attribute git commits to sessions; returns how many rows were added.""" - cache: dict[str, str | None] = {} - by_repo: dict[str, list[Command]] = defaultdict(list) - for key, prompt, start, end, sha, hint in conn.execute( - "SELECT key, prompt_idx, ts, end_ts, sha, repo FROM commits" - ): - top = _toplevel(hint, cache) - if top is not None: - by_repo[top].append((key, prompt, start, end, sha)) - added = 0 - for repo, cmds in by_repo.items(): - since = min(c[2] for c in cmds) - LEAD_S - for sha, ts, trailer in _log(repo, since): - cmd = _match(sha, ts, cmds, trailer=trailer) - if cmd is None: - continue - added += conn.execute( - "INSERT OR IGNORE INTO session_commits VALUES (?,?,?,?,?)", - (sha, repo, ts, cmd[0], cmd[1]), - ).rowcount - return added diff --git a/python_pkg/token_dashboard/context_files.py b/python_pkg/token_dashboard/context_files.py deleted file mode 100644 index b94a9ee7..00000000 --- a/python_pkg/token_dashboard/context_files.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Which files enter Claude's context, and how big each has been over time. - -:func:`find` lists them per file (token_audit's :mod:`surfaces` sums them per -surface, which cannot say which rule grew). Only the list and the size history -are stored -- the dashboard reads each file live when asked, so it never shows -a stale copy. - -Size history comes from two places, both cheap: ``~/.claude``'s own git -history (one ``git log --raw`` plus one ``git cat-file --batch-check`` for -every blob, incremental from the last commit seen) and the weekly prune's -``prune//sizes*.json`` snapshots. -""" - -from __future__ import annotations - -from dataclasses import dataclass -import json -from pathlib import Path -from typing import TYPE_CHECKING - -from python_pkg.token_audit import pricing -from python_pkg.token_audit.surfaces import CLAUDE_DIR -from python_pkg.token_dashboard import gitcmd - -if TYPE_CHECKING: - import sqlite3 - -SRC_DIR = Path.home() / "src" -MEMORY_LIMIT = 24_400 # MEMORY.md is only read up to ~24.4 KB -CLAUDE_MD_LIMIT = 40_000 # Claude Code warns about a CLAUDE.md over 40k chars -DESCRIPTION_LIMIT = 1024 # a skill description's maximum length -_BATCH_FIELDS = 3 # cat-file --batch-check: - - -@dataclass(frozen=True) -class ContextFile: - """One file and how it reaches the context.""" - - path: str - kind: str - standing: str # always | in repo | description - limit: int | None - - -def find(claude: Path | None = None, src: Path | None = None) -> list[ContextFile]: - """Every context file on disk (on-demand backups excluded).""" - claude = claude or CLAUDE_DIR - src = src or SRC_DIR - out = [ - ContextFile( - str(claude / "CLAUDE.md"), "global CLAUDE.md", "always", CLAUDE_MD_LIMIT - ) - ] - groups = ( - ("memories", "*.md", "memories (@-imported)", "always", None), - ("rules", "**/*.md", "rules (loaded recursively)", "always", None), - ("skills", "*/SKILL.md", "skill", "description", DESCRIPTION_LIMIT), - ("agents", "*.md", "agent definition", "description", DESCRIPTION_LIMIT), - ("output-styles", "*.md", "output style", "always", None), - ) - for folder, pattern, kind, standing, limit in groups: - out += [ - ContextFile(str(p), kind, standing, limit) - for p in sorted((claude / folder).glob(pattern)) - if p.is_file() - ] - memory = claude / "projects" / "-home-kuhy" / "memory" / "MEMORY.md" - out.append(ContextFile(str(memory), "auto-memory index", "always", MEMORY_LIMIT)) - for name in ("CLAUDE.md", "AGENTS.md"): - out += [ - ContextFile(str(p), f"repo {name}", "in repo", CLAUDE_MD_LIMIT) - for p in sorted(src.glob(f"*/{name}")) - ] - return [f for f in out if Path(f.path).is_file()] - - -def _git(repo: Path, *args: str, stdin: str | None = None) -> str: - """Run git, returning stdout ('' on failure).""" - return gitcmd.run(repo, *args, stdin=stdin) or "" - - -def _git_sizes(claude: Path, since: str) -> tuple[list[tuple[str, float, str]], str]: - """``(relpath, commit time, blob)`` for commits after ``since``; new head.""" - head = _git(claude, "rev-parse", "HEAD").strip() - if not head or head == since: - return [], since - span = f"{since}..HEAD" if since else "HEAD" - out = _git(claude, "log", "--raw", "--no-abbrev", "--format=@%ct", span) - rows: list[tuple[str, float, str]] = [] - stamp = 0.0 - for line in out.splitlines(): - if line.startswith("@"): - stamp = float(line[1:]) - elif line.startswith(":") and line.endswith(".md"): - meta, _, rel = line.partition("\t") - blob = meta.split()[3] - rows.append((rel, stamp, blob)) - return rows, head - - -def refresh( - conn: sqlite3.Connection, claude: Path | None = None, src: Path | None = None -) -> None: - """Re-list context files and append any new size-history points.""" - claude = claude or CLAUDE_DIR - files = find(claude, src) - conn.execute("DELETE FROM context_files") - conn.executemany( - "INSERT INTO context_files VALUES (?,?,?,?)", - [(f.path, f.kind, f.standing, f.limit) for f in files], - ) - row = conn.execute("SELECT value FROM meta WHERE key='context_git_head'").fetchone() - rows, head = _git_sizes(claude, row[0] if row else "") - zero = "0" * 40 - blobs = sorted({b for _, _, b in rows if b != zero}) - sizes: dict[str, int] = {} - batch = _git(claude, "cat-file", "--batch-check", stdin="\n".join(blobs) + "\n") - for line in batch.splitlines(): - parts = line.split() - if len(parts) == _BATCH_FIELDS and parts[2].isdigit(): - sizes[parts[0]] = int(parts[2]) - conn.executemany( - "INSERT OR IGNORE INTO context_sizes VALUES (?,?,?,?)", - [(str(claude / rel), ts, sizes.get(b, 0), "git") for rel, ts, b in rows], - ) - conn.execute("INSERT OR REPLACE INTO meta VALUES ('context_git_head', ?)", (head,)) - for path in sorted((claude / "token-report" / "prune").glob("*/sizes*.json")): - try: - data = json.loads(path.read_text(encoding="utf-8")).get("file_bytes", {}) - except ValueError, OSError, AttributeError: - continue - stamp = path.stat().st_mtime - conn.executemany( - "INSERT OR IGNORE INTO context_sizes VALUES (?,?,?,?)", - [ - (str(claude / rel), stamp, size, f"prune {path.parent.name}") - for rel, size in data.items() - if isinstance(size, int) - ], - ) - - -def price_mix(conn: sqlite3.Connection, now: float, days: float = 7.0) -> float: - """Call-weighted cache-read USD/MTok over the last ``days``, into ``meta``. - - A standing file costs its tokens x calls/day x this price: every call - re-reads the whole prefix from cache. - """ - book = pricing.load() - rows = conn.execute( - "SELECT model, COUNT(*) FROM turns WHERE ts > ? GROUP BY model", - (now - days * 86400,), - ).fetchall() - calls = sum(n for _, n in rows) - price = ( - sum(book.for_model(m).cache_read * n for m, n in rows) / calls if calls else 0.0 - ) - conn.execute( - "INSERT OR REPLACE INTO meta VALUES ('cache_read_usd_per_mtok', ?)", - (str(price),), - ) - return price diff --git a/python_pkg/token_dashboard/detail.py b/python_pkg/token_dashboard/detail.py deleted file mode 100644 index 4dda523a..00000000 --- a/python_pkg/token_dashboard/detail.py +++ /dev/null @@ -1,207 +0,0 @@ -"""Collect the detail rows (calls, marks, turn facts) during a file walk. - -The walker feeds every record, tool call, tool result and priced turn here; -:meth:`Detail.finish` settles creation and carry cost on each row. Marks come -from what Claude Code writes next to the conversation (see :mod:`marks`). -""" - -from __future__ import annotations - -import json -from typing import TYPE_CHECKING, Any - -from python_pkg.token_audit import parse, records -from python_pkg.token_audit.pricing import WRITE_1H, WRITE_5M -from python_pkg.token_dashboard import blocks, carry, classify, hook_join, marks -from python_pkg.token_dashboard.detail_rows import ( - CallRow, - DetailRows, - MarkRow, - TurnDetail, - Where, -) - -if TYPE_CHECKING: - from python_pkg.token_audit.model import Turn - from python_pkg.token_audit.pricing import PriceBook - -CHARS = parse.CHARS_PER_TOKEN - - -def write_multiplier(usage: dict[str, int]) -> float: - """Average cache-write price of one call, as a multiple of input.""" - total = usage.get("cache_creation_input_tokens", 0) - if not total: - return WRITE_5M - one_hour = min(usage.get(records.CACHE_1H, 0), total) - return (one_hour * WRITE_1H + (total - one_hour) * WRITE_5M) / total - - -def _message(rec: dict[str, Any]) -> dict[str, Any]: - """A record's ``message`` dict, empty when absent.""" - message = rec.get("message") - return message if isinstance(message, dict) else {} - - -class _Output: - """Visible output of one API message, accumulated over its records.""" - - def __init__(self, latency: float) -> None: - """Start empty; ``latency`` is the gap before the first record.""" - self.latency = latency - self.text = 0 - self.tool = 0 - self.thinking = False - - def add(self, block: dict[str, Any]) -> None: - """Count one content block.""" - kind = block.get("type") - if kind == "text": - self.text += len(str(block.get("text") or "")) - elif kind == "tool_use": - self.tool += len(json.dumps(block.get("input") or {})) - elif kind in {"thinking", "redacted_thinking"}: - self.thinking = True - - def split(self, output: int) -> tuple[int, int, int]: - """``(text, tool, thinking)`` tokens summing to ``output``. - - Visible blocks are estimated by length; billed output left over is - thinking (whose text Claude Code does not keep). With no thinking - block, or more visible text than output, the visible estimates are - scaled to the billed total instead. - """ - text, tool = self.text // CHARS, self.tool // CHARS - visible = text + tool - if not visible: - return (0, 0, output) if self.thinking else (output, 0, 0) - if visible > output or not self.thinking: - text = round(output * text / visible) - return text, output - text, 0 - return text, tool, output - visible - - -class Detail: - """Detail state for one file.""" - - def __init__(self) -> None: - """Nothing seen yet.""" - self.rows = DetailRows() - self.tracker = carry.Tracker() - self.calls: dict[str, CallRow] = {} - self.outputs: dict[str, _Output] = {} - self.last_ts = 0.0 - self.model = "" - self.hooks = hook_join.Hooks() - - def mark(self, found: marks.Found, ts: float, prompt: int) -> MarkRow: - """Turn a found mark into a row; carry it if it sits in context.""" - row = MarkRow( - Where(prompt, ts, self.model), - found.kind, - found.name, - detail=found.detail, - count=found.count, - tokens=found.tokens, - extra=found.extra, - duration_ms=found.duration_ms, - ) - self.rows.marks.append(row) - if row.tokens and found.kind not in marks.NOT_IN_CONTEXT: - self.tracker.result(row.cost, row.tokens) - self.hooks.track(row, found) - return row - - def record(self, rec: dict[str, Any], ts: float, prompt: int) -> None: - """Account one record (after the walker opened any new prompt).""" - kind = rec.get("type") - if kind == "assistant": - self.assistant(rec, ts, prompt) - elif kind == "user" and blocks.prompt_text(rec) is not None: - self.hooks.prompt_opened() - self.last_ts = ts - for found in marks.of_record(rec): - self.mark(found, ts, prompt) - - def assistant(self, rec: dict[str, Any], ts: float, prompt: int) -> None: - """Error placeholders become marks; real messages add visible output.""" - message = _message(rec) - if rec.get("isApiErrorMessage"): - text = blocks.text_of(message.get("content")) - detail = str(rec.get("apiError") or text)[: classify.TARGET_CHARS] - name = str(rec.get("error") or "unknown") - self.mark(marks.Found("api_error", name, detail=detail), ts, prompt) - return - mid = str(message.get("id") or "") - if mid not in self.outputs: - self.outputs[mid] = _Output(max(0.0, ts - (self.last_ts or ts))) - for block in parse.content_blocks(rec): - self.outputs[mid].add(block) - - def use( - self, rec: dict[str, Any], block: dict[str, Any], ts: float, prompt: int - ) -> None: - """Open a call row for a ``tool_use`` block.""" - message = _message(rec) - raw = block.get("input") - tool_input: dict[str, object] = raw if isinstance(raw, dict) else {} - name = str(block.get("name") or "unknown") - row = CallRow( - Where(prompt, ts, str(message.get("model") or self.model)), - name=name, - family=classify.family(name, tool_input), - target=classify.target(name, tool_input), - sig=classify.signature(name, tool_input), - message_id=str(message.get("id") or ""), - ) - self.calls[str(block.get("id") or "")] = row - self.rows.calls.append(row) - - def result(self, block: dict[str, Any], ts: float) -> None: - """Close the call a ``tool_result`` answers.""" - row = self.calls.pop(str(block.get("tool_use_id") or ""), None) - if row is None: - return - content = block.get("content") - images, text = marks.split_images(content) - row.end_ts = ts - row.images = images - row.result_tokens = images * parse.IMAGE_TOKENS + len(text) // CHARS - if block.get("is_error"): - row.error_kind = classify.error_kind(blocks.text_of(content)) - self.tracker.result(row.cost, row.result_tokens) - - def turn(self, turn: Turn, cost: float, prompt: int) -> None: - """Note a priced API call.""" - usage = turn.usage - self.model = turn.model - self.tracker.turn(turn.model, turn.context, write_multiplier(usage)) - self.hooks.turn(cost, turn.timestamp) - output = self.outputs.pop(turn.message_id, None) or _Output(0.0) - text, tool, think = output.split(usage.get("output_tokens", 0)) - one_hour = usage.get(records.CACHE_1H, 0) - self.rows.turns.append( - TurnDetail( - Where(prompt, turn.timestamp, turn.model), - latency_s=output.latency, - cw_1h=one_hour, - cw_5m=usage.get("cache_creation_input_tokens", 0) - one_hour, - out_text=text, - out_tool=tool, - out_think=think, - tool_calls=turn.tool_calls, - ) - ) - - def finish(self, book: PriceBook) -> DetailRows: - """Settle costs, fill batch sizes and models, merge repeated marks.""" - self.tracker.settle(book) - per_message: dict[str, int] = {} - for call in self.rows.calls: - per_message[call.message_id] = per_message.get(call.message_id, 0) + 1 - for call in self.rows.calls: - call.batch = per_message[call.message_id] - first_model = self.rows.turns[0].where.model if self.rows.turns else "" - self.hooks.resolve() - self.rows.marks = hook_join.merge(self.rows.marks, first_model) - return self.rows diff --git a/python_pkg/token_dashboard/detail_rows.py b/python_pkg/token_dashboard/detail_rows.py deleted file mode 100644 index acf09d71..00000000 --- a/python_pkg/token_dashboard/detail_rows.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Row types for the detail tables: one tool call, one context mark, one turn. - -Each call and mark carries a :class:`Cost` (see :mod:`carry`): what putting -the chunk into the prompt cache cost, and what re-reading it on every later -call of the same file cost until a compaction dropped it. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field - - -@dataclass -class Cost: - """Creation + carry of one chunk of context, and the calls that carried it.""" - - create_usd: float = 0.0 - carry_usd: float = 0.0 - carry_calls: int = 0 - - -@dataclass(frozen=True) -class Where: - """When something happened: prompt index, timestamp, serving model.""" - - prompt_idx: int - ts: float - model: str - - -@dataclass -class CallRow: - """One tool call, paired with its result (``error_kind`` empty when ok).""" - - where: Where - name: str - family: str - target: str - sig: str - message_id: str - end_ts: float = 0.0 - batch: int = 1 - result_tokens: int = 0 - images: int = 0 - error_kind: str = "" - cost: Cost = field(default_factory=Cost) - - -@dataclass -class MarkRow: - """Something that entered (or touched) context without being a tool call. - - ``kind`` is one of ``hook_run``, ``hook_context``, ``hook_block``, - ``hook_error``, ``hook_msg``, ``attach``, ``compaction``, ``api_error``, - ``interrupt``, ``image``. Rows with the same prompt, kind, name, detail - and model are summed into one, with ``count`` occurrences. - """ - - where: Where - kind: str - name: str - detail: str = "" - count: int = 1 - tokens: int = 0 - extra: int = 0 - duration_ms: int = 0 - retry_usd: float = 0.0 - cost: Cost = field(default_factory=Cost) - - -@dataclass(frozen=True) -class TurnDetail: - """Per-API-call facts the ``turns`` table does not hold.""" - - where: Where - latency_s: float - cw_1h: int - cw_5m: int - out_text: int - out_tool: int - out_think: int - tool_calls: int - - -@dataclass -class DetailRows: - """Everything one file contributes to the detail tables.""" - - calls: list[CallRow] = field(default_factory=list) - marks: list[MarkRow] = field(default_factory=list) - turns: list[TurnDetail] = field(default_factory=list) diff --git a/python_pkg/token_dashboard/detail_store.py b/python_pkg/token_dashboard/detail_store.py deleted file mode 100644 index 00d3bb2a..00000000 --- a/python_pkg/token_dashboard/detail_store.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Write one file's detail rows (calls, marks, turn facts) into the store. - -Every row repeats the owning file's key, main session key, subagent flag and -cwd, plus its own model, timestamp and local day, so the dashboard's shared -filter clause (date range, cwd, model, scope) applies to these tables as-is. -""" - -from __future__ import annotations - -from datetime import datetime -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - import sqlite3 - - from python_pkg.token_dashboard.detail_rows import Cost, Where - from python_pkg.token_dashboard.rows import FileFacts - -CLEAR = ( - "DELETE FROM calls WHERE key = ?", - "DELETE FROM marks WHERE key = ?", - "DELETE FROM turn_detail WHERE key = ?", -) - -Owner = tuple[str, str, int, str | None] - - -def day_of(stamp: float) -> str: - """Local calendar day of a unix timestamp.""" - return datetime.fromtimestamp(stamp).astimezone().strftime("%Y-%m-%d") - - -def _where(where: Where) -> tuple[str, float, str, int]: - """``model, ts, day, prompt_idx`` columns.""" - return (where.model, where.ts, day_of(where.ts), where.prompt_idx) - - -def _cost(cost: Cost) -> tuple[float, float, int]: - """``create_usd, carry_usd, carry_calls`` columns.""" - return (cost.create_usd, cost.carry_usd, cost.carry_calls) - - -def write(conn: sqlite3.Connection, facts: FileFacts, owner: Owner) -> None: - """Insert the detail rows; :func:`store.write` already cleared old ones.""" - detail = facts.rows.detail - conn.executemany( - "INSERT INTO calls (key, main_key, is_sub, cwd, model, ts, day," - " prompt_idx, end_ts, name, family, target, sig, batch, result_tokens," - " images, is_error, error_kind, create_usd, carry_usd, carry_calls)" - " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", - [ - ( - *owner, - *_where(c.where), - c.end_ts, - c.name, - c.family, - c.target, - c.sig, - c.batch, - c.result_tokens, - c.images, - int(bool(c.error_kind)), - c.error_kind, - *_cost(c.cost), - ) - for c in detail.calls - ], - ) - conn.executemany( - "INSERT INTO marks (key, main_key, is_sub, cwd, model, ts, day," - " prompt_idx, kind, name, detail, count, tokens, extra, duration_ms," - " retry_usd, create_usd, carry_usd, carry_calls)" - " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", - [ - ( - *owner, - *_where(m.where), - m.kind, - m.name, - m.detail, - m.count, - m.tokens, - m.extra, - m.duration_ms, - m.retry_usd, - *_cost(m.cost), - ) - for m in detail.marks - ], - ) - conn.executemany( - "INSERT INTO turn_detail (key, main_key, is_sub, cwd, model, ts, day," - " prompt_idx, latency_s, cw_1h, cw_5m, out_text, out_tool, out_think," - " tool_calls) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", - [ - ( - *owner, - *_where(t.where), - t.latency_s, - t.cw_1h, - t.cw_5m, - t.out_text, - t.out_tool, - t.out_think, - t.tool_calls, - ) - for t in detail.turns - ], - ) diff --git a/python_pkg/token_dashboard/efficiency.py b/python_pkg/token_dashboard/efficiency.py deleted file mode 100644 index 5cf770ac..00000000 --- a/python_pkg/token_dashboard/efficiency.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Three 0-token efficiency trends, weekly. - -1. **USD per DONE session** -- all spend of status-tagged sessions in a week - divided by how many of them ended on ``DONE:``. Failed and parked sessions - count against the done ones, which is the point. Sessions with no tag at all - (before the convention) are counted but kept out of the ratio. -2. **USD per surviving commit** -- the week's total spend divided by commits - made that week that survived :mod:`survival`'s 14-day rule. -3. **USD per estimated size unit** -- spend of sessions the task-size estimator - sized, divided by the sum of its ``est_calls`` for them. -""" - -from __future__ import annotations - -from collections import defaultdict -from dataclasses import dataclass -from datetime import datetime, timedelta -import json -from typing import TYPE_CHECKING, Any - -from python_pkg.token_dashboard import survival - -if TYPE_CHECKING: - from collections.abc import Iterable - import sqlite3 - -Week = dict[str, Any] - - -@dataclass(frozen=True) -class SessionCost: - """What one main session cost and how it ended.""" - - key: str - first_ts: float - usd: float - tag: str | None - - -def week_of(stamp: float) -> str: - """Monday (local time) of the week containing ``stamp``.""" - day = datetime.fromtimestamp(stamp).astimezone().date() - return (day - timedelta(days=day.weekday())).isoformat() - - -def _ratio(part: float, whole: float) -> float | None: - """``part / whole``, ``None`` when ``whole`` is zero.""" - return part / whole if whole else None - - -def done_weekly(rows: Iterable[SessionCost]) -> list[Week]: - """USD per DONE session and the DONE share, per week.""" - weeks: dict[str, Week] = defaultdict( - lambda: {"done": 0, "other": 0, "untagged": 0, "usd_tagged": 0.0} - ) - for row in rows: - week = weeks[week_of(row.first_ts)] - if row.tag is None: - week["untagged"] += 1 - continue - week["usd_tagged"] += row.usd - week["done" if row.tag == "DONE" else "other"] += 1 - out = [] - for name in sorted(weeks): - week = weeks[name] - tagged = week["done"] + week["other"] - week |= { - "week": name, - "usd_per_done": _ratio(week["usd_tagged"], week["done"]), - "done_share": _ratio(week["done"], tagged), - } - out.append(week) - return out - - -def commits_weekly( - verdicts: Iterable[tuple[float, str]], week_usd: dict[str, float] -) -> list[Week]: - """USD per surviving commit, with lost / pending / unknown counts.""" - weeks: dict[str, Week] = defaultdict( - lambda: {"survived": 0, "lost": 0, "pending": 0, "unknown": 0} - ) - for stamp, verdict in verdicts: - week = weeks[week_of(stamp)] - if verdict == "survived": - week["survived"] += 1 - elif verdict in survival.FINAL: - week["lost"] += 1 - elif verdict == "pending": - week["pending"] += 1 - else: - week["unknown"] += 1 - out = [] - for name in sorted(weeks): - usd = week_usd.get(name, 0.0) - week = weeks[name] - week |= { - "week": name, - "usd": usd, - "usd_per_survivor": _ratio(usd, week["survived"]), - } - out.append(week) - return out - - -def estimate_weekly(rows: Iterable[SessionCost], est: dict[str, int]) -> list[Week]: - """USD per estimated call, over sessions the estimator sized.""" - weeks: dict[str, Week] = defaultdict(lambda: {"usd": 0.0, "est_calls": 0, "n": 0}) - for row in rows: - if row.key not in est: - continue - week = weeks[week_of(row.first_ts)] - week["usd"] += row.usd - week["est_calls"] += est[row.key] - week["n"] += 1 - return [ - weeks[name] - | { - "week": name, - "usd_per_unit": _ratio(weeks[name]["usd"], weeks[name]["est_calls"]), - } - for name in sorted(weeks) - ] - - -def rebuild(conn: sqlite3.Connection, now: float) -> None: - """Recompute the three weekly series into ``meta['efficiency']``.""" - rows = [ - SessionCost(key, first, usd, tag) - for key, first, usd, tag in conn.execute( - "SELECT key, first_ts, usd, final_tag FROM session_summary" - ) - ] - week_usd: dict[str, float] = defaultdict(float) - for row in rows: - week_usd[week_of(row.first_ts)] += row.usd - judged = conn.execute( - "SELECT c.ts, v.verdict FROM session_commits c" - " LEFT JOIN commit_verdicts v ON v.sha = c.sha" - ).fetchall() - verdicts = [ - (ts, verdict or ("pending" if now - ts < survival.WINDOW_S else "unchecked")) - for ts, verdict in judged - ] - est = dict(conn.execute("SELECT session_id, est_calls FROM estimates")) - payload = { - "done": done_weekly(rows), - "commits": commits_weekly(verdicts, week_usd), - "estimate": estimate_weekly(rows, est), - } - conn.execute( - "INSERT OR REPLACE INTO meta VALUES ('efficiency', ?)", (json.dumps(payload),) - ) diff --git a/python_pkg/token_dashboard/finalize.py b/python_pkg/token_dashboard/finalize.py deleted file mode 100644 index 926b9a43..00000000 --- a/python_pkg/token_dashboard/finalize.py +++ /dev/null @@ -1,217 +0,0 @@ -"""Derive per-session, per-prompt and per-subagent summaries from raw rows. - -Run for every main session touched by an ingest pass. A subagent file may be -ingested in a different pass than its parent, so everything here is computed -from the store, never from the in-memory walk. - -Linking a subagent to the prompt that spawned it: the ``toolUseId`` in its -``.meta.json`` (or the ``agentId`` the ``Agent`` result reported) names an -``Agent`` call; that call's prompt is the answer. A depth-2 agent's call sits -in its parent agent's file, so the chain is followed upward. An agent with no -resolvable spawn -- one that outlived ``/clear`` -- stays ``prompt_idx NULL`` -("unattributed") rather than being dropped, so session cost always reconciles. -""" - -from __future__ import annotations - -from collections import defaultdict -import json -from typing import TYPE_CHECKING, Any - -from python_pkg.token_dashboard import active - -if TYPE_CHECKING: - import sqlite3 - -_MAX_DEPTH = 8 - - -def _rows(conn: sqlite3.Connection, sql: str, *args: object) -> list[Any]: - """Run a query and return all rows.""" - return conn.execute(sql, args).fetchall() - - -class _Session: - """Everything stored for one main session and its subagents.""" - - def __init__(self, conn: sqlite3.Connection, main: str) -> None: - self.main = main - self.sessions = { - r[0]: r - for r in _rows( - conn, - "SELECT key, cwd, agent_type, description, spawn_tool_use_id," - " agent_model, first_ts, last_ts, final_tag" - " FROM sessions WHERE main_key = ?", - main, - ) - } - self.turns = _rows( - conn, - "SELECT key, prompt_idx, usd, model FROM turns WHERE main_key = ?", - main, - ) - keys = json.dumps(list(self.sessions)) - self.calls = _rows( - conn, - "SELECT key, tool_use_id, agent_id, prompt_idx, description," - " subagent_type FROM agent_calls WHERE key IN" - " (SELECT value FROM json_each(?))", - keys, - ) - self.spans_by_key: dict[str, list[active.Span]] = defaultdict(list) - for key, start, end in _rows( - conn, - "SELECT key, start, end FROM activity WHERE key IN" - " (SELECT value FROM json_each(?))", - keys, - ): - self.spans_by_key[key].append((start, end)) - self.spans = [s for spans in self.spans_by_key.values() for s in spans] - self.prompts = _rows( - conn, - "SELECT idx, ts, text FROM prompts WHERE key = ? ORDER BY idx", - main, - ) - self.usd: dict[str, float] = defaultdict(float) - self.ncalls: dict[str, int] = defaultdict(int) - self.models: dict[str, dict[str, float]] = defaultdict( - lambda: defaultdict(float) - ) - for key, _, usd, model in self.turns: - self.usd[key] += usd - self.ncalls[key] += 1 - self.models[key][model] += usd - - def spawn(self, sub: str) -> tuple[Any, ...] | None: - """The ``Agent`` call that spawned ``sub``, if any is stored.""" - tool_use_id = self.sessions[sub][4] - agent_id = sub.rsplit("/agent-", 1)[-1] - for call in self.calls: - if (tool_use_id and call[1] == tool_use_id) or call[2] == agent_id: - return tuple(call) - return None - - def resolve(self, sub: str) -> tuple[int | None, int, tuple[Any, ...] | None]: - """``(prompt_idx, depth, spawning call)`` for a subagent.""" - first = self.spawn(sub) - call, depth = first, 1 - while call is not None and call[0] != self.main and depth < _MAX_DEPTH: - call = self.spawn(call[0]) if call[0] in self.sessions else None - depth += 1 - return (call[3] if call is not None else None), depth, first - - -def _sub_rows(state: _Session) -> list[tuple[Any, ...]]: - """One ``subagent_link`` row per subagent of the session.""" - out = [] - for key, row in state.sessions.items(): - if key == state.main: - continue - prompt_idx, depth, call = state.resolve(key) - models = state.models.get(key) or {} - model = max(models, key=models.__getitem__) if models else row[5] - spans = state.spans_by_key.get(key, []) - out.append( - ( - key, - state.main, - prompt_idx, - depth, - row[3] or (call[4] if call else "") or key, - row[2] or (call[5] if call else "") or "unknown", - model, - state.usd.get(key, 0.0), - state.ncalls.get(key, 0), - row[6], - row[7], - row[7] - row[6], - active.measure(spans), - ) - ) - return out - - -def refresh(conn: sqlite3.Connection, main: str) -> None: - """Rebuild every summary row for one main session.""" - state = _Session(conn, main) - conn.execute("DELETE FROM subagent_link WHERE main_key = ?", (main,)) - conn.execute("DELETE FROM prompt_summary WHERE key = ?", (main,)) - conn.execute("DELETE FROM session_summary WHERE key = ?", (main,)) - if not state.sessions: - return - subs = _sub_rows(state) - conn.executemany( - "INSERT INTO subagent_link VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", subs - ) - first = min((r[6] for r in state.sessions.values() if r[6]), default=0.0) - last = max(r[7] for r in state.sessions.values()) - _prompt_rows(conn, state, subs, last) - own = state.sessions.get(main) - usd_main = state.usd.get(main, 0.0) - usd_sub = sum(r[7] for r in subs) - merged: dict[str, float] = defaultdict(float) - for models in state.models.values(): - for model, usd in models.items(): - merged[model] += usd - conn.execute( - "INSERT INTO session_summary VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", - ( - main, - own[1] if own else next(iter(state.sessions.values()))[1], - state.prompts[0][2] if state.prompts else (subs[0][4] if subs else ""), - first, - last, - last - first, - active.measure(state.spans), - usd_main + usd_sub, - usd_main, - usd_sub, - sum(state.ncalls.values()), - len(state.prompts), - len(subs), - json.dumps(dict(merged)), - own[8] if own else None, - ), - ) - - -def _prompt_rows( - conn: sqlite3.Connection, - state: _Session, - subs: list[tuple[Any, ...]], - last: float, -) -> None: - """One ``prompt_summary`` row per prompt, plus one for pre-prompt turns.""" - own: dict[int, float] = defaultdict(float) - calls: dict[int, int] = defaultdict(int) - for key, idx, usd, _ in state.turns: - if key == state.main: - own[idx] += usd - calls[idx] += 1 - sub_usd: dict[int, float] = defaultdict(float) - for row in subs: - if row[2] is not None: - sub_usd[row[2]] += row[7] - prompts = list(state.prompts) - if own.get(-1) or sub_usd.get(-1): - start = min((s for s, _ in state.spans), default=0.0) - prompts.insert(0, (-1, start, "(before the first prompt)")) - bounds = [ts for _, ts, _ in prompts[1:]] + ([last + 1] if prompts else []) - conn.executemany( - "INSERT INTO prompt_summary VALUES (?,?,?,?,?,?,?,?,?)", - [ - ( - state.main, - idx, - ts, - text, - own.get(idx, 0.0), - sub_usd.get(idx, 0.0), - active.measure(state.spans, ts, end), - min(end, last) - ts, - calls.get(idx, 0), - ) - for (idx, ts, text), end in zip(prompts, bounds, strict=True) - ], - ) diff --git a/python_pkg/token_dashboard/gitcmd.py b/python_pkg/token_dashboard/gitcmd.py deleted file mode 100644 index 22f3d9ac..00000000 --- a/python_pkg/token_dashboard/gitcmd.py +++ /dev/null @@ -1,29 +0,0 @@ -"""One way to run git for the dashboard: absolute binary, bounded, never raises.""" - -from __future__ import annotations - -import shutil -import subprocess -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from pathlib import Path - -GIT = shutil.which("git") or "/usr/bin/git" -TIMEOUT_S = 30 - - -def run(repo: str | Path, *args: str, stdin: str | None = None) -> str | None: - """``git -C repo args``: stdout, or ``None`` on any failure.""" - try: - done = subprocess.run( - [GIT, "-C", str(repo), *args], - input=stdin, - capture_output=True, - text=True, - check=False, - timeout=TIMEOUT_S, - ) - except OSError, subprocess.TimeoutExpired: - return None - return done.stdout if done.returncode == 0 else None diff --git a/python_pkg/token_dashboard/hook_join.py b/python_pkg/token_dashboard/hook_join.py deleted file mode 100644 index d8b15fbc..00000000 --- a/python_pkg/token_dashboard/hook_join.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Join hook context to the run that produced it; price forced retries; merge. - -``hook_additional_context`` names no command, so it is matched to a -``hook_success`` run of the same tool-use id and hook name whose stdout -contains the context text. A ``hook_blocking_error`` costs the calls it -forces: a Stop bounce every call until the next prompt, any other block the -one call that reacts to it. -""" - -from __future__ import annotations - -from dataclasses import replace -import json -from typing import TYPE_CHECKING - -from python_pkg.token_dashboard.marks import SNIPPET - -if TYPE_CHECKING: - from python_pkg.token_dashboard.detail_rows import MarkRow - from python_pkg.token_dashboard.marks import Found - -_STOPS = frozenset({"Stop", "SubagentStop"}) - - -class Hooks: - """Pending joins and open retry windows for one file.""" - - def __init__(self) -> None: - """Nothing seen yet.""" - self.runs: dict[tuple[str, str], list[tuple[str, str]]] = {} - self.pending: list[tuple[MarkRow, tuple[str, str], str]] = [] - self.until_prompt: list[MarkRow] = [] - self.next_turn: list[MarkRow] = [] - - def track(self, row: MarkRow, found: Found) -> None: - """Remember what later records or turns need to complete ``row``.""" - if found.kind == "hook_run": - self.runs.setdefault(found.key, []).append((row.detail, found.text)) - elif found.kind == "hook_context": - self.pending.append((row, found.key, found.text)) - elif found.kind == "hook_block": - stop = row.name in _STOPS - (self.until_prompt if stop else self.next_turn).append(row) - - def prompt_opened(self) -> None: - """A new prompt ends every Stop-bounce retry.""" - self.until_prompt.clear() - - def turn(self, cost: float, ts: float) -> None: - """Charge an API call made after a block to that block's retry. - - Turns are released one message late, so a turn released after the - block record may still predate it: the timestamp decides. - """ - for row in [*self.until_prompt, *self.next_turn]: - if ts > row.where.ts: - row.retry_usd += cost - self.next_turn = [row for row in self.next_turn if ts <= row.where.ts] - - def resolve(self) -> None: - """Name the script behind each injected hook context.""" - for row, key, text in self.pending: - snippet = text[:SNIPPET] - escaped = json.dumps(snippet)[1:-1] - for detail, stdout in self.runs.get(key, []): - if snippet and (snippet in stdout or escaped in stdout): - row.detail = detail - break - - -def merge(rows: list[MarkRow], first_model: str) -> list[MarkRow]: - """Sum marks sharing prompt, kind, name, detail and model; oldest first.""" - merged: dict[tuple[int, str, str, str, str], MarkRow] = {} - for row in rows: - row.where = replace(row.where, model=row.where.model or first_model) - where = row.where - key = (where.prompt_idx, row.kind, row.name, row.detail, where.model) - held = merged.get(key) - if held is None: - merged[key] = row - continue - held.where = replace(held.where, ts=min(held.where.ts, where.ts)) - held.count += row.count - held.tokens += row.tokens - held.extra += row.extra - held.duration_ms += row.duration_ms - held.retry_usd += row.retry_usd - held.cost.create_usd += row.cost.create_usd - held.cost.carry_usd += row.cost.carry_usd - held.cost.carry_calls += row.cost.carry_calls - return sorted(merged.values(), key=lambda r: r.where.ts) diff --git a/python_pkg/token_dashboard/ingest.py b/python_pkg/token_dashboard/ingest.py deleted file mode 100644 index 50190ad0..00000000 --- a/python_pkg/token_dashboard/ingest.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Refresh the token store: changed transcripts first, then every aux source. - -Resumable by construction. Each pass walks changed files until its time budget -is spent (always at least one), commits, and exits :data:`EXIT_PENDING` if any -are left -- so a first backfill of months of transcripts is a handful of -sub-minute runs, and the systemd timer simply picks up where the last ended. - - python3 -m python_pkg.token_dashboard.ingest # one pass - python3 -m python_pkg.token_dashboard.ingest --budget 40 -""" - -from __future__ import annotations - -import argparse -from dataclasses import dataclass -from pathlib import Path -import sys -import time -from typing import TYPE_CHECKING - -from python_pkg.token_audit import discover, history -from python_pkg.token_dashboard import ( - ab, - aux, - commits, - context_files, - efficiency, - finalize, - schema, - store, - survival, - walk, -) - -if TYPE_CHECKING: - import sqlite3 - -ROOT = Path.home() / ".claude" / "projects" -DEFAULT_BUDGET = 40.0 -EXIT_PENDING = 75 # EX_TEMPFAIL: run again to continue - - -@dataclass(frozen=True) -class Outcome: - """What one pass did.""" - - parsed: int - finalized: int - complete: bool - - -def _changed( - conn: sqlite3.Connection, root: Path -) -> list[tuple[Path, tuple[int, float]]]: - """Transcripts whose size or mtime differ from what the store last saw.""" - seen = { - path: (size, mtime) - for path, size, mtime in conn.execute("SELECT path, size, mtime FROM files") - } - out = [] - for pattern in discover.PATTERNS: - for path in root.glob(pattern): - stat = path.stat() - now = (stat.st_size, stat.st_mtime) - if seen.get(str(path)) != now: - out.append((path, now)) - # Oldest first: a backfill lands history in order, and a live session - # (newest mtime) is re-read last, closest to the pass's end. - return sorted(out, key=lambda item: item[1][1]) - - -def run( - root: Path = ROOT, - db_path: Path = schema.DEFAULT_DB, - report_dir: Path = history.REPORT_DIR, - budget_s: float = DEFAULT_BUDGET, -) -> Outcome: - """One resumable pass.""" - started = time.monotonic() - deadline = started + budget_s - conn = schema.connect(db_path) - try: - todo = _changed(conn, root) - mains: set[str] = set() - parsed = 0 - for path, stat in todo: - if parsed and time.monotonic() > deadline: - break - facts = walk.walk_file(path, root) - store.write(conn, facts, str(path), stat) - mains.add(facts.parent_session or facts.key) - parsed += 1 - for main_key in sorted(mains): - finalize.refresh(conn, main_key) - conn.commit() - complete = parsed == len(todo) - if complete: - aux.snapshots(conn, report_dir) - aux.lever_rows(conn, report_dir) - aux.autopsy(conn) - aux.activity_logs(conn, report_dir) - commits.discover(conn) - now = time.time() - complete = survival.update(conn, now, deadline, time.monotonic) - efficiency.rebuild(conn, now) - ab.rebuild(conn) - context_files.refresh(conn) - context_files.price_mix(conn, now) - conn.execute( - "INSERT OR REPLACE INTO meta VALUES ('refreshed_at', ?)", (str(now),) - ) - conn.commit() - finally: - conn.close() - return Outcome(parsed, len(mains), complete) - - -def main(argv: list[str] | None = None) -> int: - """CLI: 0 when the store is current, :data:`EXIT_PENDING` when not.""" - parser = argparse.ArgumentParser(prog="token_dashboard.ingest") - parser.add_argument("--root", type=Path, default=ROOT) - parser.add_argument("--db", type=Path, default=schema.DEFAULT_DB) - parser.add_argument("--report-dir", type=Path, default=history.REPORT_DIR) - parser.add_argument("--budget", type=float, default=DEFAULT_BUDGET) - args = parser.parse_args(argv) - started = time.monotonic() - outcome = run(args.root, args.db, args.report_dir, args.budget) - sys.stderr.write( - f"token_dashboard: parsed {outcome.parsed} files, refreshed " - f"{outcome.finalized} sessions in {time.monotonic() - started:.1f}s" - f"{'' if outcome.complete else ' (more pending: run again)'}\n" - ) - return 0 if outcome.complete else EXIT_PENDING - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/python_pkg/token_dashboard/levers.py b/python_pkg/token_dashboard/levers.py deleted file mode 100644 index dcf309cd..00000000 --- a/python_pkg/token_dashboard/levers.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Every lever ever applied, proposed or passed over, with its outcome. - -Sources, newest form first: - -* ``history/*.json`` ``levers`` -- structured candidates (snapshots from - 2026-10-18 on; see ``token_audit.review``). -* Markdown "Candidate levers" tables -- for older snapshots. WEEKLY.md is - overwritten every run, so older tables survive only in transcript tool - results; :mod:`walk` keeps those as ``lever_tables`` rows. -* ``ledger.jsonl`` -- applied levers, scored by ``token_audit.history.score``. -* ``prune//PROPOSAL.md`` -- weekly standing-prefix prune proposals. - -Status of a candidate: ``applied`` if a ledger line on the same metric is dated -on or after the snapshot, ``not applied`` if a newer snapshot came and went -without one, else ``proposed``. Titles never match ledger text, so the join is -on the metric key. -""" - -from __future__ import annotations - -import json -import re -from typing import TYPE_CHECKING, Any - -from python_pkg.token_audit import history -from python_pkg.token_audit.recommend import TITLE_METRIC - -if TYPE_CHECKING: - from pathlib import Path - -Row = dict[str, Any] -DATE = re.compile(r"\d{4}-\d{2}-\d{2}") -_COLUMNS = 5 - - -def parse_candidates(markdown: str) -> list[Row]: - """Rows of a ``| category | lever | evidence | est $/day | action |`` table.""" - rows = [] - for line in markdown.splitlines(): - cells = [c.strip() for c in line.strip().strip("|").split("|")] - if len(cells) != _COLUMNS or cells[0] in {"category", ""}: - continue - if set(cells[0]) <= {"-"}: - continue - est = cells[3].lstrip("$").replace(",", "") - rows.append( - { - "category": cells[0], - "title": cells[1], - "evidence": cells[2], - "est_usd_per_day": float(est) - if est.replace(".", "").isdigit() - else None, - "action": cells[4], - "metric": TITLE_METRIC.get(cells[1], ""), - } - ) - return rows - - -def ledger_date(entry: Row) -> str | None: - """A ledger line's date, else one named in its text (prune lines).""" - if isinstance(entry.get("date"), str): - return str(entry["date"]) - found = DATE.search(str(entry.get("lever", ""))) - return found.group(0) if found else None - - -def _snapshots(report: Path) -> list[tuple[str, Row]]: - """``(date, snapshot)`` oldest first.""" - out = [] - for path in sorted((report / "history").glob("*.json")): - try: - snap = json.loads(path.read_text(encoding="utf-8")) - except ValueError: - continue - if isinstance(snap, dict): - out.append((path.stem[:10], snap)) - return out - - -def _applied(report: Path, latest: Row) -> list[Row]: - """Ledger lines, scored against the newest snapshot.""" - entries = history.read_ledger(report) - rows = [] - for entry, verdict in zip(entries, history.score(entries, latest), strict=True): - rows.append( - { - "source": "ledger", - "date": ledger_date(entry), - "lever": verdict.lever, - "metric": verdict.metric, - "baseline": verdict.baseline, - "predicted": verdict.predicted, - "actual": verdict.actual, - "lower_is_better": int(bool(entry.get("lower_is_better", True))), - "status": "applied", - "outcome": verdict.outcome, - "detail": f"{verdict.note}; undo: {entry.get('undo', '-')}", - } - ) - return rows - - -def _candidates( - snaps: list[tuple[str, Row]], tables: dict[str, str], applied: list[Row] -) -> list[Row]: - """Candidate rows per snapshot, structured first, markdown second.""" - rows = [] - for position, (day, snap) in enumerate(snaps): - found = snap.get("levers") - if not isinstance(found, list): - found = parse_candidates(tables.get(day, "")) - newer = position + 1 < len(snaps) - for lever in found: - metric = str(lever.get("metric") or "") - hit = any( - a["metric"] == metric and (a["date"] or "") >= day for a in applied - ) - status = "applied" if hit else ("not applied" if newer else "proposed") - rows.append( - { - "source": f"snapshot {day}", - "date": day, - "lever": lever.get("title"), - "metric": metric, - "baseline": snap.get("metrics", {}).get(metric), - "predicted": None, - "actual": None, - "lower_is_better": 1, - "status": status, - "outcome": None, - "detail": f"{lever.get('evidence')} -> {lever.get('action')}" - f" (est ${lever.get('est_usd_per_day') or '-'}/day)", - } - ) - return rows - - -def _prunes(report: Path, applied: list[Row]) -> list[Row]: - """One row per prune proposal.""" - paths = sorted((report / "prune").glob("*/PROPOSAL.md")) - rows = [] - for position, path in enumerate(paths): - day = path.parent.name - lines = [x for x in path.read_text(encoding="utf-8").splitlines() if x.strip()] - hit = any(f"prune {day}" in str(a["lever"]) for a in applied) - latest = position == len(paths) - 1 - status = "applied" if hit else ("proposed" if latest else "not applied") - rows.append( - { - "source": f"prune {day}", - "date": day, - "lever": lines[0].lstrip("# ") if lines else f"prune {day}", - "metric": "standing_tokens_est", - "baseline": None, - "predicted": None, - "actual": None, - "lower_is_better": 1, - "status": status, - "outcome": None, - "detail": lines[1] if len(lines) > 1 else "", - } - ) - return rows - - -def build(report: Path, tables: dict[str, str]) -> list[Row]: - """Every lever row, applied ones first.""" - snaps = _snapshots(report) - latest = snaps[-1][1] if snaps else {} - applied = _applied(report, latest) - return applied + _candidates(snaps, tables, applied) + _prunes(report, applied) diff --git a/python_pkg/token_dashboard/marks.py b/python_pkg/token_dashboard/marks.py deleted file mode 100644 index b98de66f..00000000 --- a/python_pkg/token_dashboard/marks.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Read context marks out of the records that are not the conversation. - -Hooks write ``attachment`` records: ``hook_success`` (every run, with its -command and duration; SessionStart / UserPromptSubmit stdout is injected as -context), ``hook_additional_context`` (injected text, without the command -- -joined back to its run by :mod:`hook_join`), ``hook_blocking_error`` (a block -or a Stop-hook bounce) and ``hook_system_message`` (shown to the user only). -Other attachment types in :data:`INJECTED` are rendered into the prompt; -the rest (``prompt_snapshot``, ``environment`` ...) only record what was sent. -""" - -from __future__ import annotations - -from dataclasses import dataclass -import json -from typing import TYPE_CHECKING, Any - -from python_pkg.token_audit import parse -from python_pkg.token_dashboard import blocks, classify - -if TYPE_CHECKING: - from collections.abc import Iterator - -CHARS = parse.CHARS_PER_TOKEN -# Marks that never sit in the model's context, so carry nothing. -NOT_IN_CONTEXT = frozenset({"hook_msg", "compaction", "api_error", "interrupt"}) -# Hook events whose plain stdout Claude Code adds to the context. -INJECTING = frozenset({"SessionStart", "UserPromptSubmit"}) -INJECTED = frozenset( - { - "nested_memory", - "edited_text_file", - "total_tokens_reminder", - "skill_listing", - "deferred_tools_delta", - "mcp_instructions_delta", - "agent_listing_delta", - "queued_command", - "task_reminder", - "silent_turn_reminder", - "file", - "todo", - "command_permissions", - } -) -_SCRIPTS = (".sh", ".py", ".js", ".ts", ".mjs") -SNIPPET = 40 - - -@dataclass(frozen=True) -class Found: - """One mark read from a record, before it becomes a row. - - ``key`` (tool-use id, hook name) and ``text`` only matter to hook runs - and hook context, which :mod:`hook_join` pairs up. - """ - - kind: str - name: str - detail: str = "" - count: int = 1 - tokens: int = 0 - extra: int = 0 - duration_ms: int = 0 - key: tuple[str, str] = ("", "") - text: str = "" - - -def short_command(command: str) -> str: - """A hook command's script name, or its first 40 characters.""" - words = command.split() - scripts = [w for w in words if w.endswith(_SCRIPTS)] - if scripts: - return scripts[0].rsplit("/", 1)[-1] - return " ".join(words)[:SNIPPET] - - -def _int(value: object) -> int: - """A number that may arrive as a string; 0 when it is neither.""" - try: - return int(float(str(value))) - except ValueError: - return 0 - - -def split_images(content: object) -> tuple[int, str]: - """Image block count and the JSON text of everything else.""" - if not isinstance(content, list): - return 0, "" if content is None else json.dumps(content) - images = [b for b in content if isinstance(b, dict) and b.get("type") == "image"] - rest = [b for b in content if b not in images] - return len(images), json.dumps(rest) - - -def _tokens(text: object) -> int: - """Estimated tokens of a text value.""" - return len(str(text or "")) // CHARS - - -def _hook(att: dict[str, Any]) -> Iterator[Found]: - """Marks of one hook attachment.""" - kind = att.get("type") - name = str(att.get("hookName") or "") - key = (str(att.get("toolUseID") or ""), name) - content = att.get("content") - if kind == "hook_success": - text = str(content or "") - injected = att.get("hookEvent") in INJECTING - injected = injected and not text.lstrip().startswith("{") - yield Found( - "hook_run", - name, - detail=short_command(str(att.get("command") or "")), - duration_ms=_int(att.get("durationMs")), - tokens=_tokens(text) if injected else 0, - key=key, - text=str(att.get("stdout") or ""), - ) - elif kind == "hook_additional_context": - for item in content if isinstance(content, list) else [content]: - text = str(item or "") - yield Found("hook_context", name, tokens=_tokens(text), key=key, text=text) - elif kind == "hook_blocking_error": - raw = att.get("blockingError") - error = raw if isinstance(raw, dict) else {"blockingError": raw} - command = short_command(str(error.get("command") or "")) - yield Found( - "hook_block", name, detail=command, tokens=_tokens(error["blockingError"]) - ) - elif kind == "hook_non_blocking_error": - yield Found("hook_error", name, tokens=_tokens(content)) - elif kind == "hook_system_message": - yield Found("hook_msg", name) - - -def _attachment(att: dict[str, Any]) -> Iterator[Found]: - """Marks of one attachment record.""" - kind = str(att.get("type") or "") - if kind.startswith("hook_"): - yield from _hook(att) - elif kind in INJECTED: - rest = {k: v for k, v in att.items() if k != "type"} - path = str(att.get("path") or "")[: classify.TARGET_CHARS] - yield Found("attach", kind, detail=path, tokens=_tokens(json.dumps(rest))) - - -def _user(rec: dict[str, Any]) -> Iterator[Found]: - """Interruptions and pasted images.""" - message = rec.get("message") - content = message.get("content") if isinstance(message, dict) else None - if blocks.text_of(content).startswith("[Request interrupted"): - yield Found("interrupt", "user") - images, _ = split_images(content) - if images: - yield Found("image", "user", count=images, tokens=images * parse.IMAGE_TOKENS) - - -def of_record(rec: dict[str, Any]) -> list[Found]: - """Every mark one non-assistant record carries.""" - kind = rec.get("type") - att = rec.get("attachment") - if kind == "attachment" and isinstance(att, dict): - return list(_attachment(att)) - if kind == "system" and rec.get("subtype") == "compact_boundary": - meta = rec.get("compactMetadata") - meta = meta if isinstance(meta, dict) else {} - found = Found( - "compaction", - str(meta.get("trigger") or "manual"), - tokens=_int(meta.get("preTokens")), - extra=_int(meta.get("postTokens")), - duration_ms=_int(meta.get("durationMs")), - ) - return [found] - if kind == "user": - return list(_user(rec)) - return [] diff --git a/python_pkg/token_dashboard/rows.py b/python_pkg/token_dashboard/rows.py deleted file mode 100644 index 079264a0..00000000 --- a/python_pkg/token_dashboard/rows.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Row types one transcript file contributes to the store (see :mod:`walk`).""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any - -from python_pkg.token_dashboard.detail_rows import DetailRows - -if TYPE_CHECKING: - from python_pkg.token_dashboard import active - - -@dataclass(frozen=True) -class TurnRow: - """One deduplicated API call, priced.""" - - ts: float - model: str - usd: float - context: int - prompt_idx: int - usage: dict[str, int] - - -@dataclass(frozen=True) -class PromptRow: - """One typed user prompt.""" - - idx: int - ts: float - prompt_id: str - text: str - - -@dataclass -class ToolRow: - """Calls of one tool inside one prompt.""" - - prompt_idx: int - name: str - calls: int = 0 - result_tokens: int = 0 - usd: float = 0.0 - carry_usd: float = 0.0 - # Settled by the carry tracker too, but not stored per tool (the calls - # table holds creation per call). - create_usd: float = 0.0 - carry_calls: int = 0 - - -@dataclass(frozen=True) -class AgentCall: - """An ``Agent`` tool call: the spawn point of a subagent.""" - - tool_use_id: str - agent_id: str - prompt_idx: int - ts: float - description: str - subagent_type: str - model: str - result_ts: float - status: str - - -@dataclass(frozen=True) -class Commit: - """A commit hash printed by a Bash ``git commit`` in this file.""" - - prompt_idx: int - ts: float - end_ts: float - sha: str - repo: str - - -@dataclass -class FileRows: - """The row lists one transcript file contributes.""" - - turns: list[TurnRow] = field(default_factory=list) - prompts: list[PromptRow] = field(default_factory=list) - tools: list[ToolRow] = field(default_factory=list) - agent_calls: list[AgentCall] = field(default_factory=list) - intervals: list[active.Span] = field(default_factory=list) - lever_tables: list[tuple[float, str]] = field(default_factory=list) - commits: list[Commit] = field(default_factory=list) - messages: list[tuple[float, str]] = field(default_factory=list) - detail: DetailRows = field(default_factory=DetailRows) - - -@dataclass -class FileFacts: - """Everything one transcript file contributes to the store.""" - - key: str - session_id: str - parent_session: str | None - cwd: str | None - meta: dict[str, Any] - rows: FileRows = field(default_factory=FileRows) - final_tag: str | None = None - span: tuple[float, float] = (0.0, 0.0) diff --git a/python_pkg/token_dashboard/schema.py b/python_pkg/token_dashboard/schema.py deleted file mode 100644 index 52ee17ca..00000000 --- a/python_pkg/token_dashboard/schema.py +++ /dev/null @@ -1,192 +0,0 @@ -"""SQLite layout of the durable token store. - -Raw tables (``files`` .. ``commits``) hold what one transcript file said and -are replaced wholesale when that file changes. They are **never** deleted when -the file disappears: Claude Code removes transcripts after -``cleanupPeriodDays``, and outliving that is the store's whole purpose. - -Summary tables (``session_summary`` .. ``subagent_link``) are derived from the -raw ones by :mod:`finalize` and may be rebuilt at any time. Aux tables hold the -sources that are not transcripts (snapshots, ledger, levers, autopsy rows). -""" - -from __future__ import annotations - -from pathlib import Path -import sqlite3 - -DEFAULT_DB = Path.home() / "data" / "token-dashboard" / "tokens.sqlite" -VERSION = "3" - -_RAW = """ -CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT); -CREATE TABLE IF NOT EXISTS files( - path TEXT PRIMARY KEY, key TEXT, size INT, mtime REAL); -CREATE TABLE IF NOT EXISTS sessions( - key TEXT PRIMARY KEY, session_id TEXT, main_key TEXT, cwd TEXT, - agent_type TEXT, description TEXT, spawn_tool_use_id TEXT, - parent_agent_id TEXT, agent_model TEXT, first_ts REAL, last_ts REAL, - final_tag TEXT); -CREATE TABLE IF NOT EXISTS turns( - key TEXT, main_key TEXT, is_sub INT, cwd TEXT, ts REAL, day TEXT, - model TEXT, usd REAL, context INT, prompt_idx INT, input INT, - cache_write INT, cache_read INT, output INT); -CREATE INDEX IF NOT EXISTS turns_key ON turns(key); -CREATE INDEX IF NOT EXISTS turns_day ON turns(day); -CREATE TABLE IF NOT EXISTS prompts( - key TEXT, idx INT, ts REAL, prompt_id TEXT, text TEXT); -CREATE INDEX IF NOT EXISTS prompts_key ON prompts(key); -CREATE TABLE IF NOT EXISTS tool_use( - key TEXT, prompt_idx INT, name TEXT, calls INT, result_tokens INT, - usd REAL, carry_usd REAL); -CREATE INDEX IF NOT EXISTS tool_use_key ON tool_use(key); -CREATE TABLE IF NOT EXISTS agent_calls( - key TEXT, tool_use_id TEXT, agent_id TEXT, prompt_idx INT, ts REAL, - description TEXT, subagent_type TEXT, model TEXT, result_ts REAL, - status TEXT); -CREATE INDEX IF NOT EXISTS agent_calls_key ON agent_calls(key); -CREATE TABLE IF NOT EXISTS activity(key TEXT, start REAL, end REAL); -CREATE INDEX IF NOT EXISTS activity_key ON activity(key); -CREATE TABLE IF NOT EXISTS sent_messages(key TEXT, ts REAL, to_agent TEXT); -CREATE TABLE IF NOT EXISTS lever_tables(key TEXT, ts REAL, markdown TEXT); -CREATE TABLE IF NOT EXISTS commits( - key TEXT, prompt_idx INT, ts REAL, end_ts REAL, sha TEXT, repo TEXT); -CREATE TABLE IF NOT EXISTS calls( - key TEXT, main_key TEXT, is_sub INT, cwd TEXT, model TEXT, ts REAL, - day TEXT, prompt_idx INT, end_ts REAL, name TEXT, family TEXT, - target TEXT, sig TEXT, batch INT, result_tokens INT, images INT, - is_error INT, error_kind TEXT, create_usd REAL, carry_usd REAL, - carry_calls INT); -CREATE INDEX IF NOT EXISTS calls_key ON calls(key); -CREATE INDEX IF NOT EXISTS calls_ts ON calls(ts); -CREATE TABLE IF NOT EXISTS marks( - key TEXT, main_key TEXT, is_sub INT, cwd TEXT, model TEXT, ts REAL, - day TEXT, prompt_idx INT, kind TEXT, name TEXT, detail TEXT, count INT, - tokens INT, extra INT, duration_ms INT, retry_usd REAL, create_usd REAL, - carry_usd REAL, carry_calls INT); -CREATE INDEX IF NOT EXISTS marks_key ON marks(key); -CREATE INDEX IF NOT EXISTS marks_ts ON marks(ts); -CREATE TABLE IF NOT EXISTS turn_detail( - key TEXT, main_key TEXT, is_sub INT, cwd TEXT, model TEXT, ts REAL, - day TEXT, prompt_idx INT, latency_s REAL, cw_1h INT, cw_5m INT, - out_text INT, out_tool INT, out_think INT, tool_calls INT); -CREATE INDEX IF NOT EXISTS turn_detail_key ON turn_detail(key); -CREATE INDEX IF NOT EXISTS turn_detail_ts ON turn_detail(ts); -CREATE TABLE IF NOT EXISTS session_commits( - sha TEXT PRIMARY KEY, repo TEXT, ts REAL, key TEXT, prompt_idx INT); -""" - -_DERIVED = """ -CREATE TABLE IF NOT EXISTS session_summary( - key TEXT PRIMARY KEY, cwd TEXT, title TEXT, first_ts REAL, last_ts REAL, - wall_s REAL, active_s REAL, usd REAL, usd_main REAL, usd_sub REAL, - calls INT, prompts INT, subagents INT, models TEXT, final_tag TEXT); -CREATE TABLE IF NOT EXISTS prompt_summary( - key TEXT, idx INT, ts REAL, text TEXT, usd_own REAL, usd_sub REAL, - active_s REAL, wall_s REAL, calls INT); -CREATE INDEX IF NOT EXISTS prompt_summary_key ON prompt_summary(key); -CREATE TABLE IF NOT EXISTS subagent_link( - sub_key TEXT PRIMARY KEY, main_key TEXT, prompt_idx INT, depth INT, - description TEXT, agent_type TEXT, model TEXT, usd REAL, calls INT, - first_ts REAL, last_ts REAL, wall_s REAL, active_s REAL); -CREATE INDEX IF NOT EXISTS subagent_link_main ON subagent_link(main_key); -CREATE TABLE IF NOT EXISTS commit_verdicts( - sha TEXT PRIMARY KEY, repo TEXT, ts REAL, added INT, kept INT, - verdict TEXT); -CREATE VIEW IF NOT EXISTS daily AS - SELECT day, SUM(usd) AS usd, COUNT(*) AS calls, - SUM(CASE WHEN is_sub THEN usd ELSE 0 END) AS usd_sub, - SUM(CASE WHEN model LIKE '%opus%' THEN usd ELSE 0 END) AS usd_opus, - SUM(CASE WHEN context > 300000 THEN usd ELSE 0 END) AS usd_over - FROM turns GROUP BY day; -""" - -_AUX = """ -CREATE TABLE IF NOT EXISTS snapshots( - name TEXT PRIMARY KEY, generated TEXT, since REAL, until REAL, - metrics TEXT); -CREATE TABLE IF NOT EXISTS levers( - source TEXT, date TEXT, lever TEXT, metric TEXT, baseline REAL, - predicted REAL, actual REAL, lower_is_better INT, status TEXT, - outcome TEXT, detail TEXT); -CREATE TABLE IF NOT EXISTS aggregate_sessions( - session_id TEXT PRIMARY KEY, source TEXT, day TEXT, started REAL, - cwd TEXT, title TEXT, input INT, cache_write INT, cache_read INT, - output INT, api_turns INT, records INT, usd_est REAL); -CREATE TABLE IF NOT EXISTS prompt_activity(day TEXT PRIMARY KEY, prompts INT); -CREATE TABLE IF NOT EXISTS events(source TEXT, ts REAL, kind TEXT, detail TEXT); -CREATE UNIQUE INDEX IF NOT EXISTS events_once ON events(source, ts, kind, detail); -CREATE TABLE IF NOT EXISTS context_files( - path TEXT PRIMARY KEY, kind TEXT, standing TEXT, size_limit INT); -CREATE TABLE IF NOT EXISTS context_sizes( - path TEXT, ts REAL, bytes INT, source TEXT, PRIMARY KEY (path, ts, source)); -CREATE TABLE IF NOT EXISTS estimates(session_id TEXT PRIMARY KEY, est_calls INT); -""" - - -# Ordered, additive migrations: MIGRATIONS[n] takes a store from n-1 to n. -# The store must never be deleted to change its schema -- rows for expired -# transcripts cannot be regenerated -- so every change lands here instead. -# New tables need no entry: CREATE TABLE IF NOT EXISTS adds them. -MIGRATIONS: dict[int, tuple[str, ...]] = { - 2: ("ALTER TABLE commits ADD COLUMN end_ts REAL",), - 3: ( - "ALTER TABLE agent_calls ADD COLUMN result_ts REAL", - "ALTER TABLE agent_calls ADD COLUMN status TEXT", - ), -} - - -class SchemaError(RuntimeError): - """The store was written by a newer (or unknown) schema.""" - - def __init__(self, found: object) -> None: - """Name the version found and the one this code knows.""" - super().__init__(f"store schema {found!r}; this code knows {VERSION}") - - -def _stored_version(conn: sqlite3.Connection) -> int | None: - """The schema the store was written with, ``None`` for a fresh file.""" - has_meta = conn.execute( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name='meta'" - ).fetchone() - if has_meta is None: - return None - row = conn.execute("SELECT value FROM meta WHERE key='schema'").fetchone() - if row is None or not str(row[0]).isdigit(): - raise SchemaError(row) - return int(row[0]) - - -def _migrate(conn: sqlite3.Connection, stored: int) -> None: - """Apply every migration newer than ``stored``, in order.""" - current = int(VERSION) - if stored > current: - raise SchemaError(stored) - for step in range(stored + 1, current + 1): - for statement in MIGRATIONS.get(step, ()): - table, column = statement.split()[2], statement.split()[5] - present = {r[1] for r in conn.execute(f"PRAGMA table_info({table})")} - if column not in present: - conn.execute(statement) - - -def connect(path: Path, *, readonly: bool = False) -> sqlite3.Connection: - """Open the store; create missing tables and migrate when writing.""" - if readonly: - return sqlite3.connect(f"file:{path}?mode=ro", uri=True) - path.parent.mkdir(parents=True, exist_ok=True) - conn = sqlite3.connect(path) - try: - stored = _stored_version(conn) - # Missing tables first, at today's shape; then columns old tables lack. - conn.executescript(_RAW + _DERIVED + _AUX) - if stored is not None: - _migrate(conn, stored) - except SchemaError: - conn.close() - raise - conn.execute("PRAGMA journal_mode=WAL") - conn.execute("INSERT OR REPLACE INTO meta VALUES ('schema', ?)", (VERSION,)) - conn.commit() - return conn diff --git a/python_pkg/token_dashboard/store.py b/python_pkg/token_dashboard/store.py deleted file mode 100644 index a2961d06..00000000 --- a/python_pkg/token_dashboard/store.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Write one file's :class:`FileFacts` into the raw tables, replacing old rows. - -The unit of change is the whole file: a changed transcript is re-walked and -its rows replaced in one transaction. Resuming from a byte offset would need -the dedup state (:class:`TurnMerger`) persisted across runs, and re-walking -even the largest transcript costs a fraction of a second. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from python_pkg.token_dashboard import detail_store -from python_pkg.token_dashboard.detail_store import day_of - -if TYPE_CHECKING: - import sqlite3 - - from python_pkg.token_dashboard.walk import FileFacts - -# One literal statement per raw table: no SQL is assembled from strings. -CLEAR = ( - "DELETE FROM turns WHERE key = ?", - "DELETE FROM prompts WHERE key = ?", - "DELETE FROM tool_use WHERE key = ?", - "DELETE FROM agent_calls WHERE key = ?", - "DELETE FROM activity WHERE key = ?", - "DELETE FROM lever_tables WHERE key = ?", - "DELETE FROM commits WHERE key = ?", - "DELETE FROM sent_messages WHERE key = ?", - *detail_store.CLEAR, -) - - -def write( - conn: sqlite3.Connection, facts: FileFacts, path: str, stat: tuple[int, float] -) -> None: - """Replace every row ``facts.key`` owns. Caller commits.""" - key = facts.key - for statement in CLEAR: - conn.execute(statement, (key,)) - main_key = facts.parent_session or key - meta = facts.meta - conn.execute( - "INSERT OR REPLACE INTO sessions VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", - ( - key, - facts.session_id, - main_key, - facts.cwd, - meta.get("agentType"), - meta.get("description"), - meta.get("toolUseId"), - meta.get("parentAgentId"), - meta.get("model"), - *facts.span, - facts.final_tag, - ), - ) - is_sub = int(facts.parent_session is not None) - conn.executemany( - "INSERT INTO turns VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", - [ - ( - key, - main_key, - is_sub, - facts.cwd, - t.ts, - day_of(t.ts), - t.model, - t.usd, - t.context, - t.prompt_idx, - t.usage.get("input_tokens", 0), - t.usage.get("cache_creation_input_tokens", 0), - t.usage.get("cache_read_input_tokens", 0), - t.usage.get("output_tokens", 0), - ) - for t in facts.rows.turns - ], - ) - conn.executemany( - "INSERT INTO prompts VALUES (?,?,?,?,?)", - [(key, p.idx, p.ts, p.prompt_id, p.text) for p in facts.rows.prompts], - ) - conn.executemany( - "INSERT INTO tool_use VALUES (?,?,?,?,?,?,?)", - [ - (key, t.prompt_idx, t.name, t.calls, t.result_tokens, t.usd, t.carry_usd) - for t in facts.rows.tools - ], - ) - conn.executemany( - "INSERT INTO agent_calls (key, tool_use_id, agent_id, prompt_idx, ts," - " description, subagent_type, model, result_ts, status)" - " VALUES (?,?,?,?,?,?,?,?,?,?)", - [ - ( - key, - a.tool_use_id, - a.agent_id, - a.prompt_idx, - a.ts, - a.description, - a.subagent_type, - a.model, - a.result_ts, - a.status, - ) - for a in facts.rows.agent_calls - ], - ) - conn.executemany( - "INSERT INTO activity VALUES (?,?,?)", - [(key, s, e) for s, e in facts.rows.intervals], - ) - conn.executemany( - "INSERT INTO lever_tables VALUES (?,?,?)", - [(key, ts, md) for ts, md in facts.rows.lever_tables], - ) - conn.executemany( - "INSERT INTO commits (key, prompt_idx, ts, end_ts, sha, repo)" - " VALUES (?,?,?,?,?,?)", - [ - (key, c.prompt_idx, c.ts, c.end_ts, c.sha, c.repo) - for c in facts.rows.commits - ], - ) - conn.executemany( - "INSERT INTO sent_messages VALUES (?,?,?)", - [(key, ts, to) for ts, to in facts.rows.messages], - ) - detail_store.write(conn, facts, (key, main_key, is_sub, facts.cwd)) - conn.execute("INSERT OR REPLACE INTO files VALUES (?,?,?,?)", (path, key, *stat)) diff --git a/python_pkg/token_dashboard/survival.py b/python_pkg/token_dashboard/survival.py deleted file mode 100644 index e8da8d0c..00000000 --- a/python_pkg/token_dashboard/survival.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Did a commit made in a session survive? Decided by git, never by a model. - -Rule (fixed here, shown in the dashboard caption): a commit is judged once it -is :data:`WINDOW_DAYS` old, at the last commit of its repository's ``HEAD`` -history that is no more than that old relative to it. - -* ``reverted`` -- a commit inside the window says ``This reverts commit ``. -* ``churned`` -- under :data:`KEEP_SHARE` of the lines it added (counted over - its :data:`MAX_FILES` largest files) still blame to it. -* ``survived`` -- otherwise, including a commit that added no lines. -* ``pending`` -- younger than the window. -* ``unknown`` -- the repo or hash is gone, or the commit is not on ``HEAD`` - (rebased, squashed or on another branch): excluded from the ratio. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -from typing import TYPE_CHECKING - -from python_pkg.token_dashboard import gitcmd - -if TYPE_CHECKING: - from collections.abc import Callable - import sqlite3 - -WINDOW_DAYS = 14 -WINDOW_S = WINDOW_DAYS * 86400 -KEEP_SHARE = 0.5 -MAX_FILES = 10 -_NUMSTAT_FIELDS = 3 -FINAL = frozenset({"survived", "churned", "reverted"}) - - -@dataclass(frozen=True) -class Verdict: - """The outcome for one commit.""" - - verdict: str - sha: str = "" - ts: float = 0.0 - added: int = 0 - kept: int = 0 - - -def judge(added: int, kept: int, *, reverted: bool) -> str: - """Apply the survival rule to measured numbers.""" - if reverted: - return "reverted" - if added and kept < KEEP_SHARE * added: - return "churned" - return "survived" - - -def _git(repo: str, *args: str) -> str | None: - """Run git in ``repo``; ``None`` on any failure.""" - return gitcmd.run(repo, *args) - - -def _added(repo: str, sha: str) -> list[tuple[int, str]]: - """``(added lines, path)`` for the commit's largest text files.""" - out = _git(repo, "show", "--numstat", "--format=", sha) or "" - rows = [] - for line in out.splitlines(): - parts = line.split("\t", 2) - if len(parts) == _NUMSTAT_FIELDS and parts[0].isdigit() and int(parts[0]): - rows.append((int(parts[0]), parts[2])) - return sorted(rows, reverse=True)[:MAX_FILES] - - -def _kept(repo: str, rev: str, sha: str, path: str) -> int: - """Lines of ``path`` at ``rev`` that still blame to ``sha``.""" - out = _git(repo, "blame", "--porcelain", rev, "--", path) or "" - prefix = sha + " " - return sum(1 for line in out.splitlines() if line.startswith(prefix)) - - -def evaluate(repo: str, sha: str, now: float) -> Verdict: - """Judge one commit in one repository.""" - if not Path(repo).is_dir(): - return Verdict("unknown") - full = ( - _git(repo, "rev-parse", "--verify", "-q", f"{sha}^{{commit}}") or "" - ).strip() - if not full: - return Verdict("unknown") - ts = float((_git(repo, "show", "-s", "--format=%ct", full) or "0").strip() or 0) - if now - ts < WINDOW_S: - return Verdict("pending", full, ts) - if _git(repo, "merge-base", "--is-ancestor", full, "HEAD") is None: - return Verdict("unknown", full, ts) - until = f"@{int(ts + WINDOW_S)}" - end = (_git(repo, "rev-list", "-1", f"--before={until}", "HEAD") or "").strip() - log = _git(repo, "log", "--format=%B", f"--since=@{int(ts)}", f"--until={until}") - reverted = f"This reverts commit {full}" in (log or "") - files = _added(repo, full) - added = sum(n for n, _ in files) - kept = sum(_kept(repo, end or full, full, path) for _, path in files) - return Verdict(judge(added, kept, reverted=reverted), full, ts, added, kept) - - -def update( - conn: sqlite3.Connection, now: float, deadline: float, clock: Callable[[], float] -) -> bool: - """Judge every session commit without a final verdict, within the deadline. - - Returns whether the backlog was cleared. A verdict once final is never - recomputed, so the backlog is only commits that turned 14 days old. - """ - rows = conn.execute( - "SELECT c.sha, c.repo, c.ts FROM session_commits c LEFT JOIN" - " commit_verdicts v ON v.sha = c.sha WHERE v.verdict IS NULL" - " OR v.verdict = 'pending'" - ).fetchall() - for sha, repo, ts in rows: - if now - ts < WINDOW_S: - continue # cannot be final yet; no git call needed - if clock() > deadline: - return False - found = evaluate(repo, sha, now) - conn.execute( - "INSERT OR REPLACE INTO commit_verdicts VALUES (?,?,?,?,?,?)", - (sha, repo, found.ts or ts, found.added, found.kept, found.verdict), - ) - return True diff --git a/python_pkg/token_dashboard/tests/__init__.py b/python_pkg/token_dashboard/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/python_pkg/token_dashboard/tests/conftest.py b/python_pkg/token_dashboard/tests/conftest.py deleted file mode 100644 index 9106fb53..00000000 --- a/python_pkg/token_dashboard/tests/conftest.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Shared dashboard test setup: source isolation and transcript builders. - -The autouse fixture keeps every test away from the real ``~/.claude`` state; -the builders below make synthetic transcript records. -""" - -from __future__ import annotations - -from datetime import UTC, datetime -import json -from typing import TYPE_CHECKING, Any - -import pytest - -from python_pkg.token_dashboard import ab, aux, context_files, walk - -if TYPE_CHECKING: - from pathlib import Path - - from python_pkg.token_dashboard.detail_rows import DetailRows - -BASE = 1_791_600_000.0 # 2026-10-09, a fixed anchor so day buckets are stable - - -@pytest.fixture(autouse=True) -def _isolated_sources(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Point every aux source at an empty tmp directory.""" - claude = tmp_path / "claude-home" - monkeypatch.setattr(aux, "AUTOPSY", claude / "autopsy.jsonl") - monkeypatch.setattr(aux, "PROMPT_HISTORY", claude / "history.jsonl") - monkeypatch.setattr(aux, "TIME_CAP", claude / "time-cap.jsonl") - monkeypatch.setattr(ab, "LOG", claude / "agent-ab.jsonl") - monkeypatch.setattr(ab, "BUILD_LOG", claude / "agent-ab-build.jsonl") - monkeypatch.setattr(context_files, "CLAUDE_DIR", claude) - monkeypatch.setattr(context_files, "SRC_DIR", claude / "src") - - -def iso(offset: float) -> str: - """ISO timestamp ``offset`` seconds after :data:`BASE`.""" - return datetime.fromtimestamp(BASE + offset, tz=UTC).isoformat() - - -def prompt(text: str, at: float, **extra: object) -> dict[str, Any]: - """A typed user prompt.""" - return { - "type": "user", - "timestamp": iso(at), - "cwd": "/repo", - "message": {"role": "user", "content": text}, - **extra, - } - - -def assistant( - mid: str, - at: float, - *, - output: int = 1000, - model: str = "claude-opus-5-5", - blocks: list[dict[str, Any]] | None = None, - context: int = 0, - **usage: object, -) -> dict[str, Any]: - """One assistant record; ``context`` becomes cache-read tokens. - - ``usage`` adds raw usage fields, e.g. a cache-write split. - """ - return { - "type": "assistant", - "timestamp": iso(at), - "message": { - "id": mid, - "model": model, - "usage": { - "output_tokens": output, - "cache_read_input_tokens": context, - **usage, - }, - "content": blocks or [{"type": "text", "text": "ok"}], - }, - } - - -def use(tid: str, name: str, **tool_input: object) -> dict[str, Any]: - """A ``tool_use`` content block.""" - return {"type": "tool_use", "id": tid, "name": name, "input": tool_input} - - -def result( - tid: str, at: float, text: str = "x" * 40, **extra: object -) -> dict[str, Any]: - """A user record carrying one ``tool_result``.""" - return { - "type": "user", - "timestamp": iso(at), - "message": { - "role": "user", - "content": [{"type": "tool_result", "tool_use_id": tid, "content": text}], - }, - **extra, - } - - -def write(path: Path, records: list[dict[str, Any]]) -> Path: - """Write records as JSONL, creating parent directories.""" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text("\n".join(json.dumps(r) for r in records) + "\n", encoding="utf-8") - return path - - -def usd_opus_output(tokens: int) -> float: - """List price of ``tokens`` Opus 5.5 output tokens (USD 20 / MTok).""" - return tokens * 20 / 1_000_000 - - -def attach(at: float, **attachment: object) -> dict[str, Any]: - """An ``attachment`` record.""" - return {"type": "attachment", "timestamp": iso(at), "attachment": attachment} - - -def walk_detail(tmp_path: Path, records: list[dict[str, Any]]) -> DetailRows: - """Walk one main-session file and return its detail rows.""" - path = write(tmp_path / "proj" / "s.jsonl", records) - return walk.walk_file(path, tmp_path).rows.detail diff --git a/python_pkg/token_dashboard/tests/test_ab.py b/python_pkg/token_dashboard/tests/test_ab.py deleted file mode 100644 index 3b9988f2..00000000 --- a/python_pkg/token_dashboard/tests/test_ab.py +++ /dev/null @@ -1,110 +0,0 @@ -from __future__ import annotations - -import json -from typing import TYPE_CHECKING - -import pytest - -from python_pkg.token_dashboard import ab, ingest, schema -from python_pkg.token_dashboard.tests.conftest import ( - assistant, - prompt, - result, - use, - write, -) - -if TYPE_CHECKING: - from pathlib import Path - - -def test_wilson_matches_reference_values() -> None: - # Reference: Wilson score interval, z = 1.96 (Newcombe 1998, table I). - low, high = ab.wilson(81, 263) - assert (round(low, 4), round(high, 4)) == (0.2553, 0.3662) - assert ab.wilson(0, 10)[0] == 0.0 - assert ab.wilson(0, 0) == (0.0, 1.0) - - -def test_newcombe_difference_contains_zero_for_equal_rates() -> None: - diff, low, high = ab.diff_ci(5, 50, 5, 50) - assert diff == 0.0 - assert low < 0 < high - - -def test_jaccard_and_correction_phrases() -> None: - assert ab.jaccard("Find RTDB rules for tutor", "find rtdb rules tutor log") >= 0.6 - assert ab.jaccard("Find RTDB rules", "Write the README") == 0.0 - assert ab.is_correction("No, that's wrong") - assert ab.is_correction("you missed the second file") - assert not ab.is_correction("nothing else, thanks") - - -def _arm(n: int, redo: int, corr: int = 0) -> ab.ArmCounts: - return ab.ArmCounts(n=n, redo=redo, correction=corr) - - -@pytest.mark.parametrize( - ("sonnet", "opus", "level"), - [ - (_arm(10, 9), _arm(10, 0), "grey"), - (_arm(40, 20), _arm(40, 2), "red"), - (_arm(40, 2, 20), _arm(40, 2, 1), "red"), - (_arm(40, 6), _arm(40, 4), "green"), - (_arm(40, 0), _arm(40, 20), "green"), # Sonnet better is not a drop - ], -) -def test_banner(sonnet: ab.ArmCounts, opus: ab.ArmCounts, level: str) -> None: - found = ab.banner(sonnet, opus) - assert found["level"] == level - assert f"n={min(sonnet.n, opus.n)}" in found["text"] or level == "red" - - -def test_rebuild_joins_assignments_to_transcripts(tmp_path: Path) -> None: - root = tmp_path / "projects" - first = use("t1", "Agent", description="Review the parser change") - again = use("t2", "Agent", description="Review the parser change again") - msg = use("t3", "SendMessage", to="abc", message="more") - write( - root / "p" / "s1.jsonl", - [ - prompt("go", 0), - assistant("m1", 1, blocks=[first]), - result("t1", 50, "done", toolUseResult={"agentId": "abc"}), - assistant("m2", 60, blocks=[msg]), - result("t3", 61), - prompt("no, that's wrong", 100), - assistant("m3", 101, blocks=[again]), - result( - "t2", 150, "err", toolUseResult={"agentId": "def", "status": "error"} - ), - ], - ) - log = tmp_path / "agent-ab.jsonl" - rows = [ - {"tool_use_id": "t1", "arm": "sonnet", "class": "review", "session_id": "s1"}, - {"tool_use_id": "t2", "arm": "opus", "class": "review", "session_id": "s1"}, - ] - log.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8") - ingest.run(root, tmp_path / "t.sqlite", tmp_path / "report", 30.0) - conn = schema.connect(tmp_path / "t.sqlite") - payload = ab.rebuild(conn, log) - stored = conn.execute( - "SELECT tool_use_id, arm, key, main_key, is_sub, cwd, redo, followup," - " error, correction, tag FROM ab_rows ORDER BY tool_use_id" - ).fetchall() - payload = ab.rebuild(conn, log) # a rebuild replaces, never appends - count = conn.execute("SELECT COUNT(*) FROM ab_rows").fetchone()[0] - conn.close() - assert stored == [ - ("t1", "sonnet", "s1", "s1", 0, "/repo", 1, 1, 0, 1, "untagged"), - ("t2", "opus", "s1", "s1", 0, "/repo", 0, 0, 1, 0, "untagged"), - ] - assert count == 2 - sonnet, opus = payload["arms"]["sonnet"], payload["arms"]["opus"] - assert (sonnet["n"], opus["n"]) == (1, 1) - assert sonnet["redo"]["k"] == 1 - assert sonnet["followup"]["k"] == 1 - assert sonnet["correction"]["k"] == 1 - assert opus["error"]["k"] == 1 - assert payload["banner"]["level"] == "grey" diff --git a/python_pkg/token_dashboard/tests/test_ab_build.py b/python_pkg/token_dashboard/tests/test_ab_build.py deleted file mode 100644 index cf8bec94..00000000 --- a/python_pkg/token_dashboard/tests/test_ab_build.py +++ /dev/null @@ -1,69 +0,0 @@ -"""The build-class A/B (agent-ab-build.jsonl) is summarised on its own.""" - -from __future__ import annotations - -import json -from typing import TYPE_CHECKING - -from python_pkg.token_dashboard import ab, ingest, schema -from python_pkg.token_dashboard.tests.conftest import assistant, prompt, result, use -from python_pkg.token_dashboard.tests.conftest import write as write_jsonl - -if TYPE_CHECKING: - from pathlib import Path - - -def _store(tmp_path: Path) -> Path: - root = tmp_path / "projects" - write_jsonl( - root / "p" / "s1.jsonl", - [ - prompt("go", 0), - assistant("m1", 1, blocks=[use("t1", "Agent", description="Build x")]), - result("t1", 50, "done", toolUseResult={"agentId": "abc"}), - assistant("m2", 60, blocks=[use("t2", "Agent", description="Review y")]), - result("t2", 70, "done", toolUseResult={"agentId": "def"}), - ], - ) - db = tmp_path / "t.sqlite" - ingest.run(root, db, tmp_path / "report", 30.0) - return db - - -def _log(path: Path, rows: list[dict[str, str]]) -> Path: - path.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8") - return path - - -def test_build_log_lands_in_its_own_meta_row(tmp_path: Path) -> None: - conn = schema.connect(_store(tmp_path)) - judgment = _log( - tmp_path / "agent-ab.jsonl", - [{"tool_use_id": "t2", "arm": "opus", "class": "review"}], - ) - build = _log( - tmp_path / "agent-ab-build.jsonl", - [{"tool_use_id": "t1", "arm": "sonnet", "class": "build"}], - ) - ab.rebuild(conn, judgment, build) - stored = { - k: json.loads(v) - for k, v in conn.execute("SELECT key, value FROM meta WHERE key LIKE 'ab%'") - } - rows = conn.execute("SELECT tool_use_id FROM ab_rows").fetchall() - conn.close() - assert stored["ab"]["arms"]["opus"]["n"] == 1 - assert stored["ab"]["arms"]["sonnet"]["n"] == 0 - assert stored["ab_build"]["arms"]["sonnet"]["n"] == 1 - assert stored["ab_build"]["arms"]["opus"]["n"] == 0 - assert stored["ab_build"]["banner"]["text"].startswith("A/B build agents") - assert stored["ab"]["banner"]["text"].startswith("A/B judgment agents") - assert rows == [("t2",)] - - -def test_missing_build_log_stores_an_empty_summary(tmp_path: Path) -> None: - conn = schema.connect(_store(tmp_path)) - ab.rebuild(conn, tmp_path / "none.jsonl", tmp_path / "none-build.jsonl") - raw = conn.execute("SELECT value FROM meta WHERE key = 'ab_build'").fetchone() - conn.close() - assert json.loads(raw[0])["arms"]["sonnet"]["n"] == 0 diff --git a/python_pkg/token_dashboard/tests/test_active.py b/python_pkg/token_dashboard/tests/test_active.py deleted file mode 100644 index ace8f013..00000000 --- a/python_pkg/token_dashboard/tests/test_active.py +++ /dev/null @@ -1,31 +0,0 @@ -from __future__ import annotations - -from python_pkg.token_dashboard import active - - -def test_gap_over_limit_contributes_zero_not_the_limit() -> None: - spans = active.intervals([0, 60, 120, 1000, 1030]) - assert spans == [(0, 120), (1000, 1030)] - assert active.measure(spans) == 150 - - -def test_union_bridges_short_gaps_between_files() -> None: - main = [(0.0, 100.0)] - sub = [(250.0, 400.0)] # 150 s after main went quiet: still active - far = [(2000.0, 2010.0)] - assert active.measure(main + sub + far) == 410 - - -def test_overlapping_intervals_count_once() -> None: - assert active.measure([(0.0, 100.0), (50.0, 150.0)]) == 150 - - -def test_clip_to_a_prompt_window() -> None: - spans = [(0.0, 100.0), (1000.0, 1100.0)] - assert active.measure(spans, 50.0, 1050.0) == 100 - assert active.measure([], 0.0, 10.0) == 0 - - -def test_empty_and_single_stamp() -> None: - assert active.intervals([]) == [] - assert active.intervals([5.0]) == [(5.0, 5.0)] diff --git a/python_pkg/token_dashboard/tests/test_aux.py b/python_pkg/token_dashboard/tests/test_aux.py deleted file mode 100644 index 0010d7be..00000000 --- a/python_pkg/token_dashboard/tests/test_aux.py +++ /dev/null @@ -1,173 +0,0 @@ -from __future__ import annotations - -from datetime import UTC, datetime -import json -from typing import TYPE_CHECKING - -import pytest - -from python_pkg.token_dashboard import aux, schema -from python_pkg.token_dashboard.tests.test_levers import TABLE - -if TYPE_CHECKING: - from collections.abc import Iterator, Sequence - from pathlib import Path - import sqlite3 - -NOON = datetime(2026, 10, 9, 12, tzinfo=UTC) -SECONDS = NOON.timestamp() - - -def _lines(path: Path, rows: Sequence[object], *junk: str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - body = [json.dumps(r) for r in rows] + list(junk) - path.write_text("\n".join(body) + "\n", encoding="utf-8") - - -@pytest.fixture -def conn(tmp_path: Path) -> Iterator[sqlite3.Connection]: - opened = schema.connect(tmp_path / "t.sqlite") - yield opened - opened.close() - - -def test_jsonl_skips_blank_malformed_and_non_object_lines(tmp_path: Path) -> None: - path = tmp_path / "x.jsonl" - _lines(path, [{"a": 1}, [1, 2], 3], "", "{broken") - assert list(aux._jsonl(path)) == [{"a": 1}] - assert list(aux._jsonl(tmp_path / "missing.jsonl")) == [] - - -def test_stamp_reads_iso_seconds_milliseconds_and_rejects_the_rest() -> None: - assert aux._stamp(NOON.isoformat()) == SECONDS - assert aux._stamp(SECONDS) == SECONDS - assert aux._stamp(int(SECONDS * 1000)) == pytest.approx(int(SECONDS)) - assert aux._stamp("yesterday") is None - assert aux._stamp(["x"]) is None - assert aux._stamp(None) is None - - -def test_snapshots_copy_window_and_metrics_and_skip_unreadable( - tmp_path: Path, conn: sqlite3.Connection -) -> None: - history = tmp_path / "report" / "history" - history.mkdir(parents=True) - snap = { - "generated": NOON.isoformat(), - "window": {"since": 1.0, "until": 2.0}, - "metrics": {"usd_total": 5}, - } - (history / "2026-10-09.json").write_text(json.dumps(snap)) - (history / "2026-10-10.json").write_text("{not json") - aux.snapshots(conn, tmp_path / "report") - rows = conn.execute("SELECT name, since, until, metrics FROM snapshots").fetchall() - assert rows == [("2026-10-09", 1.0, 2.0, '{"usd_total": 5}')] - - -def test_lever_rows_assign_each_table_to_the_nearest_snapshot( - tmp_path: Path, conn: sqlite3.Connection -) -> None: - report = tmp_path / "report" - (report / "history").mkdir(parents=True) - for day in ("2026-10-02", "2026-10-09"): - stamp = datetime.fromisoformat(f"{day}T12:00:00+00:00").isoformat() - (report / "history" / f"{day}.json").write_text( - json.dumps({"generated": stamp}) - ) - (report / "WEEKLY.md").write_text(TABLE, encoding="utf-8") - aux.snapshots(conn, report) - conn.execute("INSERT INTO lever_tables VALUES ('s', ?, ?)", (SECONDS, TABLE)) - aux.lever_rows(conn, report) - sources = {r[0] for r in conn.execute("SELECT source FROM levers")} - assert "snapshot 2026-10-09" in sources - - -def test_lever_rows_with_no_snapshots_store_nothing( - tmp_path: Path, conn: sqlite3.Connection -) -> None: - conn.execute("INSERT INTO lever_tables VALUES ('s', ?, ?)", (SECONDS, TABLE)) - aux.lever_rows(conn, tmp_path / "report") - assert conn.execute("SELECT COUNT(*) FROM levers").fetchone() == (0,) - - -def _autopsy_row( - sid: str, started: object, msgs: int, api: int | None -) -> dict[str, object]: - return { - "session_id": sid, - "meta": {"started_at": started, "cwd": "/r", "title": sid}, - "tokens": {"input": 100, "cache_write": 200, "cache_read": 300, "output": 400}, - "turns": {"api_turns": api}, - "counts": {"assistant_msgs": msgs}, - } - - -def test_autopsy_scales_by_api_turns_and_calibrates_against_transcripts( - conn: sqlite3.Connection, -) -> None: - rows = [ - _autopsy_row("both", NOON.isoformat(), 10, 5), - _autopsy_row("alone", SECONDS, 4, None), - _autopsy_row("nostart", None, 4, 2), - _autopsy_row("nomsgs", SECONDS, 0, 0), - ] - _lines(aux.AUTOPSY, rows) - conn.execute("INSERT INTO session_summary(key, usd_main) VALUES ('both', 3.0)") - aux.autopsy(conn) - got = dict(conn.execute("SELECT session_id, usd_est FROM aggregate_sessions")) - assert set(got) == {"both", "alone"} - # Same tokens: 5 API turns over 10 records scales the cost by one half. - assert got["both"] == pytest.approx(got["alone"] / 2) - factor = conn.execute("SELECT value FROM meta WHERE key='autopsy_calibration'") - assert float(factor.fetchone()[0]) == pytest.approx(3.0 / got["both"]) - - -def test_autopsy_without_overlap_keeps_factor_one(conn: sqlite3.Connection) -> None: - aux.autopsy(conn) - factor = conn.execute("SELECT value FROM meta WHERE key='autopsy_calibration'") - assert factor.fetchone() == ("1.0",) - - -def test_activity_logs_count_prompts_events_and_first_estimate( - tmp_path: Path, conn: sqlite3.Connection -) -> None: - report = tmp_path / "report" - _lines( - aux.PROMPT_HISTORY, - [{"timestamp": int(SECONDS * 1000)}, {"timestamp": SECONDS}, {"x": 1}], - ) - _lines(aux.TIME_CAP, [{"ts": SECONDS, "event": "kill", "command": "sleep 99"}]) - misses = report / "task-size" / "misses.jsonl" - _lines(misses, [{"ts": NOON.isoformat(), "kind": "under", "first_prompt": "p"}]) - verdicts = [ - {"status": "ok", "session": "a", "verdict": {"est_calls": 12}}, - {"status": "ok", "session": "a", "verdict": {"est_calls": 99}}, - {"status": "ok", "session": "b", "verdict": "none"}, - {"status": "error", "session": "c", "verdict": {"est_calls": 3}}, - ] - _lines(report / "task-size" / "verdicts.jsonl", verdicts) - aux.activity_logs(conn, report) - days = conn.execute("SELECT prompts FROM prompt_activity").fetchall() - events = conn.execute("SELECT source, kind, detail FROM events").fetchall() - estimates = conn.execute("SELECT session_id, est_calls FROM estimates").fetchall() - assert days == [(2,)] - assert events == [ - ("time-cap", "kill", "sleep 99"), - ("task-size-miss", "under", "p"), - ] - assert estimates == [("a", 12)] - - -def test_reloading_keeps_rows_a_trimmed_source_no_longer_has(tmp_path: Path) -> None: - conn = schema.connect(tmp_path / "t.sqlite") - _lines( - aux.PROMPT_HISTORY, - [{"timestamp": 1_700_000_000_000}, {"timestamp": 1_800_000_000}], - ) - aux.activity_logs(conn, tmp_path / "report") - aux.PROMPT_HISTORY.write_text("", encoding="utf-8") # rotated away - aux.activity_logs(conn, tmp_path / "report") - aux.activity_logs(conn, tmp_path / "report") - days = conn.execute("SELECT COUNT(*), SUM(prompts) FROM prompt_activity").fetchone() - conn.close() - assert days == (2, 2) diff --git a/python_pkg/token_dashboard/tests/test_check_window.py b/python_pkg/token_dashboard/tests/test_check_window.py deleted file mode 100644 index 29b8c17e..00000000 --- a/python_pkg/token_dashboard/tests/test_check_window.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -import json -from typing import TYPE_CHECKING - -import pytest - -from python_pkg.token_dashboard import check_window, schema - -if TYPE_CHECKING: - from collections.abc import Mapping - from pathlib import Path - -DAY = 86400.0 - - -def _store(tmp_path: Path, stamps: list[float]) -> Path: - db = tmp_path / "t.sqlite" - conn = schema.connect(db) - conn.executemany( - "INSERT INTO turns(key, ts, usd) VALUES ('k', ?, 1.0)", [(s,) for s in stamps] - ) - conn.commit() - conn.close() - return db - - -def _snapshot(report: Path, name: str, metrics: Mapping[str, float]) -> None: - folder = report / "history" - folder.mkdir(parents=True, exist_ok=True) - snap = {"window": {"since": 0, "until": 10 * DAY}, "metrics": metrics} - (folder / f"{name}.json").write_text(json.dumps(snap), encoding="utf-8") - - -def _run(tmp_path: Path, db: Path, *extra: str) -> int: - return check_window.main( - ["--db", str(db), "--report-dir", str(tmp_path / "report"), *extra] - ) - - -def test_matching_snapshot_exits_zero( - tmp_path: Path, capsys: pytest.CaptureFixture[str] -) -> None: - db = _store(tmp_path, [100.0, 100.0 + DAY]) - good = {"usd_total": 2.0, "usd_per_day": 2.0, "calls": 2} - _snapshot(tmp_path / "report", "2026-10-16", {**good, "calls": 99}) - _snapshot(tmp_path / "report", "2026-10-17", good) - assert _run(tmp_path, db, "--snapshot", "2026-10-17") == 0 - out = capsys.readouterr().out - assert out.count("OK") == 3 - assert "FAIL" not in out - - -def test_drift_beyond_one_percent_in_the_newest_snapshot_fails( - tmp_path: Path, capsys: pytest.CaptureFixture[str] -) -> None: - db = _store(tmp_path, [100.0, 100.0 + DAY]) - _snapshot(tmp_path / "report", "2026-10-16", {"usd_total": 2.0}) - bad = {"usd_total": 2.5, "usd_per_day": 2.0, "calls": 2} - _snapshot(tmp_path / "report", "2026-10-17", bad) - assert _run(tmp_path, db) == 1 - assert "usd_total: store 2.00 vs snapshot 2.50 (20.00%) FAIL" in ( - capsys.readouterr().out - ) - - -def test_empty_window_against_zero_metrics_is_a_match(tmp_path: Path) -> None: - db = _store(tmp_path, []) - zero = {"usd_total": 0, "usd_per_day": 0, "calls": 0} - _snapshot(tmp_path / "report", "2026-10-17", zero) - assert _run(tmp_path, db) == 0 - - -def test_missing_snapshot_or_store_exits_one( - tmp_path: Path, capsys: pytest.CaptureFixture[str] -) -> None: - db = _store(tmp_path, [1.0]) - assert _run(tmp_path, db) == 1 # no snapshot directory at all - _snapshot(tmp_path / "report", "2026-10-17", {"usd_total": 1}) - assert _run(tmp_path, db, "--snapshot", "1999-01-01") == 1 - assert _run(tmp_path, tmp_path / "gone.sqlite") == 1 - assert capsys.readouterr().err.count("missing snapshot or store") == 3 diff --git a/python_pkg/token_dashboard/tests/test_classify.py b/python_pkg/token_dashboard/tests/test_classify.py deleted file mode 100644 index 4030316d..00000000 --- a/python_pkg/token_dashboard/tests/test_classify.py +++ /dev/null @@ -1,124 +0,0 @@ -from __future__ import annotations - -import pytest - -from python_pkg.token_dashboard import classify - - -@pytest.mark.parametrize( - ("command", "family"), - [ - ("ls -la", "ls"), - ("git -C /repo status --short | head", "git status"), - ("cd /x && git log --oneline -3", "git log"), - ("~/.claude/scripts/capped.sh python3 -m pytest -q", "python3 -m pytest"), - ( - "CAP_MEM=4G CAP_CPU_PCT=20 ~/.claude/scripts/capped.sh pnpm -s lint", - "pnpm lint", - ), - ("timeout 40 nice -n 19 grep -r x .", "grep"), - ("python3 scripts/check.py --all", "python3 check.py"), - ("bash ~/src/utils/scripts/gate_stamp.sh step x", "bash gate_stamp.sh"), - ("/usr/bin/sqlite3 db 'SELECT 1'", "sqlite3"), - ("cd /x", "cd"), - ("python3 -m", "python3 -m"), - ("python3 <<'EOF'\nprint(1)\nEOF", "python3"), - ("bash -c 'x' > /dev/null", "bash -c"), - ("git", "git"), - ("FOO=1", "(empty)"), - ("", "(empty)"), - ], -) -def test_bash_family(command: str, family: str) -> None: - assert classify.bash_family(command) == family - - -@pytest.mark.parametrize( - ("name", "tool_input", "family", "target"), - [ - ("Bash", {"command": "git status\n"}, "git status", "git status"), - ("Bash", {}, "(empty)", ""), - ( - "mcp__claude-in-chrome__navigate", - {"url": "x"}, - "mcp:claude-in-chrome", - "navigate", - ), - ("Skill", {"skill": "dataviz"}, "skill", "dataviz"), - ( - "Read", - {"file_path": "/h/.claude/rules/a.md"}, - "Read:rules", - "/h/.claude/rules/a.md", - ), - ( - "Read", - {"file_path": "/h/.claude/projects/p/memory/m.md"}, - "Read:memory", - "/h/.claude/projects/p/memory/m.md", - ), - ("Read", {"file_path": "/r/CLAUDE.md"}, "Read:claude-md", "/r/CLAUDE.md"), - ( - "Edit", - {"file_path": "/h/.claude/skills/x/SKILL.md"}, - "Edit:skill", - "/h/.claude/skills/x/SKILL.md", - ), - ( - "Write", - {"file_path": "/h/.claude/settings.json"}, - "Write:claude", - "/h/.claude/settings.json", - ), - ("Read", {"file_path": "/r/shot.PNG"}, "Read:image", "/r/shot.PNG"), - ("Read", {"file_path": "/r/a.py"}, "Read:file", "/r/a.py"), - ("Read", {}, "Read:file", ""), - ("Grep", {"pattern": "def x", "path": "/r"}, "Grep", "def x"), - ( - "WebFetch", - {"url": "https://docs.example.com/a?b"}, - "WebFetch", - "docs.example.com", - ), - ("WebSearch", {"query": "sqlite window"}, "WebSearch", "sqlite window"), - ("Agent", {"subagent_type": "Explore"}, "Agent", "Explore"), - ("Agent", {"isolation": "worktree"}, "Agent", "general-purpose [worktree]"), - ("TaskStop", {"id": 1}, "TaskStop", ""), - ], -) -def test_family_and_target( - name: str, tool_input: dict[str, object], family: str, target: str -) -> None: - assert classify.family(name, tool_input) == family - assert classify.target(name, tool_input) == target - - -def test_target_is_capped() -> None: - long = {"command": "echo " + "x" * 1000} - assert len(classify.target("Bash", long)) == classify.TARGET_CHARS - - -def test_signature_ignores_key_order_but_not_values() -> None: - one = classify.signature("Bash", {"command": "ls", "timeout": 5}) - same = classify.signature("Bash", {"timeout": 5, "command": "ls"}) - other = classify.signature("Bash", {"command": "ls -a", "timeout": 5}) - assert one == same - assert one != other - assert len(one) == classify.SIG_CHARS - - -@pytest.mark.parametrize( - ("text", "kind"), - [ - ("Exit code 1\nboom", "exit_code"), - ("File has not been read yet", "tool_error"), - ("The user doesn't want to proceed with this tool use.", "rejected"), - ("Permission for this action was denied by the classifier", "denied"), - ("PreToolUse:Bash hook error: blocked by policy", "hook"), - ("TIME CAP: this command was killed at 60s", "time_cap"), - ("[Request interrupted by user for tool use]", "interrupted"), - ("something else", "error"), - ], -) -def test_error_kind(text: str, kind: str) -> None: - assert classify.error_kind(text) == kind diff --git a/python_pkg/token_dashboard/tests/test_commits.py b/python_pkg/token_dashboard/tests/test_commits.py deleted file mode 100644 index 5ae5b2e3..00000000 --- a/python_pkg/token_dashboard/tests/test_commits.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from python_pkg.token_dashboard import commits, schema -from python_pkg.token_dashboard.tests.test_efficiency import MONDAY, _commit, _git - -if TYPE_CHECKING: - from pathlib import Path - -TRAILER = "\n\nCo-Authored-By: Claude Opus 5.5 " - - -def test_git_log_attributes_trailer_commits_to_the_commit_command( - tmp_path: Path, -) -> None: - repo = tmp_path / "r" - repo.mkdir() - _git(repo, "init", "-q", when=MONDAY) - (repo / "a").write_text("1") - _git(repo, "add", "a", when=MONDAY) - _git(repo, "commit", "-qm", "ours" + TRAILER, when=MONDAY + 30) - ours = _git(repo, "rev-parse", "HEAD", when=MONDAY) - _commit(repo, "b", "2", MONDAY + 40) # no trailer: a human commit - _git( - repo, - "commit", - "-q", - "--allow-empty", - "-m", - "late" + TRAILER, - when=MONDAY + 9000, - ) - conn = schema.connect(tmp_path / "t.sqlite") - rows = [ - ("s1", 0, MONDAY, MONDAY + 60, "", str(repo)), - ("s2", 3, MONDAY - 5000, MONDAY - 4000, "", str(repo / "sub")), - ("s3", 1, MONDAY, MONDAY + 1, "", str(tmp_path / "missing")), - ] - conn.executemany("INSERT INTO commits VALUES (?,?,?,?,?,?)", rows) - commits.discover(conn) - found = conn.execute("SELECT sha, key, prompt_idx FROM session_commits").fetchall() - conn.close() - assert found == [(ours, "s1", 0)] - - -def test_a_printed_hash_is_attributed_even_without_trailer(tmp_path: Path) -> None: - repo = tmp_path / "r" - repo.mkdir() - _git(repo, "init", "-q", when=MONDAY) - sha = _commit(repo, "a", "1", MONDAY + 5) - conn = schema.connect(tmp_path / "t.sqlite") - conn.execute( - "INSERT INTO commits VALUES (?,?,?,?,?,?)", - ("s1", 2, MONDAY, MONDAY + 9, sha[:7], str(repo)), - ) - commits.discover(conn) - commits.discover(conn) # idempotent - found = conn.execute("SELECT sha, key FROM session_commits").fetchall() - conn.close() - assert found == [(sha, "s1")] diff --git a/python_pkg/token_dashboard/tests/test_context_files.py b/python_pkg/token_dashboard/tests/test_context_files.py deleted file mode 100644 index 81e1ddcc..00000000 --- a/python_pkg/token_dashboard/tests/test_context_files.py +++ /dev/null @@ -1,103 +0,0 @@ -from __future__ import annotations - -import json -import os -import subprocess -from typing import TYPE_CHECKING - -from python_pkg.token_dashboard import context_files, gitcmd, schema - -if TYPE_CHECKING: - from pathlib import Path - - -def _tree(tmp_path: Path) -> tuple[Path, Path]: - claude, src = tmp_path / ".claude", tmp_path / "src" - for rel in ( - "CLAUDE.md", - "rules/a.instructions.md", - "rules/deep/b.md", - "memories/mistakes.md", - "skills/x/SKILL.md", - "agents/reviewer.md", - "projects/-home-kuhy/memory/MEMORY.md", - "backups/rules-ondemand/not-loaded.md", - ): - (claude / rel).parent.mkdir(parents=True, exist_ok=True) - (claude / rel).write_text("# x\n" * 10, encoding="utf-8") - (src / "repo").mkdir(parents=True) - (src / "repo" / "CLAUDE.md").write_text("## Commands\n", encoding="utf-8") - (src / "repo" / "AGENTS.md").write_text("agents\n", encoding="utf-8") - return claude, src - - -def test_enumerates_every_loaded_file_and_nothing_on_demand(tmp_path: Path) -> None: - claude, src = _tree(tmp_path) - found = { - f.path.replace(str(tmp_path), ""): f for f in context_files.find(claude, src) - } - assert set(found) == { - "/.claude/CLAUDE.md", - "/.claude/rules/a.instructions.md", - "/.claude/rules/deep/b.md", - "/.claude/memories/mistakes.md", - "/.claude/skills/x/SKILL.md", - "/.claude/agents/reviewer.md", - "/.claude/projects/-home-kuhy/memory/MEMORY.md", - "/src/repo/CLAUDE.md", - "/src/repo/AGENTS.md", - } - assert found["/.claude/rules/deep/b.md"].standing == "always" - assert found["/src/repo/CLAUDE.md"].standing == "in repo" - assert found["/.claude/skills/x/SKILL.md"].standing == "description" - assert found["/.claude/projects/-home-kuhy/memory/MEMORY.md"].limit == 24_400 - - -def _git(repo: Path, *args: str, when: int) -> None: - env = {**os.environ, "GIT_AUTHOR_DATE": f"@{when} +0000"} - env |= {"GIT_COMMITTER_DATE": f"@{when} +0000", "HOME": str(repo)} - env |= {"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t"} - env |= {"GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"} - subprocess.run( - [gitcmd.GIT, "-C", str(repo), *args], env=env, check=True, capture_output=True - ) - - -def test_size_history_from_git_and_prune_sizes(tmp_path: Path) -> None: - claude, src = _tree(tmp_path) - _git(claude, "init", "-q", when=1000) - _git(claude, "add", "CLAUDE.md", when=1000) - _git(claude, "commit", "-qm", "a", when=1000) - (claude / "CLAUDE.md").write_text("longer\n" * 50, encoding="utf-8") - _git(claude, "commit", "-qam", "b", when=2000) - prune = claude / "token-report" / "prune" / "2026-10-12" - prune.mkdir(parents=True) - (prune / "sizes.json").write_text(json.dumps({"file_bytes": {"CLAUDE.md": 77}})) - conn = schema.connect(tmp_path / "t.sqlite") - context_files.refresh(conn, claude, src) - rows = conn.execute( - "SELECT ts, bytes, source FROM context_sizes WHERE path LIKE '%/CLAUDE.md'" - " AND path NOT LIKE '%/src/%' ORDER BY ts" - ).fetchall() - listed = conn.execute("SELECT COUNT(*) FROM context_files").fetchone()[0] - context_files.refresh(conn, claude, src) # incremental: no duplicates - again = conn.execute("SELECT COUNT(*) FROM context_sizes").fetchone()[0] - conn.close() - assert [(r[1], r[2]) for r in rows[:2]] == [(40, "git"), (350, "git")] - assert (77, "prune 2026-10-12") in [(r[1], r[2]) for r in rows] - assert listed == 9 - assert again == len(rows) - - -def test_price_mix_weights_cache_read_price_by_calls(tmp_path: Path) -> None: - conn = schema.connect(tmp_path / "t.sqlite") - rows = [ - ("claude-opus-5-5", 100.0), - ("claude-opus-5-5", 101.0), - ("claude-haiku-4-5", 102.0), - ("claude-haiku-4-5", 1.0), - ] - conn.executemany("INSERT INTO turns (model, ts) VALUES (?, ?)", rows) - price = context_files.price_mix(conn, now=200.0, days=150 / 86400) - conn.close() - assert round(price, 4) == round((0.2 * 2 + 0.1) / 3, 4) diff --git a/python_pkg/token_dashboard/tests/test_detail.py b/python_pkg/token_dashboard/tests/test_detail.py deleted file mode 100644 index b1c6c208..00000000 --- a/python_pkg/token_dashboard/tests/test_detail.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Per-call and per-turn detail rows the walker keeps.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -import pytest - -from python_pkg.token_dashboard import detail -from python_pkg.token_dashboard.tests.conftest import ( - assistant, - iso, - prompt, - result, - use, - walk_detail, -) - -if TYPE_CHECKING: - from pathlib import Path - -ONE_HOUR_WRITE = { - "cache_creation_input_tokens": 1000, - "cache_creation": {"ephemeral_1h_input_tokens": 1000}, -} - - -def test_result_creation_and_carry_are_priced_by_hand(tmp_path: Path) -> None: - rows = walk_detail( - tmp_path, - [ - prompt("go", 0), - assistant("m1", 2, blocks=[use("t1", "Bash", command="git status")]), - result("t1", 5, "x" * 398), # json.dumps -> 400 chars -> 100 tokens - assistant("m2", 9, **ONE_HOUR_WRITE), - assistant("m3", 12, context=2000), - ], - ) - (call,) = rows.calls - assert (call.family, call.target, call.batch) == ("git status", "git status", 1) - assert call.result_tokens == 100 - assert call.end_ts - call.where.ts == pytest.approx(3) - # Written once by m2 at the 1h rate (2x input $4/MTok) ... - assert call.cost.create_usd == pytest.approx(100 * 2.0 * 4 / 1e6) - # ... then cache-read by m2 and m3 at $0.20/MTok. - assert call.cost.carry_usd == pytest.approx(100 * 2 * 0.2 / 1e6) - assert call.cost.carry_calls == 2 - assert call.where.model == "claude-opus-5-5" - - -def test_compaction_stops_the_carry(tmp_path: Path) -> None: - boundary = { - "type": "system", - "subtype": "compact_boundary", - "timestamp": iso(4), - "compactMetadata": { - "trigger": "auto", - "preTokens": 100000, - "postTokens": 9000, - "durationMs": 5000, - }, - } - read = use("t1", "Read", file_path="/a") - rows = walk_detail( - tmp_path, - [ - prompt("go", 0), - assistant("m1", 1, blocks=[read], context=100000), - result("t1", 2, "x" * 398), - assistant("m2", 3, context=100000), - boundary, - assistant("m3", 5, context=9000), - ], - ) - assert rows.calls[0].cost.carry_calls == 1 - (mark,) = [m for m in rows.marks if m.kind == "compaction"] - assert (mark.name, mark.tokens, mark.extra) == ("auto", 100000, 9000) - assert mark.duration_ms == 5000 - assert mark.cost.carry_usd == 0 # the summary call is not context - - -def test_batch_errors_images_and_signatures(tmp_path: Path) -> None: - image = {"type": "image", "source": {"type": "base64", "data": "A" * 9000}} - blocks = [use("a", "Bash", command="ls"), use("b", "mcp__chrome__shot")] - failed = { - "type": "tool_result", - "tool_use_id": "a", - "content": "Exit code 2", - "is_error": True, - } - shot = { - "type": "tool_result", - "tool_use_id": "b", - "content": [image, {"type": "text", "text": "ok"}], - } - rows = walk_detail( - tmp_path, - [ - prompt("go", 0), - assistant("m1", 1, blocks=blocks[:1]), - assistant("m1", 1, blocks=blocks[1:]), - {"type": "user", "timestamp": iso(2), "message": {"content": [failed]}}, - {"type": "user", "timestamp": iso(3), "message": {"content": [shot]}}, - assistant("m2", 4), - ], - ) - by_name = {c.name: c for c in rows.calls} - assert {c.batch for c in rows.calls} == {2} - assert by_name["Bash"].error_kind == "exit_code" - found = by_name["mcp__chrome__shot"] - assert (found.family, found.target, found.images) == ("mcp:chrome", "shot", 1) - assert found.error_kind == "" - assert found.result_tokens < 3000 # an image is priced by area, not base64 - assert by_name["Bash"].sig != found.sig - - -def test_turn_detail_latency_cache_split_and_output_split(tmp_path: Path) -> None: - text = {"type": "text", "text": "y" * 400} # ~100 tokens of reply - thinking = {"type": "thinking", "thinking": ""} - rows = walk_detail( - tmp_path, - [ - prompt("go", 0), - assistant("m1", 7, **ONE_HOUR_WRITE, blocks=[thinking]), - assistant("m1", 8, blocks=[text]), - assistant("m2", 9, output=10, blocks=[text]), - assistant("m3", 10, output=50, blocks=[{"type": "server_tool_use"}]), - assistant("m4", 11, output=40, blocks=[thinking]), - ], - ) - first, second, third, fourth = rows.turns - assert first.latency_s == pytest.approx(7) - assert (first.cw_1h, first.cw_5m) == (1000, 0) - assert (first.out_text, first.out_tool, first.out_think) == (100, 0, 900) - # More visible text than billed output: scaled down, no thinking left. - assert (second.out_text, second.out_think) == (10, 0) - assert second.latency_s == pytest.approx(1) - # Nothing visible: all of it is text, or thinking when a block says so. - assert (third.out_text, third.out_think) == (50, 0) - assert (fourth.out_text, fourth.out_think) == (0, 40) - - -def test_text_and_tool_output_without_thinking_split_by_length( - tmp_path: Path, -) -> None: - blocks = [{"type": "text", "text": "y" * 400}, use("t", "Bash", command="z" * 385)] - rows = walk_detail( - tmp_path, [prompt("go", 0), assistant("m1", 1, output=1000, blocks=blocks)] - ) - (turn,) = rows.turns - assert (turn.out_text, turn.out_tool, turn.out_think) == (500, 500, 0) - assert turn.tool_calls == 1 - - -def test_write_multiplier_blends_the_split() -> None: - assert detail.write_multiplier({}) == pytest.approx(1.25) - half = {"cache_creation_input_tokens": 100, "cache_creation_1h": 50} - assert detail.write_multiplier(half) == pytest.approx((50 * 2 + 50 * 1.25) / 100) - - -def test_result_for_an_unknown_call_is_ignored() -> None: - found = detail.Detail() - found.result({"tool_use_id": "nope", "content": "x"}, 1.0) - assert found.rows.calls == [] diff --git a/python_pkg/token_dashboard/tests/test_detail_marks.py b/python_pkg/token_dashboard/tests/test_detail_marks.py deleted file mode 100644 index 6c1c59be..00000000 --- a/python_pkg/token_dashboard/tests/test_detail_marks.py +++ /dev/null @@ -1,198 +0,0 @@ -"""Context marks: hooks, injected attachments, API errors, interruptions.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -import pytest - -from python_pkg.token_dashboard.tests.conftest import ( - BASE, - assistant, - attach, - iso, - prompt, - result, - use, - walk_detail, -) - -if TYPE_CHECKING: - from pathlib import Path - -CTX = "cd chain wrapped in a subshell" - - -def _hook_records() -> list[dict[str, object]]: - run = {"type": "hook_success", "hookName": "PreToolUse:Bash"} - run |= {"hookEvent": "PreToolUse", "toolUseID": "t1", "content": ""} - stop = {"blockingError": "no status tag", "command": "python3 /h/tag_stop.py"} - return [ - attach( - 0, - type="hook_success", - hookName="SessionStart:startup", - hookEvent="SessionStart", - toolUseID="s", - content="x" * 400, - stdout="x" * 400, - command="bash /h/.claude/hooks/start.sh", - durationMs=30, - ), - prompt("go", 1), - assistant("m1", 2, blocks=[use("t1", "Bash", command="cd /x && ls")]), - attach( - 3, - type="hook_additional_context", - hookName="PreToolUse:Bash", - hookEvent="PreToolUse", - toolUseID="t1", - content=[CTX], - ), - attach( - 3, - **run, - stdout='{"additionalContext":"' + CTX + '"}', - command="bash /h/.claude/hooks/cd_chain.sh", - durationMs=15, - ), - attach(3, **run, stdout="{}", command="bash /h/other.sh", durationMs="7"), - result("t1", 4), - assistant("m2", 5), - attach( - 6, - type="hook_blocking_error", - hookName="Stop", - toolUseID="z", - blockingError=stop, - ), - assistant("m3", 7, output=500), - assistant("m4", 8, output=500), - prompt("next", 20), - assistant("m5", 21), - attach(22, type="hook_system_message", hookName="UserPromptSubmit"), - attach( - 22, type="hook_non_blocking_error", hookName="PostToolUse", content="oops" - ), - attach(22, type="nested_memory", path="/r/CLAUDE.md", content={"c": "z"}), - attach(22, type="prompt_snapshot", systemPrompt=["huge"]), - ] - - -def test_hooks_are_marked_matched_and_their_retry_priced(tmp_path: Path) -> None: - rows = walk_detail(tmp_path, _hook_records()) - marks = {(m.kind, m.detail): m for m in rows.marks} - start = marks[("hook_run", "start.sh")] - assert (start.tokens, start.duration_ms, start.where.prompt_idx) == (100, 30, -1) - assert start.cost.carry_usd > 0 - assert start.where.model == "claude-opus-5-5" # filled from the first turn - assert marks[("hook_context", "cd_chain.sh")].tokens == len(CTX) // 4 - assert marks[("hook_run", "other.sh")].duration_ms == 7 - assert marks[("hook_run", "cd_chain.sh")].tokens == 0 # JSON is not injected - bounce = marks[("hook_block", "tag_stop.py")] - assert bounce.name == "Stop" - assert bounce.retry_usd == pytest.approx(2 * 500 * 20 / 1e6) # m3 + m4 only - assert ("hook_msg", "") in marks - assert marks[("hook_error", "")].tokens == 1 - assert marks[("attach", "/r/CLAUDE.md")].name == "nested_memory" - assert not [m for m in rows.marks if m.name == "prompt_snapshot"] - - -def test_pretool_block_retry_is_only_the_next_call(tmp_path: Path) -> None: - rows = walk_detail( - tmp_path, - [ - prompt("go", 0), - assistant("m1", 1), - attach( - 2, - type="hook_blocking_error", - hookName="PreToolUse:Bash", - toolUseID="q", - blockingError="denied", - ), - assistant("m2", 3, output=100), - assistant("m3", 4, output=100), - ], - ) - (block,) = rows.marks - assert (block.detail, block.tokens) == ("", 1) - assert block.retry_usd == pytest.approx(100 * 20 / 1e6) - - -def test_api_errors_interruptions_and_pasted_images(tmp_path: Path) -> None: - pasted = {"type": "image", "source": {"type": "base64", "data": "A"}} - limit = {"isApiErrorMessage": True, "error": "rate_limit"} - limit |= {"apiError": "usage_limit_reached"} - text = [{"type": "text", "text": "Overloaded"}] - rows = walk_detail( - tmp_path, - [ - { - "type": "user", - "timestamp": iso(0), - "message": {"content": [{"type": "text", "text": "look"}, pasted]}, - }, - {"type": "assistant", "timestamp": iso(1), **limit, "message": {}}, - { - "type": "assistant", - "timestamp": iso(2), - "isApiErrorMessage": True, - "message": {"content": text}, - }, - { - "type": "user", - "timestamp": iso(3), - "message": { - "content": [ - {"type": "text", "text": "[Request interrupted by user]"} - ] - }, - }, - assistant("m1", 4), - ], - ) - kinds = sorted((m.kind, m.name, m.detail) for m in rows.marks) - assert kinds == [ - ("api_error", "rate_limit", "usage_limit_reached"), - ("api_error", "unknown", "Overloaded"), - ("image", "user", ""), - ("interrupt", "user", ""), - ] - assert rows.marks[0].where.ts == pytest.approx(BASE) - assert rows.marks[0].tokens > 0 - - -def test_repeats_merge_and_odd_records_are_tolerated(tmp_path: Path) -> None: - reminder = {"type": "total_tokens_reminder", "left": 5} - odd = {"hookName": "PostToolUse", "toolUseID": "u"} - rows = walk_detail( - tmp_path, - [ - prompt("go", 0), - attach(1, **reminder), - attach(2, **reminder), - attach( - 2, - type="hook_success", - **odd, - content="", - stdout="", - command="", - durationMs="n/a", - ), - attach(2, type="hook_additional_context", **odd, content=""), - attach(2, type="hook_additional_context", **odd, content="unmatched"), - attach(2, type="hook_cancelled", **odd), - {"type": "system", "subtype": "turn_duration", "timestamp": iso(2)}, - assistant("m1", 3, blocks=[{"type": "server_tool_use", "id": "w"}]), - ], - ) - (merged,) = [m for m in rows.marks if m.name == "total_tokens_reminder"] - assert merged.count == 2 - assert merged.where.ts == pytest.approx(BASE + 1) - assert merged.cost.carry_calls == 2 - run = next(m for m in rows.marks if m.kind == "hook_run") - assert (run.duration_ms, run.detail) == (0, "") - contexts = [m for m in rows.marks if m.kind == "hook_context"] - assert {m.detail for m in contexts} == {""} diff --git a/python_pkg/token_dashboard/tests/test_detail_store.py b/python_pkg/token_dashboard/tests/test_detail_store.py deleted file mode 100644 index d4e99631..00000000 --- a/python_pkg/token_dashboard/tests/test_detail_store.py +++ /dev/null @@ -1,79 +0,0 @@ -"""The detail tables are written per file, replaced on re-walk, filterable.""" - -from __future__ import annotations - -from contextlib import closing -import sqlite3 -from typing import TYPE_CHECKING - -from python_pkg.token_dashboard import ingest -from python_pkg.token_dashboard.tests.conftest import ( - assistant, - iso, - prompt, - result, - use, - write, -) - -if TYPE_CHECKING: - from pathlib import Path - -COUNTS = ( - "SELECT (SELECT COUNT(*) FROM calls), (SELECT COUNT(*) FROM marks)," - " (SELECT COUNT(*) FROM turn_detail)" -) - - -def _records(extra: int) -> list[dict[str, object]]: - hook = { - "type": "attachment", - "timestamp": iso(1), - "attachment": {"type": "hook_system_message", "hookName": "Stop"}, - } - return [ - prompt("go", 0), - hook, - assistant("m1", 2, blocks=[use("t1", "Bash", command="ls")]), - result("t1", 3), - *[assistant(f"x{i}", 4 + i) for i in range(extra)], - ] - - -def _ingest(tmp_path: Path) -> closing[sqlite3.Connection]: - ingest.run( - root=tmp_path / "projects", - db_path=tmp_path / "t.sqlite", - report_dir=tmp_path / "reports", - budget_s=30.0, - ) - return closing(sqlite3.connect(tmp_path / "t.sqlite")) - - -def test_rewalk_replaces_detail_rows(tmp_path: Path) -> None: - path = tmp_path / "projects" / "proj" / "s1.jsonl" - write(path, _records(1)) - with _ingest(tmp_path) as conn: - assert conn.execute(COUNTS).fetchone() == (1, 1, 2) - write(path, _records(2)) - with _ingest(tmp_path) as conn: - assert conn.execute(COUNTS).fetchone() == (1, 1, 3) - - -def test_detail_rows_carry_the_shared_filter_columns(tmp_path: Path) -> None: - write(tmp_path / "projects" / "proj" / "s1.jsonl", _records(1)) - with _ingest(tmp_path) as conn: - call = conn.execute( - "SELECT key, main_key, is_sub, cwd, model, day, family, target, batch," - " carry_calls FROM calls" - ).fetchone() - mark = conn.execute( - "SELECT key, main_key, is_sub, cwd, model, kind, name, count FROM marks" - ).fetchone() - turn = conn.execute( - "SELECT key, is_sub, cwd, model, latency_s FROM turn_detail ORDER BY ts" - ).fetchone() - assert call[:5] == ("s1", "s1", 0, "/repo", "claude-opus-5-5") - assert call[6:] == ("ls", "ls", 1, 1) - assert mark == ("s1", "s1", 0, "/repo", "claude-opus-5-5", "hook_msg", "Stop", 1) - assert turn == ("s1", 0, "/repo", "claude-opus-5-5", 1.0) diff --git a/python_pkg/token_dashboard/tests/test_efficiency.py b/python_pkg/token_dashboard/tests/test_efficiency.py deleted file mode 100644 index d0db538f..00000000 --- a/python_pkg/token_dashboard/tests/test_efficiency.py +++ /dev/null @@ -1,146 +0,0 @@ -from __future__ import annotations - -from datetime import UTC, datetime -import os -from pathlib import Path -import subprocess - -import pytest - -from python_pkg.token_dashboard import efficiency, gitcmd, survival - -DAY = 86400.0 -MONDAY = datetime(2026, 10, 5, 12, tzinfo=UTC).timestamp() - - -def _s(ts: float, usd: float, tag: str | None) -> efficiency.SessionCost: - return efficiency.SessionCost(key=f"k{ts}{tag}", first_ts=ts, usd=usd, tag=tag) - - -def test_cost_per_done_counts_failed_sessions_against_done_ones() -> None: - rows = [ - _s(MONDAY, 10.0, "DONE"), - _s(MONDAY + 1, 6.0, "QUESTION"), - _s(MONDAY + 2, 4.0, "DONE"), - _s(MONDAY + 3, 99.0, None), # pre-convention: excluded from the ratio - ] - (week,) = efficiency.done_weekly(rows) - assert week["done"] == 2 - assert week["other"] == 1 - assert week["untagged"] == 1 - assert week["usd_per_done"] == pytest.approx(10.0) - assert week["done_share"] == pytest.approx(2 / 3) - - -def test_week_with_no_done_has_no_ratio() -> None: - (week,) = efficiency.done_weekly([_s(MONDAY, 5.0, "RUNNING")]) - assert week["usd_per_done"] is None - - -def test_cost_per_surviving_commit_and_pending() -> None: - week_usd = {efficiency.week_of(MONDAY): 30.0} - verdicts = [ - (MONDAY, "survived"), - (MONDAY, "survived"), - (MONDAY, "churned"), - (MONDAY, "pending"), - ] - (week,) = efficiency.commits_weekly(verdicts, week_usd) - assert (week["survived"], week["lost"], week["pending"]) == (2, 1, 1) - assert week["usd_per_survivor"] == pytest.approx(15.0) - - -def test_cost_per_estimated_unit() -> None: - rows = [_s(MONDAY, 8.0, None), _s(MONDAY + 5, 2.0, "DONE")] - estimates = {rows[0].key: 30, rows[1].key: 10} - (week,) = efficiency.estimate_weekly(rows, estimates) - assert week["est_calls"] == 40 - assert week["usd_per_unit"] == pytest.approx(0.25) - - -@pytest.mark.parametrize( - ("added", "kept", "reverted", "expected"), - [ - (10, 10, False, "survived"), - (10, 5, False, "survived"), - (10, 4, False, "churned"), - (10, 10, True, "reverted"), - (0, 0, False, "survived"), - ], -) -def test_survival_rule(added: int, kept: int, *, reverted: bool, expected: str) -> None: - assert survival.judge(added, kept, reverted=reverted) == expected - - -def _git(repo: Path, *args: str, when: float) -> str: - stamp = f"@{int(when)} +0000" - env = {**os.environ, "GIT_AUTHOR_DATE": stamp, "GIT_COMMITTER_DATE": stamp} - env |= {"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "HOME": str(repo)} - env |= {"GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"} - done = subprocess.run( - [gitcmd.GIT, "-C", str(repo), *args], - env=env, - check=True, - capture_output=True, - text=True, - ) - return done.stdout.strip() - - -def _commit(repo: Path, name: str, text: str, when: float) -> str: - (repo / name).write_text(text, encoding="utf-8") - _git(repo, "add", name, when=when) - _git(repo, "commit", "-qm", name, when=when) - return _git(repo, "rev-parse", "HEAD", when=when) - - -def test_git_survival_against_a_real_repo(tmp_path: Path) -> None: - repo = tmp_path / "r" - repo.mkdir() - _git(repo, "init", "-q", when=MONDAY) - t0 = MONDAY - 40 * DAY - kept = _commit(repo, "a.txt", "1\n2\n3\n4\n", t0) - churn = _commit(repo, "b.txt", "1\n2\n3\n4\n", t0 + 1) - _commit(repo, "b.txt", "x\ny\nz\n4\n", t0 + 2 * DAY) - young = _commit(repo, "c.txt", "1\n", MONDAY) - now = MONDAY + 1 - assert survival.evaluate(str(repo), kept[:8], now).verdict == "survived" - churned = survival.evaluate(str(repo), churn, now) - assert (churned.verdict, churned.added, churned.kept) == ("churned", 4, 1) - assert survival.evaluate(str(repo), young, now).verdict == "pending" - assert survival.evaluate(str(repo), "deadbeef", now).verdict == "unknown" - assert survival.evaluate(str(tmp_path / "nope"), kept, now).verdict == "unknown" - - -def test_git_revert_within_window_is_reverted(tmp_path: Path) -> None: - repo = tmp_path / "r" - repo.mkdir() - _git(repo, "init", "-q", when=MONDAY) - t0 = MONDAY - 40 * DAY - _commit(repo, "a.txt", "base\n", t0 - 10) - bad = _commit(repo, "b.txt", "1\n", t0) - _git(repo, "revert", "--no-edit", bad, when=t0 + DAY) - assert survival.evaluate(str(repo), bad, MONDAY).verdict == "reverted" - - -def test_update_judges_old_commits_and_stops_at_the_deadline(tmp_path: Path) -> None: - from python_pkg.token_dashboard import schema - - repo = tmp_path / "r" - repo.mkdir() - _git(repo, "init", "-q", when=MONDAY) - old = _commit(repo, "a.txt", "1\n", MONDAY - 40 * DAY) - conn = schema.connect(tmp_path / "t.sqlite") - rows = [ - (old, str(repo), MONDAY - 40 * DAY, "s", 0), - ("f" * 40, "x", MONDAY, "s", 0), - ] - conn.executemany("INSERT INTO session_commits VALUES (?,?,?,?,?)", rows) - assert not survival.update(conn, MONDAY, deadline=0.0, clock=lambda: 1.0) - assert survival.update(conn, MONDAY, deadline=2.0, clock=lambda: 1.0) - efficiency.rebuild(conn, MONDAY) - verdicts = conn.execute("SELECT verdict FROM commit_verdicts").fetchall() - payload = conn.execute("SELECT value FROM meta WHERE key='efficiency'").fetchone() - conn.close() - assert verdicts == [("survived",)] - assert '"pending": 1' in payload[0] diff --git a/python_pkg/token_dashboard/tests/test_gaps_git.py b/python_pkg/token_dashboard/tests/test_gaps_git.py deleted file mode 100644 index 4b880918..00000000 --- a/python_pkg/token_dashboard/tests/test_gaps_git.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Git-backed edge cases: caching, binary files, commits outside HEAD.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from python_pkg.token_dashboard import commits, survival -from python_pkg.token_dashboard.tests.test_efficiency import DAY, MONDAY, _commit, _git - -if TYPE_CHECKING: - from pathlib import Path - - -def _repo(tmp_path: Path) -> Path: - repo = tmp_path / "r" - repo.mkdir() - _git(repo, "init", "-q", when=MONDAY) - return repo - - -def test_toplevel_is_resolved_once_per_hint(tmp_path: Path) -> None: - repo = _repo(tmp_path) - _commit(repo, "a", "1", MONDAY) - cache: dict[str, str | None] = {} - first = commits._toplevel(str(repo), cache) - assert first == str(repo.resolve()) - cache[str(repo)] = "cached" - assert commits._toplevel(str(repo), cache) == "cached" - assert commits._toplevel("", cache) is None - - -def test_binary_and_empty_files_do_not_count_as_added_lines(tmp_path: Path) -> None: - repo = _repo(tmp_path) - old = MONDAY - 40 * DAY - (repo / "blob.bin").write_bytes(b"\x00\x01\x02") - (repo / "empty.txt").write_text("", encoding="utf-8") - (repo / "t.txt").write_text("1\n2\n", encoding="utf-8") - _git(repo, "add", ".", when=old) - _git(repo, "commit", "-qm", "mixed", when=old) - sha = _git(repo, "rev-parse", "HEAD", when=old) - found = survival.evaluate(str(repo), sha, MONDAY) - assert (found.verdict, found.added, found.kept) == ("survived", 2, 2) - - -def test_a_commit_that_is_not_an_ancestor_of_head_is_unknown(tmp_path: Path) -> None: - repo = _repo(tmp_path) - old = MONDAY - 40 * DAY - _commit(repo, "a", "1", old) - trunk = _git(repo, "branch", "--show-current", when=old) - _git(repo, "checkout", "-qb", "side", when=old) - side = _commit(repo, "b", "2", old + 1) - _git(repo, "checkout", "-q", trunk, when=old) - found = survival.evaluate(str(repo), side, MONDAY) - assert (found.verdict, found.sha) == ("unknown", side) diff --git a/python_pkg/token_dashboard/tests/test_gaps_ingest.py b/python_pkg/token_dashboard/tests/test_gaps_ingest.py deleted file mode 100644 index 01427a6b..00000000 --- a/python_pkg/token_dashboard/tests/test_gaps_ingest.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Edge cases of the walker, the summaries and the aux readers.""" - -from __future__ import annotations - -import json -from typing import TYPE_CHECKING - -from python_pkg.token_dashboard import ab, context_files, finalize, ingest, schema, walk -from python_pkg.token_dashboard.tests.conftest import ( - assistant, - prompt, - result, - use, - write, -) - -if TYPE_CHECKING: - from pathlib import Path - - -def test_timestampless_orphans_and_garbage_lines_are_tolerated(tmp_path: Path) -> None: - path = tmp_path / "proj" / "s.jsonl" - path.parent.mkdir() - lines = ['{"type": "summary", "summary": "x"}', "{broken", ""] - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - facts = walk.walk_file(path, tmp_path) - assert facts.span == (0.0, 0.0) - assert facts.rows.turns == [] - assert facts.final_tag is None - - -def test_a_result_without_its_call_is_ignored(tmp_path: Path) -> None: - path = write(tmp_path / "proj" / "s.jsonl", [prompt("go", 0), result("nope", 1)]) - facts = walk.walk_file(path, tmp_path) - assert facts.rows.tools == [] - - -def test_a_nested_subagent_resolves_to_the_prompt_that_started_the_chain( - tmp_path: Path, -) -> None: - root = tmp_path / "projects" - outer = use("ta", "Agent", description="outer") - inner = use("tb", "Agent", description="inner") - write( - root / "p" / "s1.jsonl", - [ - assistant("m0", 0), # a turn before any prompt - prompt("go", 5), - assistant("m1", 10, blocks=[outer]), - result("ta", 11, "ok", toolUseResult={"agentId": "aaa"}), - ], - ) - subs = root / "p" / "s1" / "subagents" - write( - subs / "agent-aaa.jsonl", - [ - assistant("a1", 20, blocks=[inner]), - result("tb", 21, toolUseResult={"agentId": "bbb"}), - ], - ) - write(subs / "agent-bbb.jsonl", [assistant("b1", 30)]) - ingest.run(root, tmp_path / "t.sqlite", tmp_path / "report", 30.0) - conn = schema.connect(tmp_path / "t.sqlite") - links = conn.execute( - "SELECT sub_key, prompt_idx, depth FROM subagent_link ORDER BY depth" - ).fetchall() - first = conn.execute( - "SELECT idx, text FROM prompt_summary WHERE key='s1' ORDER BY idx" - ).fetchone() - conn.close() - assert links == [("s1/agent-aaa", 0, 1), ("s1/agent-bbb", 0, 2)] - assert first == (-1, "(before the first prompt)") - - -def test_refreshing_an_unknown_session_changes_nothing(tmp_path: Path) -> None: - conn = schema.connect(tmp_path / "t.sqlite") - finalize.refresh(conn, "ghost") - assert conn.execute("SELECT COUNT(*) FROM session_summary").fetchone() == (0,) - conn.close() - - -def test_unreadable_prune_sizes_are_skipped(tmp_path: Path) -> None: - claude, src = tmp_path / "claude", tmp_path / "src" - prune = claude / "token-report" / "prune" - for name, body in ( - ("a", "{broken"), - ("b", "[1]"), - ("c", '{"file_bytes": {"X.md": 9}}'), - ): - (prune / name).mkdir(parents=True) - (prune / name / "sizes.json").write_text(body, encoding="utf-8") - conn = schema.connect(tmp_path / "t.sqlite") - context_files.refresh(conn, claude, src) - sources = conn.execute("SELECT source FROM context_sizes").fetchall() - conn.close() - assert sources == [("prune c",)] - - -def test_unmatched_and_malformed_assignments_are_counted_unjoined( - tmp_path: Path, -) -> None: - log = tmp_path / "agent-ab.jsonl" - rows = [ - {"tool_use_id": "ghost", "arm": "sonnet"}, - {"tool_use_id": "t1", "arm": "haiku"}, - {"tool_use_id": "t2"}, - [1], - ] - log.write_text("\n".join(json.dumps(r) for r in rows) + "\n{broken\n") - conn = schema.connect(tmp_path / "t.sqlite") - payload = ab.rebuild(conn, log) - default = ab.rebuild(conn) # the isolated, absent default log - conn.close() - assert payload["unjoined"] == 2 - assert payload["arms"]["sonnet"]["n"] == 0 - assert default["unjoined"] == 0 diff --git a/python_pkg/token_dashboard/tests/test_gaps_units.py b/python_pkg/token_dashboard/tests/test_gaps_units.py deleted file mode 100644 index b20d4c55..00000000 --- a/python_pkg/token_dashboard/tests/test_gaps_units.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Edge cases of the small pure helpers: blocks, gitcmd, levers, schema, efficiency.""" - -from __future__ import annotations - -import json -import sqlite3 -import subprocess -from typing import TYPE_CHECKING - -import pytest - -from python_pkg.token_dashboard import blocks, efficiency, gitcmd, levers, schema - -if TYPE_CHECKING: - from pathlib import Path - - -def test_text_of_joins_text_blocks_and_ignores_the_rest() -> None: - content = [ - {"type": "text", "text": "one"}, - {"type": "image"}, - "stray", - {"type": "text", "text": "two"}, - ] - assert blocks.text_of(content) == "one\ntwo" - assert blocks.text_of("plain") == "plain" - assert blocks.text_of(None) == "" - assert blocks.text_of({"type": "text", "text": "dict"}) == "" - - -def test_compact_summaries_and_tool_results_are_not_prompts() -> None: - message = {"role": "user", "content": "summary of the chat"} - summary = {"type": "user", "isCompactSummary": True, "message": message} - assert blocks.prompt_text(summary) is None - visible = {"type": "user", "isVisibleInTranscriptOnly": True, "message": message} - assert blocks.prompt_text(visible) is None - tool = [{"type": "tool_result", "tool_use_id": "t", "content": "x"}] - reply = {"type": "user", "message": {"content": tool}} - assert blocks.prompt_text(reply) is None - - -def test_repo_hint_falls_back_to_cwd_and_resolves_relative_targets() -> None: - assert blocks.repo_hint("git commit -m x", None) == "" - assert blocks.repo_hint("git commit -m x", "/work/r") == "/work/r" - assert blocks.repo_hint("cd sub && git commit", "/work/r") == "/work/r/sub" - assert blocks.repo_hint("git -C /abs/repo commit", "/work/r") == "/abs/repo" - - -def test_lever_table_needs_the_heading_and_at_least_one_row() -> None: - heading = f"## {blocks.LEVER_HEADING}\n\n" - assert blocks.lever_table("nothing here") is None - assert blocks.lever_table(heading + "just prose") is None - assert ( - blocks.lever_table(heading + "| a | b |\n| c | d |") == "| a | b |\n| c | d |" - ) - - -@pytest.mark.parametrize( - "error", [OSError("no git"), subprocess.TimeoutExpired("git", 1)] -) -def test_git_run_never_raises( - error: Exception, monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - def boom(*_args: object, **_kwargs: object) -> None: - raise error - - monkeypatch.setattr("python_pkg.token_dashboard.gitcmd.subprocess.run", boom) - assert gitcmd.run(tmp_path, "status") is None - - -def test_unreadable_or_non_object_snapshots_are_skipped(tmp_path: Path) -> None: - history = tmp_path / "history" - history.mkdir() - (history / "2026-10-01.json").write_text("{broken") - (history / "2026-10-02.json").write_text(json.dumps([1, 2])) - (history / "2026-10-03.json").write_text(json.dumps({"metrics": {}})) - assert levers._snapshots(tmp_path) == [("2026-10-03", {"metrics": {}})] - - -def test_readonly_store_refuses_writes(tmp_path: Path) -> None: - db = tmp_path / "t.sqlite" - schema.connect(db).close() - conn = schema.connect(db, readonly=True) - try: - assert conn.execute("SELECT COUNT(*) FROM turns").fetchone() == (0,) - with pytest.raises(sqlite3.OperationalError, match="readonly"): - conn.execute("INSERT INTO meta VALUES ('k', 'v')") - finally: - conn.close() - - -def test_commits_with_an_unrecognised_verdict_count_as_unknown() -> None: - (week,) = efficiency.commits_weekly([(1.0, "mystery")], {}) - assert (week["unknown"], week["survived"], week["lost"]) == (1, 0, 0) diff --git a/python_pkg/token_dashboard/tests/test_ingest.py b/python_pkg/token_dashboard/tests/test_ingest.py deleted file mode 100644 index 4e4be8d4..00000000 --- a/python_pkg/token_dashboard/tests/test_ingest.py +++ /dev/null @@ -1,181 +0,0 @@ -from __future__ import annotations - -import json -import sqlite3 -from typing import TYPE_CHECKING - -import pytest - -from python_pkg.token_dashboard import ingest -from python_pkg.token_dashboard.tests.conftest import ( - assistant, - prompt, - result, - usd_opus_output, - use, - write, -) - -if TYPE_CHECKING: - from collections.abc import Callable, Iterator - from pathlib import Path - - -def _tree(root: Path) -> None: - agent = use("ta", "Agent", description="Find x", subagent_type="Explore") - write( - root / "proj" / "s1.jsonl", - [ - prompt("first", 0), - assistant("m1", 10, blocks=[agent]), - result("ta", 11, "launched", toolUseResult={"agentId": "abc"}), - assistant("m2", 20), - prompt("second", 1000), - assistant("m3", 1010, blocks=[{"type": "text", "text": "DONE: x"}]), - ], - ) - subdir = root / "proj" / "s1" / "subagents" - write(subdir / "agent-abc.jsonl", [prompt("task", 12), assistant("a1", 200)]) - (subdir / "agent-abc.meta.json").write_text( - json.dumps({"agentType": "Explore", "description": "Find x"}), - encoding="utf-8", - ) - # An agent that outlived /clear: nothing in s1 spawned it. - write(subdir / "agent-zzz.jsonl", [prompt("orphan", 30), assistant("z1", 40)]) - - -def _run(tmp_path: Path, budget: float = 30.0) -> ingest.Outcome: - return ingest.run( - root=tmp_path / "projects", - db_path=tmp_path / "t.sqlite", - report_dir=tmp_path / "report", - budget_s=budget, - ) - - -@pytest.fixture -def db(tmp_path: Path) -> Iterator[Callable[[], sqlite3.Connection]]: - opened: list[sqlite3.Connection] = [] - - def _open() -> sqlite3.Connection: - opened.append(sqlite3.connect(tmp_path / "t.sqlite")) - return opened[-1] - - yield _open - for conn in opened: - conn.close() - - -def test_session_cost_reconciles_to_prompts_plus_unattributed( - tmp_path: Path, db: Callable[[], sqlite3.Connection] -) -> None: - _tree(tmp_path / "projects") - outcome = _run(tmp_path) - assert outcome.complete - conn = db() - usd, active_s, wall_s, tag = conn.execute( - "SELECT usd, active_s, wall_s, final_tag FROM session_summary WHERE key='s1'" - ).fetchone() - assert usd == pytest.approx(usd_opus_output(1000) * 5) - prompts = conn.execute( - "SELECT idx, usd_own + usd_sub FROM prompt_summary WHERE key='s1' ORDER BY idx" - ).fetchall() - unattributed = conn.execute( - "SELECT SUM(usd) FROM subagent_link WHERE main_key='s1' AND prompt_idx IS NULL" - ).fetchone()[0] - assert sum(v for _, v in prompts) + unattributed == pytest.approx(usd) - # Prompt 0 carries its own two turns plus the Explore agent it spawned. - assert prompts[0][1] == pytest.approx(usd_opus_output(1000) * 3) - assert wall_s == 1010 - # Union of main + subagent activity: 0..200 bridged, then 1000..1010. - assert active_s == pytest.approx(210) - assert tag == "DONE" - - -def test_subagent_links_to_its_prompt_via_agent_id( - tmp_path: Path, db: Callable[[], sqlite3.Connection] -) -> None: - _tree(tmp_path / "projects") - _run(tmp_path) - rows = dict( - db().execute("SELECT sub_key, prompt_idx FROM subagent_link").fetchall() - ) - assert rows == {"s1/agent-abc": 0, "s1/agent-zzz": None} - - -def test_rerun_skips_unchanged_and_rows_outlive_deleted_files( - tmp_path: Path, db: Callable[[], sqlite3.Connection] -) -> None: - _tree(tmp_path / "projects") - _run(tmp_path) - assert _run(tmp_path).parsed == 0 - (tmp_path / "projects" / "proj" / "s1.jsonl").unlink() - _run(tmp_path) - count = db().execute("SELECT COUNT(*) FROM turns").fetchone()[0] - assert count == 5 - - -def test_changed_file_is_reparsed_without_duplicating_rows( - tmp_path: Path, db: Callable[[], sqlite3.Connection] -) -> None: - _tree(tmp_path / "projects") - _run(tmp_path) - path = tmp_path / "projects" / "proj" / "s1.jsonl" - with path.open("a", encoding="utf-8") as handle: - handle.write(json.dumps(assistant("m4", 1020)) + "\n") - assert _run(tmp_path).parsed == 1 - count = db().execute("SELECT COUNT(*) FROM turns").fetchone()[0] - assert count == 6 - - -def test_zero_budget_leaves_work_pending_and_resumes(tmp_path: Path) -> None: - _tree(tmp_path / "projects") - first = _run(tmp_path, budget=0.0) - assert not first.complete - assert first.parsed == 1 # always makes progress - rest = _run(tmp_path) - assert rest.complete - assert first.parsed + rest.parsed == 3 - - -def test_daily_rollup_matches_turns( - tmp_path: Path, db: Callable[[], sqlite3.Connection] -) -> None: - _tree(tmp_path / "projects") - _run(tmp_path) - conn = db() - total, calls, sub = conn.execute( - "SELECT SUM(usd), SUM(calls), SUM(usd_sub) FROM daily" - ).fetchone() - assert total == pytest.approx(usd_opus_output(1000) * 5) - assert calls == 5 - assert sub == pytest.approx(usd_opus_output(1000) * 2) - - -def test_cli_exit_codes(tmp_path: Path) -> None: - _tree(tmp_path / "projects") - args = [ - "--root", - str(tmp_path / "projects"), - "--db", - str(tmp_path / "t.sqlite"), - "--report-dir", - str(tmp_path / "report"), - ] - assert ingest.main([*args, "--budget", "0"]) == ingest.EXIT_PENDING - assert ingest.main(args) == 0 - - -def test_session_with_no_prompts_and_no_main_file( - tmp_path: Path, db: Callable[[], sqlite3.Connection] -) -> None: - sub = tmp_path / "projects" / "proj" / "gone" / "subagents" / "agent-q.jsonl" - write(sub, [assistant("q1", 5), assistant("q2", 6)]) - assert _run(tmp_path).complete - usd, prompts = ( - db() - .execute("SELECT usd, prompts FROM session_summary WHERE key='gone'") - .fetchone() - ) - assert usd == pytest.approx(usd_opus_output(1000) * 2) - assert prompts == 0 diff --git a/python_pkg/token_dashboard/tests/test_levers.py b/python_pkg/token_dashboard/tests/test_levers.py deleted file mode 100644 index d5822d45..00000000 --- a/python_pkg/token_dashboard/tests/test_levers.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import annotations - -import json -from typing import TYPE_CHECKING - -from python_pkg.token_dashboard import levers - -if TYPE_CHECKING: - from pathlib import Path - -TABLE = """## Candidate levers (rule-based) - -| category | lever | evidence | est $/day | action | -|---|---|---|---|---| -| cheaper agents | Opus runs most subagent work | subagents 64% | $14.47 | route | -| agent mechanics | Large Bash results | 40% of Bash | - | read ranges | - -## Next -""" - - -def test_parse_candidate_table() -> None: - rows = levers.parse_candidates(TABLE) - assert [r["title"] for r in rows] == [ - "Opus runs most subagent work", - "Large Bash results", - ] - assert rows[0]["est_usd_per_day"] == 14.47 - assert rows[1]["est_usd_per_day"] is None - assert rows[0]["metric"] == "subagent_opus_usd_share" - - -def test_ledger_date_falls_back_to_text_then_unknown() -> None: - assert levers.ledger_date({"date": "2026-10-04", "lever": "x"}) == "2026-10-04" - assert levers.ledger_date({"lever": "weekly prefix prune 2026-10-05"}) == ( - "2026-10-05" - ) - assert levers.ledger_date({"lever": "no date"}) is None - - -def _report(tmp_path: Path) -> Path: - report = tmp_path / "report" - (report / "history").mkdir(parents=True) - ledger = [ - { - "date": "2026-10-04", - "lever": "routing", - "metric": "subagent_opus_usd_share", - "baseline": 0.98, - "predicted": 0.55, - }, - { - "lever": "weekly prefix prune 2026-10-05", - "metric": "standing_tokens_est", - "baseline": 22187, - "predicted": 19620, - }, - ] - (report / "ledger.jsonl").write_text( - "\n".join(json.dumps(e) for e in ledger), encoding="utf-8" - ) - for day, value in (("2026-10-10", 0.8), ("2026-10-17", 0.84)): - snap = { - "generated": f"{day}T10:00:00+00:00", - "metrics": {"subagent_opus_usd_share": value, "standing_tokens_est": 23670}, - } - (report / "history" / f"{day}.json").write_text(json.dumps(snap)) - for day in ("2026-10-05", "2026-10-12"): - (report / "prune" / day).mkdir(parents=True) - (report / "prune" / day / "PROPOSAL.md").write_text( - f"# Prefix prune proposal {day}\n\nModest week. Estimated ~ -0.35k.\n" - ) - return report - - -def test_build_rows_status_and_outcome(tmp_path: Path) -> None: - report = _report(tmp_path) - tables = {"2026-10-10": TABLE} - rows = levers.build(report, tables) - applied = [r for r in rows if r["source"] == "ledger"] - assert [r["lever"] for r in applied] == [ - "routing", - "weekly prefix prune 2026-10-05", - ] - assert applied[0]["outcome"] == "helped" - assert applied[1]["outcome"] == "regressed" - assert applied[1]["date"] == "2026-10-05" - status = {r["source"]: r["status"] for r in rows if r["source"].startswith("prune")} - assert status == {"prune 2026-10-05": "applied", "prune 2026-10-12": "proposed"} - # The 10-10 Opus candidate was followed by no ledger line on its metric - # and a newer snapshot exists, so it was not applied. - opus = next(r for r in rows if r["lever"] == "Opus runs most subagent work") - assert opus["status"] == "not applied" - - -def test_structured_levers_in_a_snapshot_win_over_markdown(tmp_path: Path) -> None: - report = _report(tmp_path) - snap_path = report / "history" / "2026-10-17.json" - snap = json.loads(snap_path.read_text()) - snap["levers"] = [ - { - "category": "c", - "title": "Structured one", - "evidence": "e", - "action": "a", - "est_usd_per_day": 1.5, - "metric": "usd_per_day", - }, - ] - snap_path.write_text(json.dumps(snap)) - rows = levers.build(report, {}) - found = next(r for r in rows if r["lever"] == "Structured one") - assert found["status"] == "proposed" - assert found["source"] == "snapshot 2026-10-17" diff --git a/python_pkg/token_dashboard/tests/test_schema_migrate.py b/python_pkg/token_dashboard/tests/test_schema_migrate.py deleted file mode 100644 index a1834eae..00000000 --- a/python_pkg/token_dashboard/tests/test_schema_migrate.py +++ /dev/null @@ -1,83 +0,0 @@ -from __future__ import annotations - -import sqlite3 -from typing import TYPE_CHECKING - -import pytest - -from python_pkg.token_dashboard import schema - -if TYPE_CHECKING: - from pathlib import Path - - -def _v1_store(path: Path) -> None: - """A store as schema v1 wrote it: commits without end_ts, no A/B columns.""" - conn = sqlite3.connect(path) - conn.executescript( - "CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT);" - "INSERT INTO meta VALUES ('schema', '1');" - "CREATE TABLE commits(key TEXT, prompt_idx INT, ts REAL, sha TEXT," - " repo TEXT);" - "INSERT INTO commits VALUES ('s1', 0, 5.0, 'abc', '/r');" - "CREATE TABLE agent_calls(key TEXT, tool_use_id TEXT, agent_id TEXT," - " prompt_idx INT, ts REAL, description TEXT, subagent_type TEXT," - " model TEXT);" - "INSERT INTO agent_calls VALUES ('s1','t','a',0,1.0,'d','x','m');" - ) - conn.commit() - conn.close() - - -def test_old_store_is_migrated_in_place_and_keeps_its_rows(tmp_path: Path) -> None: - path = tmp_path / "t.sqlite" - _v1_store(path) - conn = schema.connect(path) - kept = conn.execute("SELECT key, sha, end_ts FROM commits").fetchall() - calls = conn.execute("SELECT tool_use_id, result_ts, status FROM agent_calls") - version = conn.execute("SELECT value FROM meta WHERE key='schema'").fetchone() - assert kept == [("s1", "abc", None)] - assert calls.fetchall() == [("t", None, None)] - assert version == (schema.VERSION,) - conn.close() - - -def test_newer_store_is_refused_not_relabelled(tmp_path: Path) -> None: - path = tmp_path / "t.sqlite" - conn = sqlite3.connect(path) - conn.executescript( - "CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT);" - "INSERT INTO meta VALUES ('schema', '999');" - ) - conn.commit() - conn.close() - with pytest.raises(schema.SchemaError): - schema.connect(path) - check = sqlite3.connect(path) - assert check.execute("SELECT value FROM meta").fetchone() == ("999",) - check.close() - - -def _meta_only(path: Path, version: str | None) -> None: - conn = sqlite3.connect(path) - conn.execute("CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT)") - if version is not None: - conn.execute("INSERT INTO meta VALUES ('schema', ?)", (version,)) - conn.commit() - conn.close() - - -def test_meta_without_a_version_is_refused(tmp_path: Path) -> None: - _meta_only(tmp_path / "t.sqlite", None) - with pytest.raises(schema.SchemaError): - schema.connect(tmp_path / "t.sqlite") - - -def test_table_missing_from_an_old_store_is_created_at_todays_shape( - tmp_path: Path, -) -> None: - _meta_only(tmp_path / "t.sqlite", "1") - conn = schema.connect(tmp_path / "t.sqlite") - columns = {r[1] for r in conn.execute("PRAGMA table_info(commits)")} - conn.close() - assert "end_ts" in columns diff --git a/python_pkg/token_dashboard/tests/test_walk.py b/python_pkg/token_dashboard/tests/test_walk.py deleted file mode 100644 index 2e94dbd5..00000000 --- a/python_pkg/token_dashboard/tests/test_walk.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import pytest - -from python_pkg.token_dashboard import walk -from python_pkg.token_dashboard.tests.conftest import ( - assistant, - prompt, - result, - usd_opus_output, - use, - write, -) - -if TYPE_CHECKING: - from pathlib import Path - - -def _main(tmp_path: Path, records: list[dict[str, object]]) -> walk.FileFacts: - path = write(tmp_path / "proj" / "sess-1.jsonl", records) - return walk.walk_file(path, tmp_path) - - -def test_one_api_message_written_as_three_records_is_priced_once( - tmp_path: Path, -) -> None: - blocks = [use("t1", "Bash", command="ls"), use("t2", "Read", file_path="/a")] - facts = _main( - tmp_path, - [ - prompt("do it", 0), - assistant("m1", 1, blocks=[{"type": "thinking", "thinking": ""}]), - assistant("m1", 1, blocks=blocks[:1]), - assistant("m1", 1, blocks=blocks[1:]), - result("t1", 2), - result("t2", 2), - ], - ) - assert len(facts.rows.turns) == 1 - assert facts.rows.turns[0].usd == pytest.approx(usd_opus_output(1000)) - # The one turn's dollars split evenly across the two tools it issued. - by_name = {t.name: t for t in facts.rows.tools} - assert by_name["Bash"].usd == pytest.approx(usd_opus_output(500)) - assert by_name["Bash"].calls == 1 - assert by_name["Read"].calls == 1 - assert by_name["Read"].result_tokens > 0 - - -def test_prompts_split_turns_and_text_only_turns_go_to_no_tool( - tmp_path: Path, -) -> None: - facts = _main( - tmp_path, - [ - prompt("first " + "a" * 300, 0, promptId="p1"), - assistant("m1", 10), - prompt("second", 100), - assistant("m2", 110), - assistant("m3", 120), - ], - ) - assert [p.idx for p in facts.rows.prompts] == [0, 1] - assert len(facts.rows.prompts[0].text) <= walk.PROMPT_CHARS - assert facts.rows.prompts[0].prompt_id == "p1" - assert [t.prompt_idx for t in facts.rows.turns] == [0, 1, 1] - assert {t.name for t in facts.rows.tools} == {walk.NO_TOOL} - - -def test_tool_results_and_meta_records_are_not_prompts(tmp_path: Path) -> None: - facts = _main( - tmp_path, - [ - prompt("real", 0), - prompt("injected", 1), - prompt("hidden", 2, isMeta=True), - prompt("[Request interrupted by user]", 3), - prompt("/clear", 4), - assistant("m1", 5, blocks=[use("t1", "Bash", command="ls")]), - result("t1", 6), - ], - ) - assert [p.text for p in facts.rows.prompts] == ["real", "/clear"] - - -def test_agent_call_and_commit_and_final_tag_are_captured(tmp_path: Path) -> None: - agent = use("ta", "Agent", description="Find x", subagent_type="Explore") - commit = use("tc", "Bash", command="git -C ~/src/foo commit -m x") - facts = _main( - tmp_path, - [ - prompt("go", 0), - assistant("m1", 1, blocks=[agent]), - result("ta", 2, "launched", toolUseResult={"agentId": "abc"}), - assistant("m2", 3, blocks=[commit]), - result("tc", 4, "[main 1a2b3c4d] x\n 1 file changed"), - assistant( - "m3", 5, blocks=[{"type": "text", "text": "x\nDONE: ok\nNOTE: y"}] - ), - ], - ) - call = facts.rows.agent_calls[0] - assert (call.tool_use_id, call.agent_id, call.description) == ( - "ta", - "abc", - "Find x", - ) - assert call.subagent_type == "Explore" - assert facts.rows.commits[0].sha == "1a2b3c4d" - assert facts.rows.commits[0].repo.endswith("/src/foo") - assert facts.final_tag == "DONE" - - -@pytest.mark.parametrize( - ("text", "tag"), - [ - ("all good", None), - ("**NEEDS ACTION:** run it", "NEEDS ACTION"), - ("RUNNING: tests", "RUNNING"), - ], -) -def test_final_tag_variants(tmp_path: Path, text: str, tag: str | None) -> None: - blocks = [{"type": "text", "text": text}] - facts = _main(tmp_path, [prompt("go", 0), assistant("m1", 1, blocks=blocks)]) - assert facts.final_tag == tag - - -def test_lever_table_in_a_tool_result_is_kept(tmp_path: Path) -> None: - table = ( - "## Candidate levers (rule-based)\n\n| category | lever |\n|---|---|\n" - "| a | b |\n\nafter" - ) - facts = _main( - tmp_path, - [ - prompt("go", 0), - assistant("m1", 1, blocks=[use("t1", "Bash", command="x")]), - result("t1", 2, table), - ], - ) - assert facts.rows.lever_tables[0][1].splitlines()[-1] == "| a | b |" - - -def test_activity_intervals_break_on_gaps_over_five_minutes(tmp_path: Path) -> None: - facts = _main( - tmp_path, - [prompt("a", 0), assistant("m1", 60), prompt("b", 1000), assistant("m2", 1100)], - ) - spans = [(round(e - s)) for s, e in facts.rows.intervals] - assert spans == [60, 100] - assert facts.span[1] - facts.span[0] == 1100 - - -def test_subagent_file_reads_meta_and_parent(tmp_path: Path) -> None: - sub = tmp_path / "proj" / "sess-1" / "subagents" / "agent-abc.jsonl" - write(sub, [prompt("task", 0), assistant("m1", 1, model="claude-sonnet-5-5")]) - sub.with_suffix(".meta.json").write_text( - '{"agentType": "Explore", "description": "Find", "toolUseId": "ta"}', - encoding="utf-8", - ) - facts = walk.walk_file(sub, tmp_path) - assert facts.key == "sess-1/agent-abc" - assert facts.parent_session == "sess-1" - assert facts.meta["toolUseId"] == "ta" - assert facts.rows.prompts == [] diff --git a/python_pkg/token_dashboard/walk.py b/python_pkg/token_dashboard/walk.py deleted file mode 100644 index 79f7cfed..00000000 --- a/python_pkg/token_dashboard/walk.py +++ /dev/null @@ -1,226 +0,0 @@ -"""Walk one transcript file into the rows the store keeps for it. - -Reuses token_audit's record primitives -- line loading, usage extraction, -per-API-message dedup (:class:`TurnMerger`) and list-price pricing -- rather -than :func:`parse.iter_events`, because the dashboard also needs what that -stream drops: every record's timestamp (active time), prompt boundaries, the -``Agent`` calls that spawned subagents, and commit hashes. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -from python_pkg.token_audit import discover, parse, pricing, records, turns -from python_pkg.token_dashboard import active, blocks, carry, detail -from python_pkg.token_dashboard.blocks import PROMPT_CHARS -from python_pkg.token_dashboard.rows import ( - AgentCall, - Commit, - FileFacts, - PromptRow, - ToolRow, - TurnRow, -) - -if TYPE_CHECKING: - from pathlib import Path - - from python_pkg.token_audit.model import Turn - -__all__ = ["NO_TOOL", "PROMPT_CHARS", "FileFacts", "walk_file"] - -NO_TOOL = "(no tool)" -AGENT_TOOLS = frozenset({"Agent", "Task"}) -_BOOK = pricing.load() - - -class _Walker: - """Single pass over a file's records, holding the cross-record state.""" - - def __init__(self, facts: FileFacts) -> None: - self.facts = facts - self.prompt = -1 - self.ts = 0.0 - self.stamps: list[float] = [] - self.pending: dict[str, dict[str, Any]] = {} - self.names: dict[str, list[str]] = {} - self.use_ts: dict[str, float] = {} - self.tools: dict[tuple[int, str], ToolRow] = {} - self.carry = carry.Tracker() - self.detail = detail.Detail() - self.merger = records.TurnMerger() - self.last_text = "" - - def tool(self, name: str) -> ToolRow: - """The accumulator for ``name`` in the current prompt.""" - return self.tools.setdefault((self.prompt, name), ToolRow(self.prompt, name)) - - def release(self, turn: Turn | None) -> None: - """Price a completed API call and split it across its tools.""" - if turn is None: - return - cost = pricing.usd(turn, _BOOK) - usage = turn.usage - self.facts.rows.turns.append( - TurnRow( - turn.timestamp or self.ts, - turn.model, - cost, - turn.context, - self.prompt, - usage, - ) - ) - self.carry.turn(turn.model, turn.context) - self.detail.turn(turn, cost, self.prompt) - names = self.names.pop(turn.message_id, []) or [NO_TOOL] - for name in names: - self.tool(name).usd += cost / len(names) - - def record(self, rec: dict[str, Any]) -> None: - """Account one record.""" - stamp = records.timestamp(rec) - if stamp is not None: - self.ts = stamp - self.stamps.append(stamp) - if self.facts.cwd is None and isinstance(rec.get("cwd"), str): - self.facts.cwd = rec["cwd"] - if rec.get("type") == "user": - self.release(self.merger.flush()) - self.user(rec) - self.detail.record(rec, self.ts, self.prompt) - for block in parse.content_blocks(rec): - self.block(rec, block) - found = turns.usage_of(rec) - if found is not None: - self.release(self.merger.push(turns.build_turn(rec, *found))) - - def user(self, rec: dict[str, Any]) -> None: - """Open a new prompt when this user record is one.""" - text = blocks.prompt_text(rec) - if text is None or self.facts.parent_session is not None: - return - self.prompt += 1 - prompt_id = rec.get("promptId") - self.facts.rows.prompts.append( - PromptRow(self.prompt, self.ts, str(prompt_id or ""), text) - ) - - def block(self, rec: dict[str, Any], block: dict[str, Any]) -> None: - """Remember tool calls, pair results, keep the last reply text.""" - kind = block.get("type") - if kind == "tool_use": - self.tool_use(rec, block) - elif kind == "tool_result": - self.tool_result(rec, block) - elif kind == "text" and rec.get("type") == "assistant": - self.last_text = str(block.get("text") or "") - - def tool_use(self, rec: dict[str, Any], block: dict[str, Any]) -> None: - """Hold a call until its result arrives; note which message issued it.""" - call_id = str(block.get("id") or "") - self.pending[call_id] = block - self.use_ts[call_id] = self.ts - message = rec.get("message") - mid = message.get("id") if isinstance(message, dict) else None - name = str(block.get("name") or "unknown") - self.names.setdefault(str(mid or ""), []).append(name) - self.detail.use(rec, block, self.ts, self.prompt) - - def tool_result(self, rec: dict[str, Any], block: dict[str, Any]) -> None: - """Pair a result with its call and record what it tells us.""" - call_id = str(block.get("tool_use_id") or "") - use = self.pending.get(call_id) - paired = parse.pair_result(block, self.pending) - if use is None or paired is None: - return - self.detail.result(block, self.ts) - row = self.tool(paired.name) - row.calls += 1 - row.result_tokens += paired.result_tokens - self.carry.result(row, paired.result_tokens) - text = blocks.text_of(block.get("content")) - raw_input = use.get("input") - tool_input: dict[str, Any] = raw_input if isinstance(raw_input, dict) else {} - if paired.name in AGENT_TOOLS: - failed = bool(block.get("is_error")) - self.agent_call(rec, call_id, tool_input, failed=failed) - elif paired.name == "SendMessage": - self.facts.rows.messages.append((self.ts, str(tool_input.get("to") or ""))) - if paired.commit: - # Every commit command is kept, hash or not: most run through - # finish_auto.sh with a tailed or backgrounded output, so the hash - # is usually found later by :mod:`commits` from git log instead. - hint = blocks.repo_hint(str(tool_input.get("command")), self.facts.cwd) - started = self.use_ts.pop(call_id, self.ts) - self.facts.rows.commits += [ - Commit(self.prompt, started, self.ts, sha, hint) - for sha in blocks.commit_shas(text) or [""] - ] - table = blocks.lever_table(text) - if table is not None: - self.facts.rows.lever_tables.append((self.ts, table)) - - def agent_call( - self, - rec: dict[str, Any], - call_id: str, - tool_input: dict[str, Any], - *, - failed: bool, - ) -> None: - """Record a subagent spawn with the ``agentId`` its result names.""" - result = rec.get("toolUseResult") - result = result if isinstance(result, dict) else {} - agent_id = result.get("agentId") - status = "error" if failed else str(result.get("status") or "") - self.facts.rows.agent_calls.append( - AgentCall( - tool_use_id=call_id, - agent_id=str(agent_id or ""), - prompt_idx=self.prompt, - ts=self.use_ts.get(call_id, self.ts), - description=str(tool_input.get("description") or ""), - subagent_type=str(tool_input.get("subagent_type") or ""), - model=str(tool_input.get("model") or ""), - result_ts=self.ts, - status=status, - ) - ) - - def finish(self) -> FileFacts: - """Flush state into the facts.""" - self.release(self.merger.flush()) - facts = self.facts - facts.rows.tools = list(self.tools.values()) - self.carry.settle(_BOOK) - facts.rows.detail = self.detail.finish(_BOOK) - facts.rows.intervals = active.intervals(self.stamps) - if self.stamps: - facts.span = (min(self.stamps), max(self.stamps)) - facts.final_tag = blocks.final_tag(self.last_text) - return facts - - -def walk_file(path: Path, root: Path) -> FileFacts: - """Read one transcript (main or subagent) into :class:`FileFacts`.""" - del root # keys are derived from the path's own layout - if path.parent.name == "subagents": - parent = path.parent.parent.name - facts = FileFacts( - key=f"{parent}/{path.stem}", - session_id=parent, - parent_session=parent, - cwd=None, - meta=discover.agent_meta(path), - ) - else: - facts = FileFacts(path.stem, path.stem, None, None, {}) - walker = _Walker(facts) - with path.open(encoding="utf-8", errors="replace") as handle: - for line in handle: - rec = parse.load_line(line) - if rec is not None: - walker.record(rec) - return walker.finish()