mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-10 22:07:03 +01:00
Krzysztof kuhy Rudnicki
11aea86920
Answers "what is billed on every turn regardless of use" by measurement instead of recall, which is the part earlier audits kept getting wrong. A surface is reported dead only when unused in BOTH windows. One earlier audit's seven-day dead list included six servers that had been used within thirty days, so a single window is not evidence. Shares are weighted with cache-read weight 0.1, so a cached surface is not billed as though it were re-read in full each turn. Verified by running it, not just by tests: the report renders over 44,311 turns, putting rules/ at 4,907 tokens (1.77%), MEMORY.md at 4,312 (1.55%), memories/ at 3,758 (1.35%), skill descriptions at 3,279 (1.18%) and CLAUDE.md at 2,057 (0.74%), with per-MCP 7d/30d call counts behind each verdict. The files were found untracked in the tree from an earlier session; every module is already under the 250-line cap, so nothing needed splitting, and the package was already at 100% branch coverage, so nothing needed new tests. This lands the instrument only -- no surface is unregistered on its say-so. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
112 lines
3.6 KiB
Python
112 lines
3.6 KiB
Python
"""Tests for the prefix probe and its prediction gate."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pytest
|
|
|
|
from python_pkg.token_audit import probe
|
|
|
|
if TYPE_CHECKING:
|
|
from pathlib import Path
|
|
|
|
|
|
def test_prefix_sums_cold_start() -> None:
|
|
"""A cold start reports the prefix as cache creation."""
|
|
usage = {
|
|
"input_tokens": 2,
|
|
"cache_creation_input_tokens": 46434,
|
|
"cache_read_input_tokens": 0,
|
|
}
|
|
assert probe._prefix_from_usage(usage) == 46436
|
|
|
|
|
|
def test_prefix_sums_warm_start() -> None:
|
|
"""A warm start reports the same prefix as cache read."""
|
|
usage = {"input_tokens": 2, "cache_read_input_tokens": 46434}
|
|
assert probe._prefix_from_usage(usage) == 46436
|
|
|
|
|
|
def test_prefix_handles_missing_and_null() -> None:
|
|
"""Absent or null token classes count as zero."""
|
|
assert probe._prefix_from_usage({}) == 0
|
|
assert probe._prefix_from_usage({"input_tokens": None}) == 0
|
|
|
|
|
|
def test_prediction_within_tolerance_passes() -> None:
|
|
"""A prediction close to the measurement is not flagged."""
|
|
assert probe.check_prediction(1000, 900) is None
|
|
|
|
|
|
def test_prediction_beyond_tolerance_is_flagged() -> None:
|
|
"""The 6,122-vs-4,465 miss from a real audit must fail the gate."""
|
|
warning = probe.check_prediction(6122, 4465)
|
|
assert warning is not None
|
|
assert "27%" in warning
|
|
|
|
|
|
def test_prediction_ignores_nonpositive() -> None:
|
|
"""No prediction means nothing to check."""
|
|
assert probe.check_prediction(0, 500) is None
|
|
|
|
|
|
def test_weighted_pct_applies_cache_read_weight() -> None:
|
|
"""Prefix savings are billed as cache reads, at weight 0.1."""
|
|
assert probe.weighted_pct(4465, 44311, 1230346533) == pytest.approx(1.608, abs=0.01)
|
|
|
|
|
|
def test_weighted_pct_guards_zero_total() -> None:
|
|
"""An empty week cannot divide by zero."""
|
|
assert probe.weighted_pct(100, 10, 0) == 0.0
|
|
|
|
|
|
def test_measure_runs_non_strict_by_default(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""Strict mode hides project .mcp.json, so it must be opt-in."""
|
|
seen: dict[str, list[str]] = {}
|
|
|
|
def fake_run(cmd: list[str], **_: object) -> subprocess.CompletedProcess[str]:
|
|
seen["cmd"] = cmd
|
|
return subprocess.CompletedProcess(
|
|
cmd, 0, json.dumps({"usage": {"input_tokens": 5}}), ""
|
|
)
|
|
|
|
monkeypatch.setattr(subprocess, "run", fake_run)
|
|
result = probe.measure("base", mcp_config=tmp_path / "x.json")
|
|
assert "--strict-mcp-config" not in seen["cmd"]
|
|
assert result.prefix_tokens == 5
|
|
|
|
|
|
def test_measure_can_opt_into_strict(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""Strict mode is still reachable when deliberately requested."""
|
|
seen: dict[str, list[str]] = {}
|
|
|
|
def fake_run(cmd: list[str], **_: object) -> subprocess.CompletedProcess[str]:
|
|
seen["cmd"] = cmd
|
|
return subprocess.CompletedProcess(cmd, 0, json.dumps({"usage": {}}), "")
|
|
|
|
monkeypatch.setattr(subprocess, "run", fake_run)
|
|
probe.measure("strict", mcp_config=tmp_path / "x.json", strict=True)
|
|
assert "--strict-mcp-config" in seen["cmd"]
|
|
|
|
|
|
def test_measure_without_mcp_config_omits_flags(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""The plain probe passes no MCP flags at all."""
|
|
seen: dict[str, list[str]] = {}
|
|
|
|
def fake_run(cmd: list[str], **_: object) -> subprocess.CompletedProcess[str]:
|
|
seen["cmd"] = cmd
|
|
return subprocess.CompletedProcess(cmd, 0, json.dumps({"usage": {}}), "")
|
|
|
|
monkeypatch.setattr(subprocess, "run", fake_run)
|
|
probe.measure("plain")
|
|
assert "--mcp-config" not in seen["cmd"]
|