Compare commits

..

No commits in common. "main" and "gatelock-v0.1.1" have entirely different histories.

1490 changed files with 1233 additions and 152471 deletions

View File

@ -1,48 +0,0 @@
name: crdt_sync_dart tests
on:
push:
branches: [main]
paths: ['crdt_sync_dart/**']
pull_request:
branches: [main]
paths: ['crdt_sync_dart/**']
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: crdt_sync_dart
steps:
- uses: actions/checkout@v4
# Pure Dart, no Flutter dependency -- so setup-dart, not flutter-action.
- name: Set up Dart
uses: dart-lang/setup-dart@v1
with:
sdk: stable
- name: Install dependencies
run: dart pub get
- name: Analyze
run: dart analyze --fatal-infos
- name: Run tests with coverage
run: dart test --coverage=coverage
# Unlike `flutter test --coverage`, `dart test --coverage` emits raw VM
# JSON and no lcov. Skipping this step leaves a stale lcov.info on disk
# that the gate below happily "passes" -- which is exactly how this
# package drifted to 96% while believing it was at 100%.
- name: Convert coverage to lcov
run: |
dart pub global activate coverage
dart pub global run coverage:format_coverage \
--lcov --in=coverage/test --out=coverage/lcov.info \
--report-on=lib --packages=.dart_tool/package_config.json
- name: Enforce 100% line coverage
run: scripts/check_coverage.sh coverage/lcov.info 100

View File

@ -1,36 +0,0 @@
name: crdt_sync_flutter tests
on:
push:
branches: [main]
paths: ['crdt_sync_flutter/**', 'crdt_sync_dart/**']
pull_request:
branches: [main]
paths: ['crdt_sync_flutter/**', 'crdt_sync_dart/**']
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: crdt_sync_flutter
steps:
- uses: actions/checkout@v4
- name: Set up Flutter
uses: subosito/flutter-action@v2
with:
channel: stable
- name: Install dependencies
run: flutter pub get
- name: Analyze
run: flutter analyze
- name: Run tests with coverage
run: flutter test --coverage
- name: Enforce 100% line coverage
run: scripts/check_coverage.sh coverage/lcov.info 100

View File

@ -3,10 +3,10 @@ name: crdt-sync pre-commit
on:
push:
branches: [main]
paths: ['crdt-sync/**', '.github/workflows/crdt-sync-precommit.yml']
paths: ['crdt-sync/**']
pull_request:
branches: [main]
paths: ['crdt-sync/**', '.github/workflows/crdt-sync-precommit.yml']
paths: ['crdt-sync/**']
jobs:
pre-commit:
@ -15,56 +15,10 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.x"
# Only pre-commit itself. Every linting hook runs in its own isolated
# env -- ruff, mypy, pylint, bandit and codespell each come from their
# own repo+rev, and the local hooks need nothing beyond bash and
# python3 -- so installing the full requirements.txt spent 9s a run on
# packages no pre-commit-stage hook imports. Verified by running all
# three configs green in a venv holding pre-commit and nothing else.
# The pin is read out of requirements.txt rather than repeated here, so
# it cannot drift from what the dependency-freshness gate governs.
- name: Install pre-commit
run: |
pip install "$(grep '^pre-commit==' crdt-sync/requirements.txt)"
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
# package jobs shared one key, raced to overwrite it with hook sets built
# from four different configs, and editing a per-package config did not
# invalidate it. The cache was therefore silently wrong rather than
# missing, and the hooks reinstalled from scratch on every run (38-49s).
- uses: actions/cache@v4
python-version: "3.12"
- name: Install dependencies
working-directory: crdt-sync
run: pip install -r requirements.txt
- uses: pre-commit/action@v3.0.1
with:
path: ~/.cache/pre-commit
key: >-
pre-commit-crdt-sync-${{ env.pythonLocation }}-${{
hashFiles('crdt-sync/.pre-commit-config.yaml') }}
# Prefix fallback, or this fix would evaporate on the very
# event that motivated it: bumping a hook rev changes the
# exact key, and a total miss means reinstalling every hook
# again. pre-commit's store is keyed internally by hook
# repo+rev, so a superseded entry is ignored disk rather
# than a wrong hook, and cache still re-saves under the new
# exact key after a prefix-only hit.
restore-keys: pre-commit-crdt-sync-${{ env.pythonLocation }}-
# mypy re-analyses the stdlib and every third-party stub from scratch
# without this, which measured 10s of the hook step in CI against 0.25s
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
# the config, not the sources: mypy's own cache metadata records the
# interpreter, the flag set and a hash per source file, so it re-checks
# what changed and cannot pass stale code. Deliberately NOT keyed on
# github.sha -- that re-saves 19MB x3 every commit and would evict the
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
# the expensive part, the dependency analysis, which is what costs 10s.
- uses: actions/cache@v4
with:
path: .mypy_cache
key: >-
mypy-crdt-sync-${{ env.pythonLocation }}-${{
hashFiles('crdt-sync/.pre-commit-config.yaml') }}
restore-keys: mypy-crdt-sync-${{ env.pythonLocation }}-
- name: Run the hooks
run: >-
pre-commit run --config crdt-sync/.pre-commit-config.yaml
--all-files --show-diff-on-failure
extra_args: --config crdt-sync/.pre-commit-config.yaml --all-files

View File

@ -3,10 +3,10 @@ name: crdt-sync tests
on:
push:
branches: [main]
paths: ['crdt-sync/**', '.github/workflows/crdt-sync-tests.yml']
paths: ['crdt-sync/**']
pull_request:
branches: [main]
paths: ['crdt-sync/**', '.github/workflows/crdt-sync-tests.yml']
paths: ['crdt-sync/**']
jobs:
test:
@ -14,15 +14,17 @@ jobs:
defaults:
run:
working-directory: crdt-sync
strategy:
matrix:
python-version: ["3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v4
- name: Set up Python
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: "3.x"
cache: pip
cache-dependency-path: crdt-sync/requirements.txt
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: pip install -r requirements.txt

View File

@ -1,36 +0,0 @@
name: data_transfer_flutter tests
on:
push:
branches: [main]
paths: ['data_transfer_flutter/**']
pull_request:
branches: [main]
paths: ['data_transfer_flutter/**']
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: data_transfer_flutter
steps:
- uses: actions/checkout@v4
- name: Set up Flutter
uses: subosito/flutter-action@v2
with:
channel: stable
- name: Install dependencies
run: flutter pub get
- name: Analyze
run: flutter analyze
- name: Run tests with coverage
run: flutter test --coverage
- name: Enforce 100% line coverage
run: scripts/check_coverage.sh coverage/lcov.info 100

View File

@ -1,61 +0,0 @@
name: dependency freshness
# Every dependency on its ecosystem's newest STABLE release, enforced on push
# as well as at commit time. The local pre-commit hook only sees the manifests
# in a commit and can be skipped entirely by a fresh clone that never ran
# `pre-commit install`; this runs `--all` over the whole repo.
#
# `--strict` is CI-only. Offline the gate degrades to its cached answer and
# passes, because `--no-verify` is banned locally and a hook that hard-failed
# on a flaky connection would leave no way to commit. A runner is always
# online, so here an undeterminable answer is a real failure.
#
# Expect this to go red for reasons this repo's commit did not cause: an
# upstream release overnight is enough. That is the chosen behaviour -- the
# alternative is a gate that only notices staleness when someone remembers to
# look.
on:
push:
branches: [main]
pull_request:
workflow_dispatch:
jobs:
dependency-freshness:
name: dependencies on newest stable
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
# The gate lives in the shared utils repo so every repo resolves the
# same "latest stable" the same way. Checking it out beats vendoring a
# copy, which is exactly how a gate and its survey start disagreeing.
- name: Check out the shared gate
uses: actions/checkout@v4
with:
repository: kuhyx/utils
path: .utils
# Any recent Python runs the gate. The Python TARGET is python.org's
# newest release (registries/toolchain.python_latest), not this
# interpreter: when it was, "3.x" resolved to 3.15.0 on one runner and
# 3.14.8 on the next, and spellcode's allowlist flipped dead/required.
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"
- name: Install the gate's two runtime dependencies
run: python -m pip install --upgrade pyyaml packaging
- name: Cache registry lookups
uses: actions/cache@v4
with:
path: ~/.cache/dep-freshness
key: dep-freshness-${{ hashFiles('**/pubspec.yaml', '**/pyproject.toml',
'**/requirements*.txt', '**/package.json', '**/Cargo.toml', '**/go.mod') }}
restore-keys: dep-freshness-
- name: Every dependency is on newest stable
run: bash .utils/scripts/check_dependency_freshness.sh --all --strict

View File

@ -1,36 +0,0 @@
name: design_system tests
on:
push:
branches: [main]
paths: ['design_system/**']
pull_request:
branches: [main]
paths: ['design_system/**']
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: design_system
steps:
- uses: actions/checkout@v4
- name: Set up Flutter
uses: subosito/flutter-action@v2
with:
channel: stable
- name: Install dependencies
run: flutter pub get
- name: Analyze
run: flutter analyze
- name: Run tests with coverage
run: flutter test --coverage
- name: Enforce 100% line coverage
run: scripts/check_coverage.sh coverage/lcov.info 100

View File

@ -1,73 +0,0 @@
name: earned_time pre-commit
on:
push:
branches: [main]
# The workflow's own file counts as earned_time: editing only this file would
# otherwise trigger nothing, so a fix to the job itself could never prove
# itself.
paths: ['earned_time/**', '.github/workflows/earned-time-precommit.yml']
pull_request:
branches: [main]
paths: ['earned_time/**', '.github/workflows/earned-time-precommit.yml']
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.x"
# Only pre-commit itself. Every linting hook runs in its own isolated
# env -- ruff, mypy, pylint, bandit and codespell each come from their
# own repo+rev, and the local hooks need nothing beyond bash and
# python3 -- so installing the full requirements.txt spent 9s a run on
# packages no pre-commit-stage hook imports. Verified by running all
# three configs green in a venv holding pre-commit and nothing else.
# The pin is read out of requirements.txt rather than repeated here, so
# it cannot drift from what the dependency-freshness gate governs.
- name: Install pre-commit
run: |
pip install "$(grep '^pre-commit==' earned_time/requirements.txt)"
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
# package jobs shared one key, raced to overwrite it with hook sets built
# from four different configs, and editing a per-package config did not
# invalidate it. The cache was therefore silently wrong rather than
# missing, and the hooks reinstalled from scratch on every run (38-49s).
- uses: actions/cache@v4
with:
path: ~/.cache/pre-commit
key: >-
pre-commit-earned_time-${{ env.pythonLocation }}-${{
hashFiles('earned_time/.pre-commit-config.yaml') }}
# Prefix fallback, or this fix would evaporate on the very
# event that motivated it: bumping a hook rev changes the
# exact key, and a total miss means reinstalling every hook
# again. pre-commit's store is keyed internally by hook
# repo+rev, so a superseded entry is ignored disk rather
# than a wrong hook, and cache still re-saves under the new
# exact key after a prefix-only hit.
restore-keys: pre-commit-earned_time-${{ env.pythonLocation }}-
# mypy re-analyses the stdlib and every third-party stub from scratch
# without this, which measured 10s of the hook step in CI against 0.25s
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
# the config, not the sources: mypy's own cache metadata records the
# interpreter, the flag set and a hash per source file, so it re-checks
# what changed and cannot pass stale code. Deliberately NOT keyed on
# github.sha -- that re-saves 19MB x3 every commit and would evict the
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
# the expensive part, the dependency analysis, which is what costs 10s.
- uses: actions/cache@v4
with:
path: .mypy_cache
key: >-
mypy-earned_time-${{ env.pythonLocation }}-${{
hashFiles('earned_time/.pre-commit-config.yaml') }}
restore-keys: mypy-earned_time-${{ env.pythonLocation }}-
- name: Run the hooks
run: >-
pre-commit run --config earned_time/.pre-commit-config.yaml
--all-files --show-diff-on-failure

View File

@ -1,36 +0,0 @@
name: earned_time tests
on:
push:
branches: [main]
# The workflow's own file counts as earned_time: editing only this file would
# otherwise trigger nothing, so a fix to the job itself could never prove
# itself.
paths: ['earned_time/**', '.github/workflows/earned-time-tests.yml']
pull_request:
branches: [main]
paths: ['earned_time/**', '.github/workflows/earned-time-tests.yml']
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: earned_time
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"
cache: pip
cache-dependency-path: earned_time/requirements.txt
- name: Install dependencies
run: pip install -r requirements.txt && pip install --no-deps -e .
# No display needed: earned_time is stdlib-only policy, imported
# headless by the root gaming daemon.
- name: Run tests with coverage
run: python -m pytest -v

View File

@ -1,34 +0,0 @@
name: file length
# The 250-line cap, enforced on push as well as in pre-commit
# (refactor_claude_todo.md, "make it permanent").
#
# Unlike the pre-commit hook, which only sees the files in a commit, this runs
# over the whole tree: the hook keeps a violation from being *introduced*, and
# this keeps one from surviving a path the hook never saw (a merge, a push
# from a machine without hooks installed, a rebase that resurrects a file).
#
# No paths filter on purpose. The cap applies to every file in the repo, so
# scoping this job to a subdirectory would be the one way to reintroduce the
# problem it exists to catch.
on:
push:
branches: [main]
pull_request:
branches: [main]
jobs:
file-length:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"
- name: Every file is at most 250 lines
run: bash scripts/check_file_length.sh --all

View File

@ -1,73 +0,0 @@
name: focus-project pre-commit
on:
push:
branches: [main]
# The workflow's own file counts as focus_project: editing only this file would
# otherwise trigger nothing, so a fix to the job itself could never prove
# itself.
paths: ['focus_project/**', '.github/workflows/focus-project-precommit.yml']
pull_request:
branches: [main]
paths: ['focus_project/**', '.github/workflows/focus-project-precommit.yml']
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.x"
# Only pre-commit itself. Every linting hook runs in its own isolated
# env -- ruff, mypy, pylint, bandit and codespell each come from their
# own repo+rev, and the local hooks need nothing beyond bash and
# python3 -- so installing the full requirements.txt spent 9s a run on
# packages no pre-commit-stage hook imports. Verified by running all
# three configs green in a venv holding pre-commit and nothing else.
# The pin is read out of requirements.txt rather than repeated here, so
# it cannot drift from what the dependency-freshness gate governs.
- name: Install pre-commit
run: |
pip install "$(grep '^pre-commit==' focus_project/requirements.txt)"
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
# package jobs shared one key, raced to overwrite it with hook sets built
# from four different configs, and editing a per-package config did not
# invalidate it. The cache was therefore silently wrong rather than
# missing, and the hooks reinstalled from scratch on every run (38-49s).
- uses: actions/cache@v4
with:
path: ~/.cache/pre-commit
key: >-
pre-commit-focus-project-${{ env.pythonLocation }}-${{
hashFiles('focus_project/.pre-commit-config.yaml') }}
# Prefix fallback, or this fix would evaporate on the very
# event that motivated it: bumping a hook rev changes the
# exact key, and a total miss means reinstalling every hook
# again. pre-commit's store is keyed internally by hook
# repo+rev, so a superseded entry is ignored disk rather
# than a wrong hook, and cache still re-saves under the new
# exact key after a prefix-only hit.
restore-keys: pre-commit-focus-project-${{ env.pythonLocation }}-
# mypy re-analyses the stdlib and every third-party stub from scratch
# without this, which measured 10s of the hook step in CI against 0.25s
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
# the config, not the sources: mypy's own cache metadata records the
# interpreter, the flag set and a hash per source file, so it re-checks
# what changed and cannot pass stale code. Deliberately NOT keyed on
# github.sha -- that re-saves 19MB x3 every commit and would evict the
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
# the expensive part, the dependency analysis, which is what costs 10s.
- uses: actions/cache@v4
with:
path: .mypy_cache
key: >-
mypy-focus-project-${{ env.pythonLocation }}-${{
hashFiles('focus_project/.pre-commit-config.yaml') }}
restore-keys: mypy-focus-project-${{ env.pythonLocation }}-
- name: Run the hooks
run: >-
pre-commit run --config focus_project/.pre-commit-config.yaml
--all-files --show-diff-on-failure

View File

@ -1,34 +0,0 @@
name: focus-project tests
on:
push:
branches: [main]
# The workflow's own file counts as focus_project: editing only this file would
# otherwise trigger nothing, so a fix to the job itself could never prove
# itself.
paths: ['focus_project/**', '.github/workflows/focus-project-tests.yml']
pull_request:
branches: [main]
paths: ['focus_project/**', '.github/workflows/focus-project-tests.yml']
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: focus_project
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"
cache: pip
cache-dependency-path: focus_project/requirements.txt
- name: Install dependencies
run: pip install -r requirements.txt && pip install --no-deps -e .
- name: Run tests with coverage
run: python -m pytest -v

View File

@ -1,73 +0,0 @@
name: freedays pre-commit
on:
push:
branches: [main]
# The workflow's own file counts as freedays: editing only this file would
# otherwise trigger nothing, so a fix to the job itself could never prove
# itself.
paths: ['freedays/**', '.github/workflows/freedays-precommit.yml']
pull_request:
branches: [main]
paths: ['freedays/**', '.github/workflows/freedays-precommit.yml']
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.x"
# Only pre-commit itself. Every linting hook runs in its own isolated
# env -- ruff, mypy, pylint, bandit and codespell each come from their
# own repo+rev, and the local hooks need nothing beyond bash and
# python3 -- so installing the full requirements.txt spent 9s a run on
# packages no pre-commit-stage hook imports. Verified by running all
# three configs green in a venv holding pre-commit and nothing else.
# The pin is read out of requirements.txt rather than repeated here, so
# it cannot drift from what the dependency-freshness gate governs.
- name: Install pre-commit
run: |
pip install "$(grep '^pre-commit==' freedays/requirements.txt)"
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
# package jobs shared one key, raced to overwrite it with hook sets built
# from four different configs, and editing a per-package config did not
# invalidate it. The cache was therefore silently wrong rather than
# missing, and the hooks reinstalled from scratch on every run (38-49s).
- uses: actions/cache@v4
with:
path: ~/.cache/pre-commit
key: >-
pre-commit-freedays-${{ env.pythonLocation }}-${{
hashFiles('freedays/.pre-commit-config.yaml') }}
# Prefix fallback, or this fix would evaporate on the very
# event that motivated it: bumping a hook rev changes the
# exact key, and a total miss means reinstalling every hook
# again. pre-commit's store is keyed internally by hook
# repo+rev, so a superseded entry is ignored disk rather
# than a wrong hook, and cache still re-saves under the new
# exact key after a prefix-only hit.
restore-keys: pre-commit-freedays-${{ env.pythonLocation }}-
# mypy re-analyses the stdlib and every third-party stub from scratch
# without this, which measured 10s of the hook step in CI against 0.25s
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
# the config, not the sources: mypy's own cache metadata records the
# interpreter, the flag set and a hash per source file, so it re-checks
# what changed and cannot pass stale code. Deliberately NOT keyed on
# github.sha -- that re-saves 19MB x3 every commit and would evict the
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
# the expensive part, the dependency analysis, which is what costs 10s.
- uses: actions/cache@v4
with:
path: .mypy_cache
key: >-
mypy-freedays-${{ env.pythonLocation }}-${{
hashFiles('freedays/.pre-commit-config.yaml') }}
restore-keys: mypy-freedays-${{ env.pythonLocation }}-
- name: Run the hooks
run: >-
pre-commit run --config freedays/.pre-commit-config.yaml
--all-files --show-diff-on-failure

View File

@ -1,44 +0,0 @@
name: freedays tests
on:
push:
branches: [main]
# The workflow's own file counts as freedays: editing only this file would
# otherwise trigger nothing, so a fix to the job itself could never prove
# itself.
paths: ['freedays/**', '.github/workflows/freedays-tests.yml']
pull_request:
branches: [main]
paths: ['freedays/**', '.github/workflows/freedays-tests.yml']
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: freedays
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"
cache: pip
cache-dependency-path: freedays/requirements.txt
# crdt-sync is a sibling in this repo, so it installs from the checkout
# rather than from a tag -- CI must test what is on this branch, not
# what was last released. Deliberately NOT gatelock: importing it pulls
# in tkinter, and this job has no display precisely because freedays
# must work without one.
- name: Install the sibling package
run: pip install ../crdt-sync
- name: Install dependencies
run: pip install -r requirements.txt && pip install --no-deps -e .
# No display needed: freedays draws nothing. It is policy, a JSON file
# and a CLI.
- name: Run tests with coverage
run: python -m pytest -v

View File

@ -3,12 +3,10 @@ name: gatelock pre-commit
on:
push:
branches: [main]
# The workflow's own file counts as gatelock: editing only this file used
# to trigger nothing, so a fix to the job itself could never prove itself.
paths: ['gatelock/**', '.github/workflows/gatelock-precommit.yml']
paths: ['gatelock/**']
pull_request:
branches: [main]
paths: ['gatelock/**', '.github/workflows/gatelock-precommit.yml']
paths: ['gatelock/**']
jobs:
pre-commit:
@ -17,56 +15,10 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.x"
# Only pre-commit itself. Every linting hook runs in its own isolated
# env -- ruff, mypy, pylint, bandit and codespell each come from their
# own repo+rev, and the local hooks need nothing beyond bash and
# python3 -- so installing the full requirements.txt spent 9s a run on
# packages no pre-commit-stage hook imports. Verified by running all
# three configs green in a venv holding pre-commit and nothing else.
# The pin is read out of requirements.txt rather than repeated here, so
# it cannot drift from what the dependency-freshness gate governs.
- name: Install pre-commit
run: |
pip install "$(grep '^pre-commit==' gatelock/requirements.txt)"
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
# package jobs shared one key, raced to overwrite it with hook sets built
# from four different configs, and editing a per-package config did not
# invalidate it. The cache was therefore silently wrong rather than
# missing, and the hooks reinstalled from scratch on every run (38-49s).
- uses: actions/cache@v4
python-version: "3.12"
- name: Install dependencies
working-directory: gatelock
run: pip install -r requirements.txt
- uses: pre-commit/action@v3.0.1
with:
path: ~/.cache/pre-commit
key: >-
pre-commit-gatelock-${{ env.pythonLocation }}-${{
hashFiles('gatelock/.pre-commit-config.yaml') }}
# Prefix fallback, or this fix would evaporate on the very
# event that motivated it: bumping a hook rev changes the
# exact key, and a total miss means reinstalling every hook
# again. pre-commit's store is keyed internally by hook
# repo+rev, so a superseded entry is ignored disk rather
# than a wrong hook, and cache still re-saves under the new
# exact key after a prefix-only hit.
restore-keys: pre-commit-gatelock-${{ env.pythonLocation }}-
# mypy re-analyses the stdlib and every third-party stub from scratch
# without this, which measured 10s of the hook step in CI against 0.25s
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
# the config, not the sources: mypy's own cache metadata records the
# interpreter, the flag set and a hash per source file, so it re-checks
# what changed and cannot pass stale code. Deliberately NOT keyed on
# github.sha -- that re-saves 19MB x3 every commit and would evict the
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
# the expensive part, the dependency analysis, which is what costs 10s.
- uses: actions/cache@v4
with:
path: .mypy_cache
key: >-
mypy-gatelock-${{ env.pythonLocation }}-${{
hashFiles('gatelock/.pre-commit-config.yaml') }}
restore-keys: mypy-gatelock-${{ env.pythonLocation }}-
- name: Run the hooks
run: >-
pre-commit run --config gatelock/.pre-commit-config.yaml
--all-files --show-diff-on-failure
extra_args: --config gatelock/.pre-commit-config.yaml --all-files

View File

@ -3,12 +3,10 @@ name: gatelock tests
on:
push:
branches: [main]
# The workflow's own file counts as gatelock: editing only this file used
# to trigger nothing, so a fix to the job itself could never prove itself.
paths: ['gatelock/**', '.github/workflows/gatelock-tests.yml']
paths: ['gatelock/**']
pull_request:
branches: [main]
paths: ['gatelock/**', '.github/workflows/gatelock-tests.yml']
paths: ['gatelock/**']
jobs:
test:
@ -16,26 +14,20 @@ jobs:
defaults:
run:
working-directory: gatelock
strategy:
matrix:
python-version: ["3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v4
- name: Set up Python
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: "3.x"
cache: pip
cache-dependency-path: gatelock/requirements.txt
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: pip install -r requirements.txt
# The viewport, fit-check and keyboard tests drive real Tk: focus
# delivery, canvas scrolling and widget heights are Tk's own behaviour,
# and mocking them is what let a viewport that scrolled itself pass
# every check for a week. Headless they skip, silently taking coverage
# with them -- so the runner gets a display instead.
- name: Install a virtual X display
run: sudo apt-get update && sudo apt-get install -y xvfb
- name: Run tests with coverage
run: xvfb-run -a -s "-screen 0 1600x1200x24" python -m pytest -v
run: python -m pytest -v

View File

@ -1,36 +0,0 @@
name: github_device_auth tests
on:
push:
branches: [main]
paths: ['github_device_auth/**']
pull_request:
branches: [main]
paths: ['github_device_auth/**']
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: github_device_auth
steps:
- uses: actions/checkout@v4
- name: Set up Flutter
uses: subosito/flutter-action@v2
with:
channel: stable
- name: Install dependencies
run: flutter pub get
- name: Analyze
run: flutter analyze
- name: Run tests with coverage
run: flutter test --coverage
- name: Enforce 100% line coverage
run: scripts/check_coverage.sh coverage/lcov.info 100

View File

@ -1,36 +0,0 @@
name: google_account_token tests
on:
push:
branches: [main]
paths: ['google_account_token/**']
pull_request:
branches: [main]
paths: ['google_account_token/**']
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: google_account_token
steps:
- uses: actions/checkout@v4
- name: Set up Flutter
uses: subosito/flutter-action@v2
with:
channel: stable
- name: Install dependencies
run: flutter pub get
- name: Analyze
run: flutter analyze
- name: Run tests with coverage
run: flutter test --coverage
- name: Enforce 100% line coverage
run: scripts/check_coverage.sh coverage/lcov.info 100

View File

@ -1,73 +0,0 @@
name: holds pre-commit
on:
push:
branches: [main]
# The workflow's own file counts as holds: editing only this file would
# otherwise trigger nothing, so a fix to the job itself could never prove
# itself.
paths: ['holds/**', '.github/workflows/holds-precommit.yml']
pull_request:
branches: [main]
paths: ['holds/**', '.github/workflows/holds-precommit.yml']
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.x"
# Only pre-commit itself. Every linting hook runs in its own isolated
# env -- ruff, mypy, pylint, bandit and codespell each come from their
# own repo+rev, and the local hooks need nothing beyond bash and
# python3 -- so installing the full requirements.txt spent 9s a run on
# packages no pre-commit-stage hook imports. Verified by running all
# three configs green in a venv holding pre-commit and nothing else.
# The pin is read out of requirements.txt rather than repeated here, so
# it cannot drift from what the dependency-freshness gate governs.
- name: Install pre-commit
run: |
pip install "$(grep '^pre-commit==' holds/requirements.txt)"
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
# package jobs shared one key, raced to overwrite it with hook sets built
# from four different configs, and editing a per-package config did not
# invalidate it. The cache was therefore silently wrong rather than
# missing, and the hooks reinstalled from scratch on every run (38-49s).
- uses: actions/cache@v4
with:
path: ~/.cache/pre-commit
key: >-
pre-commit-holds-${{ env.pythonLocation }}-${{
hashFiles('holds/.pre-commit-config.yaml') }}
# Prefix fallback, or this fix would evaporate on the very
# event that motivated it: bumping a hook rev changes the
# exact key, and a total miss means reinstalling every hook
# again. pre-commit's store is keyed internally by hook
# repo+rev, so a superseded entry is ignored disk rather
# than a wrong hook, and cache still re-saves under the new
# exact key after a prefix-only hit.
restore-keys: pre-commit-holds-${{ env.pythonLocation }}-
# mypy re-analyses the stdlib and every third-party stub from scratch
# without this, which measured 10s of the hook step in CI against 0.25s
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
# the config, not the sources: mypy's own cache metadata records the
# interpreter, the flag set and a hash per source file, so it re-checks
# what changed and cannot pass stale code. Deliberately NOT keyed on
# github.sha -- that re-saves 19MB x3 every commit and would evict the
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
# the expensive part, the dependency analysis, which is what costs 10s.
- uses: actions/cache@v4
with:
path: .mypy_cache
key: >-
mypy-holds-${{ env.pythonLocation }}-${{
hashFiles('holds/.pre-commit-config.yaml') }}
restore-keys: mypy-holds-${{ env.pythonLocation }}-
- name: Run the hooks
run: >-
pre-commit run --config holds/.pre-commit-config.yaml
--all-files --show-diff-on-failure

View File

@ -1,34 +0,0 @@
name: holds tests
on:
push:
branches: [main]
# The workflow's own file counts as holds: editing only this file would
# otherwise trigger nothing, so a fix to the job itself could never prove
# itself.
paths: ['holds/**', '.github/workflows/holds-tests.yml']
pull_request:
branches: [main]
paths: ['holds/**', '.github/workflows/holds-tests.yml']
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: holds
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"
cache: pip
cache-dependency-path: holds/requirements.txt
- name: Install dependencies
run: pip install -r requirements.txt && pip install --no-deps -e .
- name: Run tests with coverage
run: python -m pytest -v

View File

@ -1,73 +0,0 @@
name: leetcode-coach pre-commit
on:
push:
branches: [main]
# The workflow's own file counts as leetcode_coach: editing only this file would
# otherwise trigger nothing, so a fix to the job itself could never prove
# itself.
paths: ['leetcode_coach/**', '.github/workflows/leetcode-coach-precommit.yml']
pull_request:
branches: [main]
paths: ['leetcode_coach/**', '.github/workflows/leetcode-coach-precommit.yml']
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.x"
# Only pre-commit itself. Every linting hook runs in its own isolated
# env -- ruff, mypy, pylint, bandit and codespell each come from their
# own repo+rev, and the local hooks need nothing beyond bash and
# python3 -- so installing the full requirements.txt spent 9s a run on
# packages no pre-commit-stage hook imports. Verified by running all
# three configs green in a venv holding pre-commit and nothing else.
# The pin is read out of requirements.txt rather than repeated here, so
# it cannot drift from what the dependency-freshness gate governs.
- name: Install pre-commit
run: |
pip install "$(grep '^pre-commit==' leetcode_coach/requirements.txt)"
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
# package jobs shared one key, raced to overwrite it with hook sets built
# from four different configs, and editing a per-package config did not
# invalidate it. The cache was therefore silently wrong rather than
# missing, and the hooks reinstalled from scratch on every run (38-49s).
- uses: actions/cache@v4
with:
path: ~/.cache/pre-commit
key: >-
pre-commit-leetcode-coach-${{ env.pythonLocation }}-${{
hashFiles('leetcode_coach/.pre-commit-config.yaml') }}
# Prefix fallback, or this fix would evaporate on the very
# event that motivated it: bumping a hook rev changes the
# exact key, and a total miss means reinstalling every hook
# again. pre-commit's store is keyed internally by hook
# repo+rev, so a superseded entry is ignored disk rather
# than a wrong hook, and cache still re-saves under the new
# exact key after a prefix-only hit.
restore-keys: pre-commit-leetcode-coach-${{ env.pythonLocation }}-
# mypy re-analyses the stdlib and every third-party stub from scratch
# without this, which measured 10s of the hook step in CI against 0.25s
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
# the config, not the sources: mypy's own cache metadata records the
# interpreter, the flag set and a hash per source file, so it re-checks
# what changed and cannot pass stale code. Deliberately NOT keyed on
# github.sha -- that re-saves 19MB x3 every commit and would evict the
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
# the expensive part, the dependency analysis, which is what costs 10s.
- uses: actions/cache@v4
with:
path: .mypy_cache
key: >-
mypy-leetcode-coach-${{ env.pythonLocation }}-${{
hashFiles('leetcode_coach/.pre-commit-config.yaml') }}
restore-keys: mypy-leetcode-coach-${{ env.pythonLocation }}-
- name: Run the hooks
run: >-
pre-commit run --config leetcode_coach/.pre-commit-config.yaml
--all-files --show-diff-on-failure

View File

@ -1,34 +0,0 @@
name: leetcode-coach tests
on:
push:
branches: [main]
# The workflow's own file counts as leetcode_coach: editing only this file would
# otherwise trigger nothing, so a fix to the job itself could never prove
# itself.
paths: ['leetcode_coach/**', '.github/workflows/leetcode-coach-tests.yml']
pull_request:
branches: [main]
paths: ['leetcode_coach/**', '.github/workflows/leetcode-coach-tests.yml']
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: leetcode_coach
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"
cache: pip
cache-dependency-path: leetcode_coach/requirements.txt
- name: Install dependencies
run: pip install -r requirements.txt && pip install --no-deps -e .
- name: Run tests with coverage
run: python -m pytest -v

View File

@ -1,33 +0,0 @@
name: md-naming
# The markdown namespaces (README / CLAUDE* / DOCS*, no TODO*), enforced on push as well as at commit time.
# The local pre-commit hook can be skipped (a fresh clone that never ran
# `pre-commit install` has no hooks at all), so the convention needs a second
# gate that no one can forget to install.
#
# Deliberately a SEPARATE workflow from any release job: a badly named doc
# should fail lint, not block publishing an artifact.
on:
push:
branches: [main]
pull_request:
workflow_dispatch:
jobs:
md-naming:
name: markdown naming convention
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
# The gate and its exemption list live in the shared utils repo so every
# repo enforces the same namespaces. Checking it out beats vendoring a
# copy here, which is exactly how a gate and a survey start disagreeing.
- name: Check out the shared gate
uses: actions/checkout@v4
with:
repository: kuhyx/utils
path: .utils
- name: Enforce the naming convention
run: bash .utils/scripts/check_md_naming.sh --all

View File

@ -1,31 +0,0 @@
name: music_theory tests
on:
push:
branches: [main]
# The workflow's own file counts as music_theory: editing only this file used
# to trigger nothing, so a fix to the job itself could never prove itself.
paths: ['music_theory/**', '.github/workflows/music-theory-tests.yml']
pull_request:
branches: [main]
paths: ['music_theory/**', '.github/workflows/music-theory-tests.yml']
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: music_theory
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"
- name: Install dependencies
run: pip install -r requirements.txt
- name: Run tests with coverage
run: python -m pytest -v --cov=music_theory

View File

@ -1,73 +0,0 @@
name: omniroute_offload pre-commit
on:
push:
branches: [main]
# The workflow's own file counts as omniroute_offload: editing only this file would
# otherwise trigger nothing, so a fix to the job itself could never prove
# itself.
paths: ['omniroute_offload/**', '.github/workflows/omniroute_offload-precommit.yml']
pull_request:
branches: [main]
paths: ['omniroute_offload/**', '.github/workflows/omniroute_offload-precommit.yml']
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.x"
# Only pre-commit itself. Every linting hook runs in its own isolated
# env -- ruff, mypy, pylint, bandit and codespell each come from their
# own repo+rev, and the local hooks need nothing beyond bash and
# python3 -- so installing the full requirements.txt spent 9s a run on
# packages no pre-commit-stage hook imports. Verified by running all
# three configs green in a venv holding pre-commit and nothing else.
# The pin is read out of requirements.txt rather than repeated here, so
# it cannot drift from what the dependency-freshness gate governs.
- name: Install pre-commit
run: |
pip install "$(grep '^pre-commit==' omniroute_offload/requirements.txt)"
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
# package jobs shared one key, raced to overwrite it with hook sets built
# from four different configs, and editing a per-package config did not
# invalidate it. The cache was therefore silently wrong rather than
# missing, and the hooks reinstalled from scratch on every run (38-49s).
- uses: actions/cache@v4
with:
path: ~/.cache/pre-commit
key: >-
pre-commit-omniroute_offload-${{ env.pythonLocation }}-${{
hashFiles('omniroute_offload/.pre-commit-config.yaml') }}
# Prefix fallback, or this fix would evaporate on the very
# event that motivated it: bumping a hook rev changes the
# exact key, and a total miss means reinstalling every hook
# again. pre-commit's store is keyed internally by hook
# repo+rev, so a superseded entry is ignored disk rather
# than a wrong hook, and cache still re-saves under the new
# exact key after a prefix-only hit.
restore-keys: pre-commit-omniroute_offload-${{ env.pythonLocation }}-
# mypy re-analyses the stdlib and every third-party stub from scratch
# without this, which measured 10s of the hook step in CI against 0.25s
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
# the config, not the sources: mypy's own cache metadata records the
# interpreter, the flag set and a hash per source file, so it re-checks
# what changed and cannot pass stale code. Deliberately NOT keyed on
# github.sha -- that re-saves 19MB x3 every commit and would evict the
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
# the expensive part, the dependency analysis, which is what costs 10s.
- uses: actions/cache@v4
with:
path: .mypy_cache
key: >-
mypy-omniroute_offload-${{ env.pythonLocation }}-${{
hashFiles('omniroute_offload/.pre-commit-config.yaml') }}
restore-keys: mypy-omniroute_offload-${{ env.pythonLocation }}-
- name: Run the hooks
run: >-
pre-commit run --config omniroute_offload/.pre-commit-config.yaml
--all-files --show-diff-on-failure

View File

@ -1,38 +0,0 @@
name: palette drift
# Deliberately NOT path-filtered. Every other workflow here scopes itself to
# one package, which is right for tests but wrong for this check: drift is a
# disagreement *between* packages, so editing design_system alone has to be
# able to fail it. A `paths:` filter would let exactly the edit that causes
# drift be the one that skips the check.
on:
push:
branches: [main]
pull_request:
branches: [main]
jobs:
palette:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.x"
# Compares tokens.md (prose freeze), web_ui/src/tokens.css,
# design_system tokens.dart and gatelock's LockConfig. Fails on a hex
# that disagrees across stacks, and on a colour token that no stack's
# map accounts for.
- name: One palette across all four token sources
run: python3 unified-design-system/scripts/palette_check.py
- name: Categorical ramp still passes its constraints
run: python3 unified-design-system/scripts/ramp_check.py
# Spacing, radius, type, shadow and motion: palette_check.py parses
# hex only, so these scales were unguarded until structural_check.py.
- name: Structural scales agree across all token sources
run: python3 unified-design-system/scripts/structural_check.py

View File

@ -1,19 +0,0 @@
name: repo-contract
# utils owns the shared gate, so it runs its own copy directly (no shim, no
# second checkout). Other repos use repo_contract/templates/repo-contract.yml.
on:
push:
branches: [main]
pull_request:
workflow_dispatch:
jobs:
repo-contract:
name: repo contract
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Enforce the repo contract
run: bash scripts/check_repo_contract.sh

View File

@ -1,82 +0,0 @@
name: shared gate tests
# The gates in this repo are enforced in ~40 other repos, so a bug here is a
# bug everywhere at once. Until now `md_naming` had a test suite that no
# workflow ran; `dep_freshness` ships with one from day one.
#
# The dependency-freshness suite never touches the network -- every registry
# answer is a fixture, because the gate's worst failure mode (accepting a
# pre-release as "latest stable") fails GREEN and only a fixture can pin it.
on:
push:
branches: [main]
pull_request:
workflow_dispatch:
jobs:
shared-gate-tests:
name: gate unit tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"
- name: Install test dependencies
run: python -m pip install --upgrade pytest pytest-cov pyyaml packaging
- name: dep_freshness — 100% branch coverage on the gate itself
run: |
PYTHONPATH=. python -m pytest dep_freshness/tests -q \
--cov=dep_freshness --cov-branch \
--cov-report=term-missing \
--cov-fail-under=99
- name: md_naming
run: PYTHONPATH=. python -m pytest md_naming/tests -q
# Whole package at 100% branch: the gate is enforced in every repo and
# now reads a per-repo `.file-length-exempt`, whose fail-closed rule is
# exactly the kind of branch a partial suite would leave green by accident.
- name: file_length — 100% branch coverage on the gate itself
run: |
PYTHONPATH=. python -m pytest file_length/tests -q \
--cov=file_length --cov-branch \
--cov-report=term-missing \
--cov-fail-under=100
- name: repo_contract — 100% branch coverage on the gate itself
run: |
PYTHONPATH=. python -m pytest repo_contract/tests -q \
--cov=repo_contract --cov-branch \
--cov-report=term-missing \
--cov-fail-under=100
# Every web repo's pre-commit and CI run this checker against its dist/,
# so a false green here ships an un-agent-ready site fleet-wide.
- name: agent_ready — 100% branch coverage on the checker
run: |
PYTHONPATH=. python -m pytest agent_ready/tests -q \
--cov=agent_ready --cov-branch \
--cov-report=term-missing \
--cov-fail-under=100
- name: coverage_gaps — 100% branch coverage on the lister
run: |
PYTHONPATH=. python -m pytest coverage_gaps/tests -q \
--cov=coverage_gaps --cov-branch \
--cov-report=term-missing \
--cov-fail-under=100
# gate_stamp.sh is the pre-push gate in every repo. Its tests drive real
# pre-commit in throwaway repos, and the hook-id helper runs under
# /usr/bin/python3 (the deployment interpreter), hence apt's PyYAML.
- name: gate_stamp (step / full --list)
run: |
sudo apt-get update -q && sudo apt-get install -y python3-yaml
python -m pip install --upgrade pre-commit
PYTHONPATH=. python -m pytest scripts/tests -q

View File

@ -1,73 +0,0 @@
name: sickdays pre-commit
on:
push:
branches: [main]
# The workflow's own file counts as sickdays: editing only this file would
# otherwise trigger nothing, so a fix to the job itself could never prove
# itself.
paths: ['sickdays/**', '.github/workflows/sickdays-precommit.yml']
pull_request:
branches: [main]
paths: ['sickdays/**', '.github/workflows/sickdays-precommit.yml']
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.x"
# Only pre-commit itself. Every linting hook runs in its own isolated
# env -- ruff, mypy, pylint, bandit and codespell each come from their
# own repo+rev, and the local hooks need nothing beyond bash and
# python3 -- so installing the full requirements.txt spent 9s a run on
# packages no pre-commit-stage hook imports. Verified by running all
# three configs green in a venv holding pre-commit and nothing else.
# The pin is read out of requirements.txt rather than repeated here, so
# it cannot drift from what the dependency-freshness gate governs.
- name: Install pre-commit
run: |
pip install "$(grep '^pre-commit==' sickdays/requirements.txt)"
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
# package jobs shared one key, raced to overwrite it with hook sets built
# from four different configs, and editing a per-package config did not
# invalidate it. The cache was therefore silently wrong rather than
# missing, and the hooks reinstalled from scratch on every run (38-49s).
- uses: actions/cache@v4
with:
path: ~/.cache/pre-commit
key: >-
pre-commit-sickdays-${{ env.pythonLocation }}-${{
hashFiles('sickdays/.pre-commit-config.yaml') }}
# Prefix fallback, or this fix would evaporate on the very
# event that motivated it: bumping a hook rev changes the
# exact key, and a total miss means reinstalling every hook
# again. pre-commit's store is keyed internally by hook
# repo+rev, so a superseded entry is ignored disk rather
# than a wrong hook, and cache still re-saves under the new
# exact key after a prefix-only hit.
restore-keys: pre-commit-sickdays-${{ env.pythonLocation }}-
# mypy re-analyses the stdlib and every third-party stub from scratch
# without this, which measured 10s of the hook step in CI against 0.25s
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
# the config, not the sources: mypy's own cache metadata records the
# interpreter, the flag set and a hash per source file, so it re-checks
# what changed and cannot pass stale code. Deliberately NOT keyed on
# github.sha -- that re-saves 19MB x3 every commit and would evict the
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
# the expensive part, the dependency analysis, which is what costs 10s.
- uses: actions/cache@v4
with:
path: .mypy_cache
key: >-
mypy-sickdays-${{ env.pythonLocation }}-${{
hashFiles('sickdays/.pre-commit-config.yaml') }}
restore-keys: mypy-sickdays-${{ env.pythonLocation }}-
- name: Run the hooks
run: >-
pre-commit run --config sickdays/.pre-commit-config.yaml
--all-files --show-diff-on-failure

View File

@ -1,49 +0,0 @@
name: sickdays tests
on:
push:
branches: [main]
# The workflow's own file counts as sickdays: editing only this file would
# otherwise trigger nothing, so a fix to the job itself could never prove
# itself.
paths: ['sickdays/**', '.github/workflows/sickdays-tests.yml']
pull_request:
branches: [main]
paths: ['sickdays/**', '.github/workflows/sickdays-tests.yml']
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: sickdays
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"
cache: pip
cache-dependency-path: sickdays/requirements.txt
# crdt-sync and freedays are siblings in this repo, so they install from
# the checkout rather than from a tag -- CI must test what is on this
# branch, not what was last released. Deliberately NOT gatelock:
# importing it pulls in tkinter, and this job has no display precisely
# because sickdays must work without one.
- name: Install the sibling packages
run: pip install ../crdt-sync && pip install --no-deps ../freedays
- name: Install dependencies
run: pip install -r requirements.txt && pip install --no-deps -e .
# The library draws nothing, but sickdays.gui (0.3.0) does, and its
# tests drive real Tk: option names and Checkbutton/variable wiring are
# Tk's own behaviour. Headless they skip, taking coverage with them --
# so the runner gets a virtual display, as gatelock's does.
- name: Install a virtual X display
run: sudo apt-get update && sudo apt-get install -y xvfb
- name: Run tests with coverage
run: xvfb-run -a -s "-screen 0 1600x1200x24" python -m pytest -v

View File

@ -1,73 +0,0 @@
name: sql-drills pre-commit
on:
push:
branches: [main]
# The workflow's own file counts as sql_drills: editing only this file would
# otherwise trigger nothing, so a fix to the job itself could never prove
# itself.
paths: ['sql_drills/**', '.github/workflows/sql-drills-precommit.yml']
pull_request:
branches: [main]
paths: ['sql_drills/**', '.github/workflows/sql-drills-precommit.yml']
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.x"
# Only pre-commit itself. Every linting hook runs in its own isolated
# env -- ruff, mypy, pylint, bandit and codespell each come from their
# own repo+rev, and the local hooks need nothing beyond bash and
# python3 -- so installing the full requirements.txt spent 9s a run on
# packages no pre-commit-stage hook imports. Verified by running all
# three configs green in a venv holding pre-commit and nothing else.
# The pin is read out of requirements.txt rather than repeated here, so
# it cannot drift from what the dependency-freshness gate governs.
- name: Install pre-commit
run: |
pip install "$(grep '^pre-commit==' sql_drills/requirements.txt)"
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
# package jobs shared one key, raced to overwrite it with hook sets built
# from four different configs, and editing a per-package config did not
# invalidate it. The cache was therefore silently wrong rather than
# missing, and the hooks reinstalled from scratch on every run (38-49s).
- uses: actions/cache@v4
with:
path: ~/.cache/pre-commit
key: >-
pre-commit-sql-drills-${{ env.pythonLocation }}-${{
hashFiles('sql_drills/.pre-commit-config.yaml') }}
# Prefix fallback, or this fix would evaporate on the very
# event that motivated it: bumping a hook rev changes the
# exact key, and a total miss means reinstalling every hook
# again. pre-commit's store is keyed internally by hook
# repo+rev, so a superseded entry is ignored disk rather
# than a wrong hook, and cache still re-saves under the new
# exact key after a prefix-only hit.
restore-keys: pre-commit-sql-drills-${{ env.pythonLocation }}-
# mypy re-analyses the stdlib and every third-party stub from scratch
# without this, which measured 10s of the hook step in CI against 0.25s
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
# the config, not the sources: mypy's own cache metadata records the
# interpreter, the flag set and a hash per source file, so it re-checks
# what changed and cannot pass stale code. Deliberately NOT keyed on
# github.sha -- that re-saves 19MB x3 every commit and would evict the
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
# the expensive part, the dependency analysis, which is what costs 10s.
- uses: actions/cache@v4
with:
path: .mypy_cache
key: >-
mypy-sql-drills-${{ env.pythonLocation }}-${{
hashFiles('sql_drills/.pre-commit-config.yaml') }}
restore-keys: mypy-sql-drills-${{ env.pythonLocation }}-
- name: Run the hooks
run: >-
pre-commit run --config sql_drills/.pre-commit-config.yaml
--all-files --show-diff-on-failure

View File

@ -1,34 +0,0 @@
name: sql-drills tests
on:
push:
branches: [main]
# The workflow's own file counts as sql_drills: editing only this file would
# otherwise trigger nothing, so a fix to the job itself could never prove
# itself.
paths: ['sql_drills/**', '.github/workflows/sql-drills-tests.yml']
pull_request:
branches: [main]
paths: ['sql_drills/**', '.github/workflows/sql-drills-tests.yml']
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: sql_drills
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"
cache: pip
cache-dependency-path: sql_drills/requirements.txt
- name: Install dependencies
run: pip install -r requirements.txt && pip install --no-deps -e .
- name: Run tests with coverage
run: python -m pytest -v

View File

@ -1,36 +0,0 @@
name: sync_settings_ui tests
on:
push:
branches: [main]
paths: ['sync_settings_ui/**']
pull_request:
branches: [main]
paths: ['sync_settings_ui/**']
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: sync_settings_ui
steps:
- uses: actions/checkout@v4
- name: Set up Flutter
uses: subosito/flutter-action@v2
with:
channel: stable
- name: Install dependencies
run: flutter pub get
- name: Analyze
run: flutter analyze
- name: Run tests with coverage
run: flutter test --coverage
- name: Enforce 100% line coverage
run: scripts/check_coverage.sh coverage/lcov.info 100

View File

@ -1,37 +0,0 @@
name: ts_config tests
on:
push:
branches: [main]
paths: ['ts_config/**']
pull_request:
branches: [main]
paths: ['ts_config/**']
workflow_dispatch:
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: ts_config
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 12
- uses: actions/setup-node@v4
with:
node-version: "24.22.1"
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Lint (the preset lints itself)
run: pnpm lint
- name: Fixture + factory tests with the 100% coverage gate
run: pnpm coverage

View File

@ -1,57 +0,0 @@
name: ts_core tests
on:
push:
branches: [main]
paths: ['ts_core/**', 'ts_config/**']
pull_request:
branches: [main]
paths: ['ts_core/**', 'ts_config/**']
workflow_dispatch:
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: ts_core
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 12
- uses: actions/setup-node@v4
with:
node-version: "24.22.1"
# The preset is a `link:../ts_config` dependency. pnpm links the
# directory but installs nothing inside it, and a flat config imports
# its plugins from the preset's own node_modules -- so without this
# step the lint dies on "Cannot find package '@eslint/js'".
- name: Install the shared lint preset's dependencies
run: pnpm install --frozen-lockfile
working-directory: ts_config
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Lint
run: pnpm lint
- name: Run tests with the 100% coverage gate
run: pnpm coverage
# The tag ships a prebuilt dist/ because Vite does not transpile
# dependencies. A committed dist/ that does not match src/ would ship
# stale code to every consumer with no error anywhere, so rebuild and
# fail if the result differs from what is checked in.
- name: dist/ is in sync with src/
run: |
pnpm build
if ! git diff --exit-code -- dist; then
echo "::error::dist/ is stale — run 'pnpm build' in ts_core and commit the result."
exit 1
fi

View File

@ -1,61 +0,0 @@
name: web_ui tests
on:
push:
branches: [main]
paths: ['web_ui/**', 'ts_config/**']
pull_request:
branches: [main]
paths: ['web_ui/**', 'ts_config/**']
workflow_dispatch:
jobs:
test:
runs-on: ubuntu-latest
defaults:
run:
working-directory: web_ui
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 12
- uses: actions/setup-node@v4
with:
node-version: "24.22.1"
# The preset is a `link:../ts_config` dependency. pnpm links the
# directory but installs nothing inside it, and a flat config imports
# its plugins from the preset's own node_modules -- so without this
# step the lint dies on "Cannot find package '@eslint/js'".
- name: Install the shared lint preset's dependencies
run: pnpm install --frozen-lockfile
working-directory: ts_config
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Lint
run: pnpm lint
- name: Run tests with the 100% coverage gate
run: pnpm coverage
# The tag ships a prebuilt dist/ because Vite does not transpile
# dependencies. A committed dist/ that does not match src/ would ship
# stale code to every consumer with no error anywhere, so rebuild and
# fail if the result differs from what is checked in.
- name: dist/ is in sync with src/
run: |
pnpm build
if ! git diff --exit-code -- dist; then
echo "::error::dist/ is stale — run 'pnpm build' in web_ui and commit the result."
exit 1
fi
- name: Categorical ramp still passes its constraints
working-directory: .
run: python3 unified-design-system/scripts/ramp_check.py

25
.gitignore vendored
View File

@ -1,25 +0,0 @@
# Python build/test artifacts, across every subproject in this monorepo.
__pycache__/
*.py[cod]
*.egg-info/
build/
dist/
.venv/
.mypy_cache/
.ruff_cache/
.pytest_cache/
# Coverage output. Each subproject writes these next to its own pyproject.toml,
# and a run from the repo root drops a pair here too.
.coverage
.coverage.*
coverage.lcov
htmlcov/
# Firebase CLI debug exhaust. Written next to whichever directory the CLI was
# invoked from (crdt-sync/ in practice), regenerated on every failed command,
# and never worth committing.
firebase-debug.log
firebase-debug.*.log
firestore-debug.log
database-debug.log

View File

@ -1,84 +0,0 @@
# Repo-wide gate for the shared 250-line file cap (refactor_claude_todo.md).
#
# A file that cannot be read in one piece forces re-reads and partial edits,
# which is the largest avoidable cost in an LLM-assisted workflow. A refactor
# without a gate silently regrows, so this is the part that makes it stick.
#
# Checks only the files in the commit, so an unrelated commit never breaks on
# a violation it did not introduce, and it FAILS (exit 1) rather than warning.
# No baseline and no allowlist -- those are suppressions.
#
# The subprojects (gatelock/, crdt-sync/, web_ui/, design_system/,
# sync_settings_ui/) each keep their own .pre-commit-config.yaml for their own
# linters, and carry this same hook so committing with one of those configs is
# gated too. This root config is what `pre-commit install` wires up, and it is
# the only one that covers the subprojects without a config of their own
# (guard-lib/, crdt_sync_dart/, file_length/, ...).
#
# The path is repo-relative on purpose. pre-commit always runs hooks from the
# git root, so this resolves the same locally and on a CI runner -- an absolute
# /home/... path exits 127 in Actions, which is how this was first written.
repos:
- repo: local
hooks:
- id: file-length-cap
name: file length <= 250 lines
entry: bash scripts/check_file_length.sh
language: system
pass_filenames: true
# Three markdown namespaces (README / CLAUDE* / DOCS*); TODO*.md is banned
# (tasks live in the todo app).
# Shared gate in ~/src/utils owns the namespaces and the exemption list, so
# one repo's idea of a valid name cannot drift from every other repo's.
# Since the 2026-09-11 reorganisation ~ holds eleven visible entries and
# home-tidy sweeps anything else to inbox/ within the hour. The migration's
# rewriter matched *literal* /home/kuhy/<name> strings, so paths assembled
# at run time survived it -- the todo wrapper exported to a dead ~/todo for
# a day while its MCP read ~/src/todo. This gate matches the shape, so it
# stays correct as repos come and go.
- id: home-path-layout
name: home-relative paths stay inside the ~ layout
entry: bash scripts/check_home_paths.sh
language: system
pass_filenames: true
- id: repo-contract
name: repo contract (CLAUDE.md Commands + test_changed)
entry: scripts/check_repo_contract.sh
language: system
pass_filenames: false
files: ^(CLAUDE\.md|AGENTS\.md|scripts/test_changed\.sh)$|(^|/)(AndroidManifest\.xml|build\.gradle(\.kts)?|strings\.xml|pubspec\.yaml|repo-contract\.toml|[^/]+\.desktop|assets/icon/[^/]+\.png)$
- id: md-naming
name: markdown naming convention
entry: scripts/check_md_naming.sh
language: system
files: \.(md|markdown)$
# Every dependency on its ecosystem's newest STABLE release, exact-pinned
# so the question is mechanically answerable at all.
#
# `files:` is what keeps this cheap: a commit touching only source makes
# ZERO network requests, and manifest commits mostly hit a 6h disk cache.
#
# Runs WITHOUT --strict on purpose. Offline it degrades to the cached
# answer and passes with a loud banner; `--no-verify` is banned here, so
# a hook that hard-failed on a flaky connection would leave no way to
# commit at all. CI, always online, is the strict authority.
#
# Scoped to manifests this branch changed (vs merge-base with main), so
# `--all-files` asks "did THIS change leave a stale pin?" and not "is
# every project in the monorepo current?". CI `--all --strict` keeps
# the repo-wide answer. See scripts/dep_freshness_changed.sh.
- id: dependency-freshness
name: dependencies on newest stable
entry: bash scripts/dep_freshness_changed.sh
language: system
pass_filenames: true
files: >-
(pubspec\.(yaml|lock)|pyproject\.toml|requirements.*\.txt|uv\.lock
|package\.json|package-lock\.json|pnpm-lock\.yaml
|Cargo\.(toml|lock)|go\.(mod|sum)
|\.fvmrc|\.nvmrc|\.python-version
|\.github/workflows/.*\.ya?ml)$

View File

@ -1,27 +0,0 @@
# Hooks other repos consume with `repo: https://github.com/kuhyx/utils`.
# All three are post-* stages, so each clone must install them (see
# todo_hooks/README.md). They never fail a git operation: the work has
# already happened, so server/gh trouble prints one warning and exits 0.
- id: todo-closer
name: close Todo trailer tasks
entry: scripts/todo_hooks.sh closer
language: script
stages: [post-commit]
always_run: true
pass_filenames: false
- id: todo-import-pr-merge
name: import PR Todo sections (pull)
entry: scripts/todo_hooks.sh import-pr post-merge
language: script
stages: [post-merge]
always_run: true
pass_filenames: false
- id: todo-import-pr-rewrite
name: import PR Todo sections (pull --rebase)
entry: scripts/todo_hooks.sh import-pr post-rewrite
language: script
stages: [post-rewrite]
always_run: true
pass_filenames: false

View File

@ -1,18 +0,0 @@
# utils
Monorepo of shared gates and libraries (dep_freshness, file_length, md_naming,
repo_contract, coverage_gaps, crdt-sync, gatelock, freedays, sickdays, earned_time, music_theory,
flutter_web_drive (headless Flutter-web E2E driver + MCP), ...).
Python packages with a root-level `<pkg>/tests` run with `PYTHONPATH=.`;
subprojects with their own `pyproject.toml` run via `scripts/run_subproject_tests.sh <dir>`.
`scripts/check_repo_contract.sh` is the REAL shared gate (not a shim): never
overwrite it with `repo_contract/templates/check_repo_contract.shim.sh`.
## Commands
- run: n/a: collection of gates and libraries, no single entry point
- test: `PYTHONPATH=. python -m pytest -q agent_ready/tests coverage_gaps/tests dep_freshness/tests file_length/tests flutter_web_drive/tests home_paths/tests md_naming/tests repo_contract/tests scripts/tests unified-design-system/tests`
- test-changed: `scripts/test_changed.sh`
- lint: `ruff check agent_ready coverage_gaps flutter_web_drive repo_contract`
- coverage: `PYTHONPATH=. python -m pytest -q agent_ready/tests flutter_web_drive/tests repo_contract/tests unified-design-system/tests --cov=agent_ready --cov=flutter_web_drive --cov=repo_contract --cov=conformance --cov=conformance_diff --cov-branch --cov-report=xml --cov-fail-under=100`
- coverage-gaps: `coverage-gaps coverage.xml`

View File

@ -18,45 +18,10 @@ dependencies:
crdt_sync:
git:
url: https://github.com/kuhyx/utils
ref: crdt_sync_dart-v0.10.0
ref: crdt_sync_dart-v0.2.0
path: crdt_sync_dart
```
## sync_settings_ui/
Shared Flutter "Sync settings" screen (pub package name: `sync_settings_ui`,
not published to pub.dev). Firebase sync (primary) plus an optional local
backup slot — no GitHub mirror UI. Pure UI + a `FirebaseSyncController`
service layer over injected closures (keystore/Google-plugin access stays
in each consuming app, exactly as before this package existed). Consumed by
`todo`, `diet_guard_app`, `workout_app`, `wake_alarm/phone_app`, and
`home_inventory`'s dedicated "Sync settings" route.
Consume via:
```yaml
dependencies:
sync_settings_ui:
git:
url: https://github.com/kuhyx/utils
ref: sync_settings_ui-v0.1.0
path: sync_settings_ui
```
## google_account_token/
Flutter plugin (Android only; other platforms return null): one-tap Google ID
token via `AccountManager`, replacing `google_sign_in` in the sync apps.
Consume via:
```yaml
dependencies:
google_account_token:
git:
url: https://github.com/kuhyx/utils
ref: google_account_token-v0.1.0
path: google_account_token
```
## crdt-sync/
Python CRDT sync library (package: `crdt_sync`, not published to PyPI). Same
@ -86,45 +51,3 @@ published to PyPI). Consumed via pip by `screen-locker`, `diet-guard`, and
```
gatelock @ git+https://github.com/kuhyx/utils@gatelock-v0.1.0#subdirectory=gatelock
```
## music_theory/
Music theory from first principles as typed Python (package: `music_theory`,
not published to PyPI): pitch, tuning, scales, chords, progressions, a numpy
synth and a `Recipe` → WAV composer with a `degrade` transform. Follows Luke
Haas's "Music Theory for Programmers" section by section; `python -m
music_theory demo all --out DIR` renders one WAV per section. Consumed by
`roadside-assistance` for its three world-state music beds.
Consume via:
```
music_theory @ git+https://github.com/kuhyx/utils@music-theory-v0.1.0#subdirectory=music_theory
```
## unified-design-system/
Not a package — the frozen design tokens (colors, spacing, type, radius,
shadow policy) and per-stack (Flutter/web/Tkinter) implementation patterns
that keep every one of kuhy's repos visually identical, plus annotated
component references (`components.html`, `button.html`). Read
[`unified-design-system/README.md`](unified-design-system/README.md) and
`DOCS-tokens.md` before touching any theme/CSS/style file in any repo. The
`unified-design-system` Claude Code skill is a thin pointer here, not a
duplicate.
## ts_config/
`@kuhyx/ts-config` — the one lint / TypeScript bar every kuhyx TypeScript
repo extends (pnpm only, like `ts_core` and `web_ui`). A strict
`tsconfig.base.json`, ESLint flat-config presets (`defineConfig()` stacking
base / react / vitest / playwright / layer `boundaries()`), a type-free
`eslint/fast` subset for editor hooks, and Knip / jscpd / Stryker bases.
Plain JS, no build step; the git tag is the release. Every plugin is a
`dependency` of the package so consumers list none. A fixture repo with one
planted violation per rule family proves each gate bites.
```jsonc
"devDependencies": {
"@kuhyx/ts-config": "github:kuhyx/utils#ts_config-v0.1.0&path:/ts_config"
}
```

View File

@ -1,97 +0,0 @@
# AgentReady spec notes
These notes record what the checker implements and where each rule comes from.
Fetched 2026-10-22.
## Sources
- AgentReady v2.0 (October 2026, MIT):
<https://github.com/agentready-org/standard/blob/main/spec/spec.md>,
canonical site <https://agentready.org>.
- AI Catalog (the format behind `AR-FIND-04`):
<https://github.com/Agent-Card/ai-catalog/blob/main/specification/ai-catalog.md>.
AgentReady defines no catalog schema of its own ("AgentReady does not
introduce its own bespoke manifest"), so the checker uses this one.
- llms.txt: the spec's informative "llms.txt shape" example, and <https://llmstxt.org/>.
## Requirements (quoted from spec.md)
| ID | Level | Requirement | Checker |
|---|---|---|---|
| AR-FIND-01 | MUST | "The site **MUST NOT** block agent and crawler access to public content, in `robots.txt` or at the network layer." | robots.txt is 200 `text/plain` and not HTML. It has a `User-agent: *` group, and no group has a blanket `Disallow: /`. In url mode the homepage returns 200 with no challenge page for ClaudeBot, GPTBot, Claude-SearchBot, OAI-SearchBot, Claude-User and ChatGPT-User. |
| AR-FIND-02 | SHOULD | "Answers agents are expected to cite **SHOULD** be published on one canonical, current, first-party docs page." | SKIP (manual) |
| AR-FIND-03 | SHOULD | "The site **SHOULD** list its pages in a `sitemap.xml` referenced from `robots.txt`." | robots.txt has a `Sitemap:` line. The sitemap is XML (by content type in url mode), and every `<loc>` is on the site's own host. |
| AR-FIND-04 | MAY | "...publish an AI Catalog at `/.well-known/ai-catalog.json` listing every agent-facing resource it ships ... linked from the homepage head with `<link rel="ai-catalog">` and an HTTP `Link` header." | The catalog is valid (see the schema below) and lists `/llms.txt`. The homepage has `<link rel="ai-catalog">`. In url mode the homepage also sends a `Link: <...>; rel="ai-catalog"` header. |
| AR-READ-01 | MUST | "The answer **MUST** be present in the initial HTML — server-rendered or prerendered, not injected by a single-page app after load." | The homepage and each sitemap page return 200 `text/html` with at least `--min-text` characters of visible body text. A page that is byte-identical to the homepage fails. An empty sitemap is a FAIL. |
| AR-READ-02 | MUST | "...accurate HTTP status codes: `404` (or a valid redirect) for missing paths ... never a `200` for a missing page" | url mode: two random `/agentready-probe-<hex>` paths must end, after redirects, on a 404 or 410. dist mode: a `404.html` with text must exist. |
| AR-READ-03 | SHOULD | "The homepage **SHOULD** be served as raw HTML with real links to docs, product, and pricing." | The homepage HTML has at least one `<a href>` to another page on the site. |
| AR-READ-04 | SHOULD | "Docs pages **SHOULD** answer the tasks users bring to agents." | SKIP (manual) |
| AR-READ-05 | SHOULD | "Code snippets **SHOULD** be fenced and language-tagged." | Every `<pre><code>` carries `language-*`, `lang-*` or `data-language`. Every fence in a markdown mirror has an info string. |
| AR-READ-06 | SHOULD | "Discovery files (`llms.txt`, `.well-known/*`, `openapi.json`, `AGENTS.md`) **SHOULD** be linked from content agents already visit" | `/llms.txt` and the catalog (plus `/openapi.json` with `--act`) exist and are linked from the homepage, by `<link>` or `<a>`. |
| AR-READ-07 | SHOULD | "The site **SHOULD** ship an `llms.txt`: a lean, described index of its key pages." | 200 `text/plain` or `text/markdown`, not HTML. It needs an H1 first, a `>` summary, and an `## H2` with a `- [name](url)` list; other prose is allowed. Every on-site link resolves. |
| AR-READ-08 | SHOULD | "Key pages **SHOULD** carry JSON-LD — the homepage above all" | The homepage has a parseable `application/ld+json` block with `@context`. |
| AR-READ-09 | MAY | "Pages **MAY** advertise a markdown mirror via `<link rel="alternate" type="text/markdown">` or an equivalent HTTP `Link:` header." | Every checked page advertises a mirror. The mirror is 200 `text/markdown` or `text/plain`, and is not HTML. |
| AR-READ-10 | n/a | "`AR-READ-10` is retired." | not emitted |
| AR-READ-11 | SHOULD | "Pricing **SHOULD** be published on a first-party page" | SKIP (n/a: no paid plans) |
| AR-ACT-01 | SHOULD | auth completable by an agent; "OAuth 2.0 with discoverable metadata under `/.well-known`" | `--act`: either `oauth-protected-resource` has `resource`, or `oauth-authorization-server` has `issuer`. |
| AR-ACT-02 | SHOULD | "A public API **SHOULD** be described by a machine-readable OpenAPI contract." | `--act`: `/openapi.json` has `openapi` and `paths`, and every operation has an `operationId`. |
| AR-ACT-03 | SHOULD | "...expose an MCP server over Streamable HTTP at `/mcp`." | `--act`, url mode: a POST `initialize` gets a JSON-RPC result, or a 401 with `WWW-Authenticate`. |
| AR-ACT-04 | MAY | "An MCP server **MAY** ship a server card" | `--act`: the catalog has an `application/mcp-server-card+json` entry. |
| AR-ACT-05/06/08/09 | MAY/SHOULD | MCP Apps, SDKs+CLI, payments, NLWeb `/ask` | `--act`: SKIP (manual) |
| AR-ACT-07 | SHOULD | "...register WebMCP tools on `document.modelContext`." | `--act`: `modelContext` appears in the homepage HTML or in its same-site scripts. |
| AR-ACT-10 | MUST | "...**MUST** expose at least one interface an agent can act through: an MCP server (`AR-ACT-03`) or a CLI (`AR-ACT-06`)." | `--act`: same verdict as the `/mcp` probe. |
## kuhy's choices that are stricter than the spec
- Every in-scope item is required, whatever its strength. That includes the
MAY items (FIND-04, READ-09).
- The spec says a site "**MAY** block a training crawler". kuhy chose to allow
training crawlers too, so a blanket `Disallow` for GPTBot, ClaudeBot or any
other agent is a FAIL.
- The act stage is off by default. Turn it on with `--act` only for sites with
user actions. The spec allows this: "a purely informational site conforms on
`AR-FIND-*` and `AR-READ-*` alone".
## AI Catalog schema (ai-catalog.md, specVersion 1.0)
- The media type is `application/ai-catalog+json`. When fetched by link
discovery, plain `application/json` is also acceptable.
- At the top level, the object MUST have:
- `specVersion`: a string in "Major.Minor" form, for example `"1.0"`.
- `entries`: an array, which may be empty.
- The top level MAY have `host`, `extensions` and `signature`.
- If `host` is present, it MUST have `displayName`. It MAY have
`identifier`, `documentationUrl` and `logoUrl`.
- Each entry MUST have:
- `identifier`: a URI or URN. The recommended form is
`urn:air:{publisher}:{namespace}:{name}`.
- `type`: a media type, for example `application/mcp-server-card+json`.
- **exactly one** of `url` or `data`.
- `displayName` is optional on entries (ai-catalog ADR-0016).
- Link discovery uses `Link: <...>; rel="ai-catalog"` or
`<link rel="ai-catalog" href="..." type="application/ai-catalog+json">`.
Minimal valid example:
```json
{"specVersion": "1.0",
"host": {"displayName": "kuhy"},
"entries": [{"identifier": "urn:air:kuhy.duckdns.org:docs:llms",
"type": "text/plain", "url": "https://kuhy.duckdns.org/llms.txt"}]}
```
Agentic Resource Discovery v0.91 (<https://agenticresourcediscovery.org/spec>)
renamed the path to `/.well-known/ard.json` and the relation to `rel="ard"`.
It keeps `ai-catalog.json` only as an optional predecessor. AgentReady v2.0
still names `ai-catalog.json`, so that is the path the checker tests. It does
not test `ard.json`.
## llms.txt shape (spec.md, informative)
```md
# Acme
> Acme ships a payments API for developers.
## Docs
- [Quickstart](https://example.com/docs/quickstart): first charge in one request
```

View File

@ -1,93 +0,0 @@
# agent_ready
This tool checks kuhy's websites against [AgentReady](https://agentready.org)
v2.0. It runs every find and read item, and the act items when you pass `--act`.
It is Python 3 standard library only, with no dependencies.
`DOCS-spec-notes.md` maps each `AR-*` ID to the check that tests it.
## Usage
```sh
~/src/utils/agent_ready/check.sh dist <build-dir> [--act] [--min-text N] [--max-pages N]
~/src/utils/agent_ready/check.sh url <base-url> [--act] [--min-text N] [--max-pages N] [--timeout S] [--budget S]
```
The output is one line per check:
```text
PASS AR-FIND-03 sitemap lists 12 pages
FAIL AR-READ-02 soft 404, unknown paths must return 404: /agentready-probe-1a2b -> HTTP 200
SKIP AR-READ-11 pricing page: n/a for sites without paid plans
```
| exit | meaning |
|---|---|
| 0 | every check passed (SKIPs do not fail) |
| 1 | at least one FAIL, at any strength: MUST, SHOULD or MAY |
| 2 | usage error (bad arguments, missing directory, not an http(s) URL) |
| 3 | network undeterminable: the homepage was unreachable, or a fetch timed out or hit the 45 s budget, and nothing failed |
## Pre-commit and CI (dist mode)
Point the checker at the build output after the build step. Dist mode checks
that each file exists and is well-formed:
- robots.txt
- sitemap.xml
- llms.txt
- the AI Catalog
- 404.html
- the text in every sitemap page's HTML
- each page's markdown mirror
```yaml
# .github/workflows/*.yml, after the build
- run: ~/src/utils/agent_ready/check.sh dist dist/
```
```yaml
# .pre-commit-config.yaml (local hook; build first, e.g. in a wrapper script)
- id: agent-ready
name: AgentReady (build output)
entry: bash -c 'pnpm build >/dev/null && ~/src/utils/agent_ready/check.sh dist dist'
language: system
pass_filenames: false
```
Dist mode cannot check HTTP behaviour: content types, the `Link` header, real
404 status, or responses to agent user-agents. Those checks print as SKIP, or
the check falls back to "404.html exists".
## Post-deploy (url mode)
Run this once the deploy is live:
```sh
~/src/utils/agent_ready/check.sh url https://kuhy.duckdns.org
```
Url mode checks status codes **and** content types. A server that answers
every path with `200 text/html` (an SPA fallback) fails on robots.txt, the
sitemap, llms.txt, the catalog and the 404 probe. A status-only check would
pass it. Url mode also checks that:
- `/agentready-probe-<random>` ends on a 404, after following redirects.
- No sitemap page is byte-identical to the homepage shell.
- The homepage answers 200 to ClaudeBot, GPTBot and the other agent
user-agents.
## Tests
```sh
~/.claude/scripts/capped.sh python -m pytest ~/src/utils/agent_ready/tests -q
```
The test fixtures are generated in `tmp_path` by `tests/sitegen.py`, so there
are no static mirror `.md` files to trip the markdown-naming gate. The good
and bad build trees are served by loopback `http.server` instances in four
shapes:
- compliant
- SPA fallback
- JSON 404 on every path
- 403 to GPTBot

View File

@ -1 +0,0 @@
"""AgentReady (agentready.org v2.0) compliance checker for kuhy's websites."""

View File

@ -1,188 +0,0 @@
"""Two backends behind one ``get(path)``: a build directory and a live site.
DirSite answers like a plain static host (index.html for ``/x/``, a 404 for
anything missing), so the find/read checks are written once for both modes.
"""
from __future__ import annotations
import http.client
import threading
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from pathlib import Path
from urllib.parse import unquote, urljoin, urlsplit
from agent_ready.report import Undeterminable
DEFAULT_UA = "Claude-User/1.0 (+agentready-check)"
MAX_BODY = 5 * 1024 * 1024
_EXT_MIME = {
".html": "text/html",
".md": "text/markdown",
".txt": "text/plain",
".xml": "application/xml",
".json": "application/json",
}
@dataclass(frozen=True)
class Response:
"""A fetched resource: final status, bare media type, headers, body."""
status: int
ctype: str
body: bytes
headers: dict[str, str] = field(default_factory=dict)
@property
def text(self) -> str:
"""Body decoded as UTF-8 (lenient)."""
return self.body.decode("utf-8", "replace")
class Site:
"""Common URL handling; subclasses implement ``_fetch``."""
live = False
def __init__(self, origin: str) -> None:
self.origin = origin.rstrip("/")
self._cache: dict[tuple[str, str], Response | Undeterminable] = {}
self._lock = threading.Lock()
def to_path(self, url: str, base: str = "/") -> str | None:
"""Site path for a URL or relative href; None when off-site."""
absolute = urljoin(self.origin + base, url.strip())
parts = urlsplit(absolute)
if (
parts.scheme not in ("http", "https")
or parts.netloc != urlsplit(self.origin).netloc
):
return None
return parts.path or "/"
def get(self, path: str, ua: str = DEFAULT_UA) -> Response:
"""Fetch ``path`` (memoized, including network failures)."""
key = (path, ua)
with self._lock:
hit = self._cache.get(key)
if hit is None:
try:
hit = self._fetch(path, ua)
except Undeterminable as exc:
hit = exc
with self._lock:
self._cache[key] = hit
if isinstance(hit, Undeterminable):
raise hit
return hit
def prefetch(self, paths: list[str], uas: tuple[str, ...] = (DEFAULT_UA,)) -> None:
"""Warm the cache for every (path, ua) pair; parallel for live sites."""
if not self.live:
return
pairs = [(p, ua) for p in dict.fromkeys(paths) for ua in uas]
with ThreadPoolExecutor(max_workers=8) as pool:
list(pool.map(lambda pair: self._safe_get(*pair), pairs))
def _safe_get(self, path: str, ua: str) -> None:
try:
self.get(path, ua)
except Undeterminable:
pass
def _fetch(self, path: str, ua: str) -> Response:
raise NotImplementedError
class DirSite(Site):
"""A build output directory served the way a static host serves it."""
def __init__(self, root: Path) -> None:
super().__init__("https://dist.invalid")
self.root = root.resolve()
def file_for(self, path: str) -> Path | None:
"""The file a static host would serve for ``path``."""
rel = unquote(path.split("?", 1)[0]).lstrip("/")
base = (self.root / rel).resolve()
if not base.is_relative_to(self.root):
return None
cands = (
[base / "index.html"]
if path.endswith("/") or not rel
else [base, base.with_name(base.name + ".html"), base / "index.html"]
)
return next((c for c in cands if c.is_file()), None)
def _fetch(self, path: str, ua: str) -> Response:
target = self.file_for(path)
if target is None:
fallback = self.root / "404.html"
body = fallback.read_bytes() if fallback.is_file() else b""
return Response(404, "text/html", body)
return Response(
200,
_EXT_MIME.get(target.suffix, "application/octet-stream"),
target.read_bytes(),
)
class HttpSite(Site):
"""A live site. Follows redirects; HTTP errors are responses, not raises."""
live = True
def __init__(self, base: str, timeout: float = 8.0, budget: float = 45.0) -> None:
parts = urlsplit(base)
super().__init__(f"{parts.scheme}://{parts.netloc}")
self.timeout = timeout
self.deadline = time.monotonic() + budget
def _remaining(self) -> float:
left = self.deadline - time.monotonic()
if left < 1:
raise Undeterminable("overall time budget used up")
return min(self.timeout, left)
def _fetch(self, path: str, ua: str) -> Response:
return self.request(path, ua)
def request(
self,
path: str,
ua: str = DEFAULT_UA,
data: bytes | None = None,
headers: dict[str, str] | None = None,
) -> Response:
"""Uncached GET (or POST when ``data`` is given)."""
req = urllib.request.Request(
self.origin + path,
data=data,
headers={"User-Agent": ua, "Accept": "*/*", **(headers or {})},
)
try:
with urllib.request.urlopen(req, timeout=self._remaining()) as resp:
return _response(resp.status, resp.headers, resp.read(MAX_BODY))
except urllib.error.HTTPError as exc:
return _response(exc.code, exc.headers, exc.read(MAX_BODY))
except (
urllib.error.URLError,
TimeoutError,
OSError,
http.client.HTTPException,
) as exc:
raise Undeterminable(f"{path}: {getattr(exc, 'reason', exc)}") from exc
def _response(status: int, headers: http.client.HTTPMessage, body: bytes) -> Response:
"""Normalise headers: lowercase keys, repeated headers joined by ', '."""
flat: dict[str, str] = {}
for key in set(headers.keys()):
flat[key.lower()] = ", ".join(headers.get_all(key) or [])
ctype = flat.get("content-type", "").split(";", 1)[0].strip().lower()
return Response(status, ctype, body, flat)

View File

@ -1,137 +0,0 @@
#!/usr/bin/env python3
"""AgentReady compliance checker (agentready.org spec v2.0).
check.py dist <build-dir> [--act] # pre-commit / CI: build output
check.py url <base-url> [--act] # post-deploy: the live site
Prints ``PASS|FAIL|SKIP AR-XXX-NN <msg>`` per check. Exit 0 all pass, 1 any
FAIL (every in-scope item is required, whatever its MUST/SHOULD/MAY
strength), 2 usage error, 3 network undeterminable.
"""
from __future__ import annotations
import argparse
import sys
from collections.abc import Callable
from pathlib import Path
from urllib.parse import urlsplit
if __package__ in (None, ""): # run as a plain script: make the package importable
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from agent_ready import checks_find, checks_read
from agent_ready.backends import DirSite, HttpSite, Site
from agent_ready.checks_act import run_act
from agent_ready.context import HOME, Ctx
from agent_ready.report import (
EXIT_NETWORK,
EXIT_USAGE,
Report,
Undeterminable,
)
# (AR id used when a network failure aborts the check, check function)
CHECKS: list[tuple[str, Callable[[Ctx], None]]] = [
("AR-FIND-01", checks_find.check_robots),
("AR-FIND-01", checks_find.check_agent_access),
("AR-FIND-03", checks_find.check_sitemap),
("AR-FIND-04", checks_find.check_catalog),
("AR-READ-01", checks_read.check_pages),
("AR-READ-02", checks_read.check_status_codes),
("AR-READ-03", checks_read.check_homepage),
("AR-READ-06", checks_read.check_discovery_links),
("AR-READ-07", checks_read.check_llms),
("AR-FIND-02", checks_read.skip_manual),
("AR-ACT-10", run_act),
]
def parse_args(argv: list[str] | None) -> argparse.Namespace:
"""Parse the command line (argparse exits 2 on usage errors)."""
parser = argparse.ArgumentParser(
prog="agent_ready", description=__doc__.split("\n\n")[0]
)
sub = parser.add_subparsers(dest="mode", required=True)
for mode, target in (
("dist", "build output directory"),
("url", "base URL, e.g. https://example.org"),
):
cmd = sub.add_parser(mode, help=f"check a {target}")
cmd.add_argument("target", help=target)
cmd.add_argument(
"--act",
action="store_true",
help="also check the act stage (OpenAPI, MCP, auth)",
)
cmd.add_argument(
"--min-text",
type=int,
default=100,
help="min visible chars per page (default 100)",
)
cmd.add_argument(
"--max-pages",
type=int,
default=40,
help="url mode: max pages to fetch (default 40; dist checks all)",
)
url = sub.choices["url"]
url.add_argument(
"--timeout",
type=float,
default=8.0,
help="per-request timeout in s (default 8)",
)
url.add_argument(
"--budget",
type=float,
default=45.0,
help="overall network budget in s (default 45)",
)
return parser.parse_args(argv)
def make_site(args: argparse.Namespace) -> Site | None:
"""Build the backend, or report a usage error and return None."""
if args.mode == "dist":
root = Path(args.target)
if not root.is_dir():
print(f"agent_ready: not a directory: {root}", file=sys.stderr)
return None
return DirSite(root)
parts = urlsplit(args.target)
if parts.scheme not in ("http", "https") or not parts.netloc:
print(f"agent_ready: not an http(s) URL: {args.target}", file=sys.stderr)
return None
return HttpSite(args.target, timeout=args.timeout, budget=args.budget)
def main(argv: list[str] | None = None) -> int:
"""Run every check and return the exit code."""
args = parse_args(argv)
site = make_site(args)
if site is None:
return EXIT_USAGE
report = Report(out=sys.stdout)
ctx = Ctx(
site=site,
report=report,
min_text=args.min_text,
max_pages=args.max_pages,
act=args.act,
)
try:
site.get(HOME)
except Undeterminable as exc:
report.skip(
"AR-FIND-01", f"undeterminable (network): homepage unreachable: {exc}"
)
return EXIT_NETWORK
for ar_id, check in CHECKS:
report.guard(ar_id, lambda check=check: check(ctx))
return report.exit_code()
if __name__ == "__main__":
sys.exit(main())

View File

@ -1,10 +0,0 @@
#!/bin/bash
# Thin wrapper: run the AgentReady checker from any cwd.
# check.sh dist <build-dir> [--act]
# check.sh url <base-url> [--act]
# Exit codes: 0 all pass, 1 any FAIL, 2 usage error, 3 network undeterminable.
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly HERE
PYTHONPATH="$(dirname "$HERE")${PYTHONPATH:+:$PYTHONPATH}" exec python3 -m agent_ready.check "$@"

View File

@ -1,156 +0,0 @@
"""Act stage: AR-ACT-01..10. Opt-in (``--act``): only for sites with user actions."""
from __future__ import annotations
import json
from agent_ready import formats
from agent_ready.backends import HttpSite
from agent_ready.context import CATALOG, Ctx
_INIT = {
"jsonrpc": "2.0",
"id": 1,
"method": "initialize",
"params": {
"protocolVersion": "2025-06-18",
"capabilities": {},
"clientInfo": {"name": "agentready-check", "version": "1"},
},
}
def _json(ctx: Ctx, path: str) -> object | None:
"""Parsed JSON at ``path`` when it is 200 and parses, else None."""
resp = ctx.site.get(path)
if resp.status != 200:
return None
try:
return json.loads(resp.body)
except ValueError:
return None
def check_auth(ctx: Ctx) -> None:
"""AR-ACT-01: OAuth metadata (RFC 9728 ``resource`` or RFC 8414 ``issuer``)."""
prm = _json(ctx, "/.well-known/oauth-protected-resource")
asm = _json(ctx, "/.well-known/oauth-authorization-server")
ok = (isinstance(prm, dict) and "resource" in prm) or (
isinstance(asm, dict) and "issuer" in asm
)
ctx.report.verdict(
"AR-ACT-01",
[]
if ok
else [
"no OAuth metadata under /.well-known (oauth-protected-resource / oauth-authorization-server)"
],
"OAuth metadata discoverable under /.well-known",
)
def check_openapi(ctx: Ctx) -> None:
"""AR-ACT-02: /openapi.json with paths and an operationId on every operation."""
doc = _json(ctx, "/openapi.json")
if not isinstance(doc, dict) or not isinstance(doc.get("openapi"), str):
ctx.report.fail("AR-ACT-02", "/openapi.json missing or not an OpenAPI document")
return
paths = doc.get("paths") if isinstance(doc.get("paths"), dict) else {}
ops = [
op
for item in paths.values()
if isinstance(item, dict)
for m, op in item.items()
if m in ("get", "post", "put", "patch", "delete")
]
missing = sum(
1 for op in ops if not (isinstance(op, dict) and op.get("operationId"))
)
problems = ["no operations in paths"] if not ops else []
if missing:
problems.append(f"{missing} operations lack operationId")
ctx.report.verdict(
"AR-ACT-02", problems, f"/openapi.json describes {len(ops)} operations"
)
def _mcp_ok(ctx: Ctx) -> bool | None:
"""True if /mcp answers initialize (or is OAuth-gated); None in dist mode."""
if not isinstance(ctx.site, HttpSite):
return None
headers = {
"Content-Type": "application/json",
"Accept": "application/json, text/event-stream",
}
resp = ctx.site.request("/mcp", data=json.dumps(_INIT).encode(), headers=headers)
if resp.status == 401:
return "www-authenticate" in resp.headers
return resp.status == 200 and '"jsonrpc"' in resp.text and '"result"' in resp.text
def check_mcp(ctx: Ctx) -> None:
"""AR-ACT-03 MCP at /mcp, AR-ACT-04 server card, AR-ACT-10 an interface exists."""
ok = _mcp_ok(ctx)
if ok is None:
ctx.report.skip("AR-ACT-03", "/mcp is a server endpoint: check in url mode")
ctx.report.skip("AR-ACT-10", "MCP/CLI interface: check in url mode")
else:
ctx.report.verdict(
"AR-ACT-03",
[]
if ok
else [
"/mcp does not answer an MCP initialize (nor 401 + WWW-Authenticate)"
],
"/mcp answers MCP initialize",
)
ctx.report.verdict(
"AR-ACT-10",
[]
if ok
else ["no MCP server at /mcp; a CLI (AR-ACT-06) cannot be auto-detected"],
"agents can act through /mcp",
)
cards = [
t
for t in formats.catalog_types(_json(ctx, CATALOG))
if t == "application/mcp-server-card+json"
]
ctx.report.verdict(
"AR-ACT-04",
[] if cards else ["AI Catalog lists no application/mcp-server-card+json entry"],
"AI Catalog lists an MCP server card",
)
def check_webmcp(ctx: Ctx) -> None:
"""AR-ACT-07: WebMCP tools registered on document.modelContext."""
resp, page = ctx.home()
sources = [resp.text]
for src in page.scripts[:10]:
path = ctx.site.to_path(src)
if path:
sources.append(ctx.site.get(path).text)
found = any("modelContext" in s for s in sources)
ctx.report.verdict(
"AR-ACT-07",
[] if found else ["no document.modelContext usage in homepage HTML/scripts"],
"homepage registers WebMCP tools",
)
def run_act(ctx: Ctx) -> None:
"""All act checks, or one SKIP line when the stage is off."""
if not ctx.act:
ctx.report.skip(
"AR-ACT-10", "act stage off (pass --act for sites with user actions)"
)
return
ctx.report.guard("AR-ACT-01", lambda: check_auth(ctx))
ctx.report.guard("AR-ACT-02", lambda: check_openapi(ctx))
ctx.report.guard("AR-ACT-03", lambda: check_mcp(ctx))
ctx.report.guard("AR-ACT-07", lambda: check_webmcp(ctx))
ctx.report.skip("AR-ACT-05", "MCP Apps ui:// view: manual review")
ctx.report.skip("AR-ACT-06", "SDKs on npm/PyPI and a CLI: manual review")
ctx.report.skip("AR-ACT-08", "agent payments: n/a unless the product charges")
ctx.report.skip("AR-ACT-09", "NLWeb /ask endpoint: manual review")

View File

@ -1,186 +0,0 @@
"""Find stage: AR-FIND-01..04 (robots.txt, agent access, sitemap, AI Catalog)."""
from __future__ import annotations
import json
from urllib.parse import urlsplit
from agent_ready import formats
from agent_ready.context import CATALOG, HOME, LLMS, Ctx, link_header_href
from agent_ready.report import summarize
# Training, search-index and user-initiated fetchers named in the spec's
# crawl-policy notes. Real UA strings for the two training crawlers.
AGENT_UAS = {
"ClaudeBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
"GPTBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.1; +https://openai.com/gptbot)",
"Claude-SearchBot": "Claude-SearchBot/1.0",
"OAI-SearchBot": "OAI-SearchBot/1.0; +https://openai.com/searchbot",
"Claude-User": "Claude-User/1.0",
"ChatGPT-User": "ChatGPT-User/1.0; +https://openai.com/bot",
}
_CHALLENGE = (
"just a moment",
"cf-chl",
"challenge-platform",
"attention required",
"captcha",
)
XML_TYPES = ("application/xml", "text/xml", "application/rss+xml")
def check_robots(ctx: Ctx) -> None:
"""AR-FIND-01: robots.txt exists, is plain text and blocks no agent."""
resp = ctx.site.get("/robots.txt")
if resp.status != 200:
ctx.report.fail("AR-FIND-01", f"/robots.txt: HTTP {resp.status}")
return
problems = ctx.type_problem(resp, ("text/plain",), "/robots.txt")
if formats.looks_like_html(resp.text):
ctx.report.fail(
"AR-FIND-01",
"; ".join([*problems, "/robots.txt body is an HTML page (SPA fallback?)"]),
)
return
ctx.robots_text = resp.text
groups = formats.parse_robots(resp.text)
if not formats.has_wildcard_group(groups):
problems.append("no 'User-agent: *' group (spec: one rule for all agents)")
blocked = formats.blocked_agents(groups)
if blocked:
problems.append(f"blanket Disallow for: {', '.join(blocked)}")
ctx.report.verdict(
"AR-FIND-01",
problems,
"robots.txt is plain text, has a * group and blocks no agent",
)
def check_agent_access(ctx: Ctx) -> None:
"""AR-FIND-01 (bot protection): homepage is 200 for each agent UA (url mode)."""
if not ctx.live:
ctx.report.skip("AR-FIND-01", "agent user-agent fetches need url mode")
return
base, _ = ctx.home()
ctx.site.prefetch([HOME], tuple(AGENT_UAS.values()))
bad = []
for name, ua in AGENT_UAS.items():
resp = ctx.site.get(HOME, ua)
lowered = resp.text[:20000].lower()
if resp.status != 200:
bad.append(f"{name} HTTP {resp.status}")
elif resp.body != base.body and any(m in lowered for m in _CHALLENGE):
bad.append(f"{name} got a challenge page")
ctx.report.verdict("AR-FIND-01", bad, f"homepage 200 for {', '.join(AGENT_UAS)}")
def _sitemap_paths(ctx: Ctx) -> list[str]:
"""Sitemap locations from robots.txt (also fixes the dist-mode origin)."""
urls = formats.robots_sitemaps(ctx.robots_text or "")
if urls and not ctx.live:
parts = urlsplit(urls[0])
ctx.site.origin = f"{parts.scheme}://{parts.netloc}"
paths = [p for u in urls if (p := ctx.site.to_path(u))]
if ctx.robots_text is None:
ctx.report.fail("AR-FIND-03", "no valid robots.txt to reference the sitemap")
elif not urls:
ctx.report.fail("AR-FIND-03", "robots.txt has no 'Sitemap:' line")
elif not paths:
ctx.report.fail(
"AR-FIND-03", f"robots.txt Sitemap is on another host: {urls[0]}"
)
else:
ctx.report.ok("AR-FIND-03", f"robots.txt references {summarize(urls)}")
return paths or ["/sitemap.xml"]
def _read_sitemap(ctx: Ctx, path: str, problems: list[str]) -> tuple[str, list[str]]:
"""Fetch and parse one sitemap; record problems; return (kind, locs)."""
resp = ctx.site.get(path)
if resp.status != 200:
problems.append(f"{path}: HTTP {resp.status}")
return "", []
problems.extend(ctx.type_problem(resp, XML_TYPES, path))
try:
return formats.parse_sitemap(resp.body)
except ValueError as exc:
problems.append(f"{path}: {exc}")
return "", []
def check_sitemap(ctx: Ctx) -> None:
"""AR-FIND-03: sitemap referenced from robots.txt, valid XML, on-site <loc>s."""
problems: list[str] = []
locs: list[str] = []
for path in _sitemap_paths(ctx)[:5]:
kind, found = _read_sitemap(ctx, path, problems)
if kind == "sitemapindex":
for child in [p for u in found[:10] if (p := ctx.site.to_path(u))]:
locs += _read_sitemap(ctx, child, problems)[1]
else:
locs += found
if locs and not ctx.live and ctx.site.origin.endswith("dist.invalid"):
parts = urlsplit(locs[0])
ctx.site.origin = f"{parts.scheme}://{parts.netloc}"
pages = [p for loc in locs if (p := ctx.site.to_path(loc))]
if len(pages) < len(locs):
problems.append(
f"{len(locs) - len(pages)} <loc> URLs are not on {ctx.site.origin}"
)
if not locs and not problems:
problems.append("sitemap has no <loc> entries")
ctx.sitemap_pages = list(dict.fromkeys(pages))
ctx.report.verdict(
"AR-FIND-03", problems, f"sitemap lists {len(ctx.sitemap_pages)} pages"
)
def check_catalog(ctx: Ctx) -> None:
"""AR-FIND-04: valid AI Catalog, lists llms.txt, linked from homepage head."""
resp = ctx.site.get(CATALOG)
problems: list[str] = []
if resp.status != 200:
problems.append(f"{CATALOG}: HTTP {resp.status}")
else:
problems += ctx.type_problem(
resp, ("application/ai-catalog+json", "application/json"), CATALOG
)
try:
doc = json.loads(resp.body)
except ValueError:
doc = None
problems.append(f"{CATALOG} is not JSON")
if doc is not None:
problems += formats.catalog_problems(doc)
listed = {ctx.site.to_path(u) for u in formats.catalog_urls(doc)}
if ctx.exists(LLMS) and LLMS not in listed:
problems.append("catalog has no entry for /llms.txt")
ctx.report.verdict("AR-FIND-04", problems, f"{CATALOG} is a valid AI Catalog")
home, scan = ctx.home()
href = scan.link_href("ai-catalog")
ok = href is not None and ctx.site.to_path(href) == CATALOG
ctx.report.verdict(
"AR-FIND-04",
[]
if ok
else [
'homepage head lacks <link rel="ai-catalog" href="/.well-known/ai-catalog.json">'
],
"homepage head links the catalog",
)
if not ctx.live:
ctx.report.skip(
"AR-FIND-04", "HTTP Link header can only be checked in url mode"
)
return
header = link_header_href(home, "ai-catalog")
ok = header is not None and ctx.site.to_path(header) == CATALOG
ctx.report.verdict(
"AR-FIND-04",
[]
if ok
else [
'homepage response lacks HTTP header Link: <...ai-catalog.json>; rel="ai-catalog"'
],
"homepage sends the ai-catalog Link header",
)

View File

@ -1,212 +0,0 @@
"""Read stage: AR-READ-01..09 and 11 (AR-READ-10 is retired in spec v2.0)."""
from __future__ import annotations
import secrets
from agent_ready import formats
from agent_ready.context import CATALOG, HOME, LLMS, TEXT_TYPES, Ctx, link_header_href
from agent_ready.htmlscan import scan
from agent_ready.report import summarize
def _pages(ctx: Ctx) -> list[str]:
"""Homepage plus sitemap pages, capped. An empty sitemap is a READ-01 FAIL."""
if not ctx.sitemap_pages:
ctx.report.fail(
"AR-READ-01", "sitemap yielded no pages; only the homepage was checked"
)
every = list(dict.fromkeys([HOME, *ctx.sitemap_pages]))
pages = ctx.cap(every)
if len(pages) < len(every):
ctx.report.skip(
"AR-READ-01",
f"checked {len(pages)} of {len(every)} pages (--max-pages); rest unchecked",
)
ctx.site.prefetch(pages)
return pages
def _mirror_href(ctx: Ctx, path: str) -> str | None:
"""Markdown mirror advertised by <link rel=alternate type=text/markdown> or Link header."""
resp, page = ctx.page(path)
href = page.link_href("alternate", "text/markdown")
if href is None and ctx.live:
href = link_header_href(resp, "alternate", "text/markdown")
return href
def check_pages(ctx: Ctx) -> None:
"""AR-READ-01 text in initial HTML, AR-READ-05 tagged code, AR-READ-09 mirrors."""
pages = _pages(ctx)
home, _ = ctx.home()
no_text: list[str] = []
untagged: list[str] = []
blocks = 0
for path in pages:
resp, page = ctx.page(path)
if resp.status != 200:
no_text.append(f"{path} (HTTP {resp.status})")
continue
if ctx.type_problem(resp, ("text/html",), path):
no_text.append(f"{path} ({resp.ctype or 'no content-type'})")
elif path != HOME and resp.body == home.body:
no_text.append(f"{path} (byte-identical to the homepage shell)")
elif len(page.text) < ctx.min_text:
no_text.append(f"{path} ({len(page.text)} chars of text < {ctx.min_text})")
blocks += page.code_blocks
if page.untagged_code:
untagged.append(f"{path} ({page.untagged_code} untagged <pre><code>)")
ctx.report.verdict(
"AR-READ-01",
["answer not in initial HTML: " + summarize(no_text)] if no_text else [],
f"{len(pages)} pages carry >= {ctx.min_text} chars of text without JS",
)
md_untagged = _check_mirrors(ctx, pages)
untagged += md_untagged
ctx.report.verdict(
"AR-READ-05",
["code without a language tag: " + summarize(untagged)] if untagged else [],
f"{blocks} HTML code blocks (plus mirror fences) all language-tagged",
)
def _check_mirrors(ctx: Ctx, pages: list[str]) -> list[str]:
"""AR-READ-09 for every page; returns mirrors with untagged fences (READ-05)."""
bad: list[str] = []
untagged: list[str] = []
targets: dict[str, str] = {}
for path in pages:
if ctx.site.get(path).status != 200:
continue
href = _mirror_href(ctx, path)
target = ctx.site.to_path(href, path) if href else None
if target is None:
bad.append(f"{path} (no text/markdown alternate link)")
else:
targets[path] = target
ctx.site.prefetch(list(targets.values()))
for path, target in targets.items():
resp = ctx.site.get(target)
if resp.status != 200:
bad.append(f"{target} (HTTP {resp.status})")
elif (
ctx.type_problem(resp, TEXT_TYPES, target)
or formats.looks_like_html(resp.text)
or not resp.text.strip()
):
bad.append(f"{target} (not markdown: {resp.ctype or 'empty'})")
elif n := formats.untagged_fences(resp.text):
untagged.append(f"{target} ({n} untagged fences)")
ctx.report.verdict(
"AR-READ-09",
["markdown mirror missing/broken: " + summarize(bad)] if bad else [],
f"{len(targets)} pages advertise a working markdown mirror",
)
return untagged
def check_status_codes(ctx: Ctx) -> None:
"""AR-READ-02: unknown paths are a real 404 (url) / a 404.html exists (dist)."""
if not ctx.live:
resp = ctx.site.get(f"/agentready-probe-{secrets.token_hex(6)}")
text = scan(resp.text).text if resp.body else ""
ok = resp.status == 404 and len(text) >= 10
ctx.report.verdict(
"AR-READ-02",
[] if ok else ["no 404.html with real text in the build output"],
"404.html exists for unknown paths",
)
return
token = secrets.token_hex(6)
probes = [f"/agentready-probe-{token}", f"/agentready-probe-{token}/page.html"]
ctx.site.prefetch(probes)
bad = [
f"{p} -> HTTP {s}"
for p in probes
if (s := ctx.site.get(p).status) not in (404, 410)
]
ctx.report.verdict(
"AR-READ-02",
["soft 404, unknown paths must return 404: " + ", ".join(bad)] if bad else [],
"unknown paths return 404",
)
def check_homepage(ctx: Ctx) -> None:
"""AR-READ-03 real links on the homepage, AR-READ-08 JSON-LD on it."""
resp, page = ctx.home()
links = {
p
for a in page.anchors
if not a.startswith(("#", "javascript:", "mailto:"))
and (p := ctx.site.to_path(a))
and p != HOME
}
ok = resp.status == 200 and bool(links)
ctx.report.verdict(
"AR-READ-03",
[] if ok else ["homepage HTML has no <a href> links to other pages"],
f"homepage links {len(links)} pages in raw HTML",
)
problems = []
if not page.jsonld_ok:
problems.append(
'homepage has no <script type="application/ld+json"> with @context'
)
if page.jsonld_bad:
problems.append(
f"{page.jsonld_bad} JSON-LD blocks fail to parse or lack @context"
)
ctx.report.verdict(
"AR-READ-08", problems, f"homepage carries {page.jsonld_ok} JSON-LD blocks"
)
def check_discovery_links(ctx: Ctx) -> None:
"""AR-READ-06: discovery files are linked from the homepage."""
_, page = ctx.home()
linked = {
ctx.site.to_path(h)
for h in [*page.anchors, *(href for _, _, href in page.links)]
}
files = [LLMS, CATALOG] + (["/openapi.json"] if ctx.act else [])
for path in files:
problems = []
if not ctx.exists(path):
problems.append(f"{path} does not exist")
if path not in linked:
problems.append(f"{path} is not linked from the homepage (<link> or <a>)")
ctx.report.verdict(
"AR-READ-06", problems, f"{path} exists and is linked from the homepage"
)
def check_llms(ctx: Ctx) -> None:
"""AR-READ-07: llms.txt in the spec shape, every on-site link resolving."""
resp = ctx.site.get(LLMS)
if resp.status != 200:
ctx.report.fail("AR-READ-07", f"{LLMS}: HTTP {resp.status}")
return
problems = ctx.type_problem(resp, TEXT_TYPES, LLMS) + formats.llms_problems(
resp.text
)
if not formats.looks_like_html(resp.text):
paths = [
p for u in formats.llms_links(resp.text) if (p := ctx.site.to_path(u, LLMS))
]
paths = ctx.cap(paths)
ctx.site.prefetch(paths)
broken = [p for p in paths if ctx.site.get(p).status != 200]
if broken:
problems.append("broken links: " + summarize(broken))
ctx.report.verdict("AR-READ-07", problems, f"{LLMS} has H1, summary and link lists")
def skip_manual(ctx: Ctx) -> None:
"""Items with no mechanical check (reported, never silently dropped)."""
ctx.report.skip(
"AR-FIND-02", "first-party docs page answering user questions: manual review"
)
ctx.report.skip("AR-READ-04", "docs pages answer users' tasks: manual review")
ctx.report.skip("AR-READ-11", "pricing page: n/a for sites without paid plans")

View File

@ -1,85 +0,0 @@
"""Shared state for one checker run (site, report, options, discoveries)."""
from __future__ import annotations
import re
from dataclasses import dataclass, field
from agent_ready.backends import Response, Site
from agent_ready.formats import looks_like_html
from agent_ready.htmlscan import PageScan, scan
from agent_ready.report import Report
HOME = "/"
LLMS = "/llms.txt"
CATALOG = "/.well-known/ai-catalog.json"
TEXT_TYPES = ("text/plain", "text/markdown")
@dataclass
class Ctx:
"""Everything a check needs; filled in as checks discover the site."""
site: Site
report: Report
min_text: int = 100
max_pages: int = 40
act: bool = False
robots_text: str | None = None
sitemap_pages: list[str] = field(default_factory=list)
_scans: dict[str, PageScan] = field(default_factory=dict)
@property
def live(self) -> bool:
"""True in url mode (headers and content types are real)."""
return self.site.live
def cap(self, items: list[str]) -> list[str]:
"""Apply --max-pages in url mode only; dist mode is free, so check everything."""
return items[: self.max_pages] if self.live else items
def page(self, path: str) -> tuple[Response, PageScan]:
"""Fetch and scan an HTML page (scan memoized)."""
resp = self.site.get(path)
if path not in self._scans:
self._scans[path] = scan(resp.text) if resp.status == 200 else PageScan()
return resp, self._scans[path]
def home(self) -> tuple[Response, PageScan]:
"""The homepage and its scan."""
return self.page(HOME)
def type_problem(
self, resp: Response, allowed: tuple[str, ...], what: str
) -> list[str]:
"""Content-type problem for live responses (dist mode has none)."""
if (
not self.live
or resp.ctype in allowed
or any(a.endswith("+xml") and resp.ctype.endswith("+xml") for a in allowed)
):
return []
return [
f"{what} served as {resp.ctype or 'no content-type'}, want {' or '.join(allowed)}"
]
def exists(self, path: str) -> bool:
"""True when ``path`` is 200 and, for a non-page file, not an HTML fallback."""
resp = self.site.get(path)
is_page = path.endswith(("/", ".html"))
return resp.status == 200 and (is_page or not looks_like_html(resp.text))
def link_header_href(resp: Response, rel: str, mime: str | None = None) -> str | None:
"""href of the first RFC 8288 ``Link`` header value with ``rel`` (and type)."""
for part in re.split(r",\s*(?=<)", resp.headers.get("link", "")):
match = re.match(r"\s*<([^>]*)>(.*)", part)
if not match:
continue
params = match.group(2).lower()
rels = re.search(r'rel="?([^";]+)"?', params)
if not rels or rel not in rels.group(1).split():
continue
if mime is None or re.search(rf'type="?{re.escape(mime)}"?', params):
return match.group(1)
return None

View File

@ -1,213 +0,0 @@
"""Validators for the file formats AgentReady points at.
robots.txt (RFC 9309), sitemap.xml (sitemaps.org), llms.txt (agentready
spec "llms.txt shape" / llmstxt.org) and the AI Catalog
(github.com/Agent-Card/ai-catalog, specification/ai-catalog.md).
"""
from __future__ import annotations
import re
import xml.etree.ElementTree as ET
from dataclasses import dataclass, field
_LINK_ITEM = re.compile(r"^\s*[-*]\s+\[[^\]]+\]\(([^)\s]+)\)")
_FENCE = re.compile(r"^\s{0,3}(```+|~~~+)\s*(\S*)")
_SPEC_VERSION = re.compile(r"^\d+\.\d+$")
def looks_like_html(text: str) -> bool:
"""True when a supposedly plain-text/XML/JSON body is an HTML page."""
head = text.lstrip()[:200].lower()
return head.startswith(("<!doctype html", "<html")) or "<head" in head
# --------------------------------------------------------------- robots.txt
@dataclass
class RobotsGroup:
"""One ``User-agent`` group and its rules, in file order."""
agents: list[str] = field(default_factory=list)
rules: list[tuple[str, str]] = field(default_factory=list)
def parse_robots(text: str) -> list[RobotsGroup]:
"""Parse robots.txt into groups (consecutive User-agent lines share one)."""
groups: list[RobotsGroup] = []
current: RobotsGroup | None = None
for raw in text.splitlines():
line = raw.split("#", 1)[0].strip()
if ":" not in line:
continue
key, value = (part.strip() for part in line.split(":", 1))
key = key.lower()
if key == "user-agent":
if current is None or current.rules:
current = RobotsGroup()
groups.append(current)
current.agents.append(value.lower())
elif key in ("allow", "disallow") and current is not None:
current.rules.append((key, value))
return groups
def has_wildcard_group(groups: list[RobotsGroup]) -> bool:
"""True when a ``User-agent: *`` group exists (one rule for all agents)."""
return any("*" in g.agents for g in groups)
def blocked_agents(groups: list[RobotsGroup]) -> list[str]:
"""Agents whose group disallows the whole site (``/`` or ``/*``).
An empty ``Disallow:`` allows everything; an ``Allow: /`` in the same
group wins over the blanket disallow (longest-match tie goes to allow).
"""
blocked: list[str] = []
for group in groups:
disallow_all = any(k == "disallow" and v in ("/", "/*") for k, v in group.rules)
allow_all = any(k == "allow" and v in ("/", "/*") for k, v in group.rules)
if disallow_all and not allow_all:
blocked.extend(group.agents)
return blocked
def robots_sitemaps(text: str) -> list[str]:
"""Return every ``Sitemap:`` URL."""
out = []
for raw in text.splitlines():
key, _, value = raw.partition(":")
if key.strip().lower() == "sitemap" and value.strip():
out.append(value.strip())
return out
# -------------------------------------------------------------- sitemap.xml
def parse_sitemap(body: bytes) -> tuple[str, list[str]]:
"""Return (root kind ``urlset``/``sitemapindex``, list of <loc> values).
Raises ValueError for anything that is not a sitemap document.
"""
try:
root = ET.fromstring(body)
except ET.ParseError as exc:
raise ValueError(f"not XML: {exc}") from exc
kind = root.tag.rsplit("}", 1)[-1]
if kind not in ("urlset", "sitemapindex"):
raise ValueError(f"root element <{kind}> is not <urlset>/<sitemapindex>")
locs = [
(el.text or "").strip()
for el in root.iter()
if el.tag.rsplit("}", 1)[-1] == "loc"
]
return kind, [loc for loc in locs if loc]
# ----------------------------------------------------------------- llms.txt
def llms_problems(text: str) -> list[str]:
"""Check llms.txt shape: H1 first, blockquote summary, H2 link lists."""
if looks_like_html(text):
return ["body is an HTML page, not llms.txt markdown"]
lines = [ln for ln in text.splitlines() if ln.strip()]
problems = []
if not lines or not re.match(r"^# \S", lines[0]):
problems.append("first line must be an H1 title ('# Name')")
if not any(ln.startswith(">") for ln in lines[:4]):
problems.append("missing blockquote summary ('> ...') after the H1")
in_h2 = False
links = 0
for ln in lines:
if ln.startswith("## "):
in_h2 = True
elif in_h2 and _LINK_ITEM.match(ln):
links += 1
if links == 0:
problems.append("no '## Section' with a '- [name](url): notes' link list")
return problems
def llms_links(text: str) -> list[str]:
"""Return every link-list URL in llms.txt, in order."""
return [m.group(1) for ln in text.splitlines() if (m := _LINK_ITEM.match(ln))]
# --------------------------------------------------------------- ai-catalog
def catalog_problems(doc: object) -> list[str]:
"""Validate an AI Catalog (Level 1 + Host Info if present)."""
if not isinstance(doc, dict):
return ["top level must be a JSON object"]
problems = []
version = doc.get("specVersion")
if not isinstance(version, str):
problems.append("missing string specVersion")
elif not _SPEC_VERSION.match(version):
problems.append(f"specVersion {version!r} is not 'Major.Minor'")
host = doc.get("host")
if host is not None and not (
isinstance(host, dict) and isinstance(host.get("displayName"), str)
):
problems.append("host.displayName (string) is required when host is present")
entries = doc.get("entries")
if not isinstance(entries, list):
return [*problems, "missing entries array"]
for i, entry in enumerate(entries):
problems.extend(f"entries[{i}]: {p}" for p in _entry_problems(entry))
return problems
def _entry_problems(entry: object) -> list[str]:
"""Problems with one Catalog Entry."""
if not isinstance(entry, dict):
return ["must be an object"]
problems = [
f"missing string {k}"
for k in ("identifier", "type")
if not isinstance(entry.get(k), str)
]
if ("url" in entry) == ("data" in entry):
problems.append("must carry exactly one of url/data")
return problems
def catalog_urls(doc: object) -> list[str]:
"""Every entry ``url`` in a catalog (empty for invalid docs)."""
if not isinstance(doc, dict) or not isinstance(doc.get("entries"), list):
return []
return [
e["url"]
for e in doc["entries"]
if isinstance(e, dict) and isinstance(e.get("url"), str)
]
def catalog_types(doc: object) -> list[str]:
"""Every entry ``type`` in a catalog."""
if not isinstance(doc, dict) or not isinstance(doc.get("entries"), list):
return []
return [
e["type"]
for e in doc["entries"]
if isinstance(e, dict) and isinstance(e.get("type"), str)
]
# ----------------------------------------------------------------- markdown
def untagged_fences(text: str) -> int:
"""Count opening code fences with no language info string."""
count = 0
open_fence = ""
for line in text.splitlines():
match = _FENCE.match(line)
if not match:
continue
marker, info = match.groups()
if open_fence:
if (
marker[0] == open_fence[0]
and len(marker) >= len(open_fence)
and not info
):
open_fence = ""
continue
open_fence = marker
count += not info
return count

View File

@ -1,121 +0,0 @@
"""One-pass HTML scan: visible body text, links, JSON-LD and code blocks."""
from __future__ import annotations
import json
import re
from dataclasses import dataclass, field
from html.parser import HTMLParser
_HIDDEN = {"script", "style", "noscript", "template", "svg", "head", "title"}
_LANG = re.compile(r"\b(language|lang)-\S+")
@dataclass
class PageScan:
"""What an agent that does not run JavaScript sees in a page."""
text: str = ""
links: list[tuple[set[str], str, str]] = field(default_factory=list)
anchors: list[str] = field(default_factory=list)
scripts: list[str] = field(default_factory=list)
jsonld_ok: int = 0
jsonld_bad: int = 0
code_blocks: int = 0
untagged_code: int = 0
def link_href(self, rel: str, mime: str | None = None) -> str | None:
"""First ``<link>`` href with ``rel`` (and ``type`` if given)."""
for rels, typ, href in self.links:
if rel in rels and (mime is None or typ == mime):
return href
return None
def _tagged(attrs: dict[str, str]) -> bool:
"""True when an element carries a code-language marker."""
return bool(_LANG.search(attrs.get("class", ""))) or bool(
attrs.get("data-language") or attrs.get("data-lang")
)
class _Scanner(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.out = PageScan()
self._hidden: list[str] = []
self._text: list[str] = []
self._pre: list[list[bool]] = [] # [pre_tagged, code_seen]
self._ld: list[str] | None = None
def handle_starttag(
self, tag: str, attr_list: list[tuple[str, str | None]]
) -> None:
attrs = {k: v or "" for k, v in attr_list}
if tag == "body": # </head> is optional in HTML; <body> always ends it
self._hidden = [h for h in self._hidden if h not in ("head", "title")]
if tag == "link" and attrs.get("href"):
rels = set(attrs.get("rel", "").lower().split())
self.out.links.append((rels, attrs.get("type", "").lower(), attrs["href"]))
elif tag == "a" and attrs.get("href"):
self.out.anchors.append(attrs["href"])
elif tag == "script":
if attrs.get("src"):
self.out.scripts.append(attrs["src"])
if attrs.get("type", "").lower() == "application/ld+json":
self._ld = []
elif tag == "pre":
self._pre.append([_tagged(attrs), False])
elif tag == "code" and self._pre and not self._pre[-1][1]:
self._pre[-1][1] = True
self.out.code_blocks += 1
self.out.untagged_code += not (self._pre[-1][0] or _tagged(attrs))
if tag in _HIDDEN:
self._hidden.append(tag)
def handle_endtag(self, tag: str) -> None:
if tag == "script" and self._ld is not None:
self._finish_ld("".join(self._ld))
self._ld = None
if tag == "pre" and self._pre:
self._pre.pop()
if tag in _HIDDEN and tag in self._hidden:
while self._hidden and self._hidden.pop() != tag:
pass
def handle_data(self, data: str) -> None:
if self._ld is not None:
self._ld.append(data)
elif not self._hidden:
self._text.append(data)
def _finish_ld(self, raw: str) -> None:
try:
doc = json.loads(raw)
except ValueError:
self.out.jsonld_bad += 1
return
nodes = doc if isinstance(doc, list) else [doc]
ok = (
all(
isinstance(n, dict) and ("@context" in n or "@graph" in n)
for n in nodes
)
and nodes
)
if ok:
self.out.jsonld_ok += 1
else:
self.out.jsonld_bad += 1
def result(self) -> PageScan:
self.out.text = re.sub(r"\s+", " ", " ".join(self._text)).strip()
return self.out
def scan(html: str) -> PageScan:
"""Scan an HTML document."""
parser = _Scanner()
parser.feed(html)
parser.close()
return parser.result()

View File

@ -1,73 +0,0 @@
"""Result lines (``PASS|FAIL|SKIP AR-XXX-NN msg``) and the exit-code policy."""
from __future__ import annotations
import sys
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import TextIO
EXIT_OK, EXIT_FAIL, EXIT_USAGE, EXIT_NETWORK = 0, 1, 2, 3
class Undeterminable(Exception):
"""A network-level failure (refused, timeout, deadline): no verdict."""
@dataclass
class Report:
"""Collects check lines and decides the exit code.
Every in-scope item is required for kuhy's sites, so a FAIL of any
strength (MUST/SHOULD/MAY) fails the run. SKIP never fails it.
"""
out: TextIO = field(default_factory=lambda: sys.stdout)
fails: int = 0
undetermined: int = 0
def emit(self, status: str, ar_id: str, msg: str) -> None:
"""Print one result line."""
if status == "FAIL":
self.fails += 1
print(f"{status} {ar_id} {msg}", file=self.out, flush=True)
def ok(self, ar_id: str, msg: str) -> None:
"""Emit PASS."""
self.emit("PASS", ar_id, msg)
def fail(self, ar_id: str, msg: str) -> None:
"""Emit FAIL."""
self.emit("FAIL", ar_id, msg)
def skip(self, ar_id: str, msg: str) -> None:
"""Emit SKIP."""
self.emit("SKIP", ar_id, msg)
def verdict(self, ar_id: str, problems: list[str], pass_msg: str) -> bool:
"""FAIL with the problems joined, or PASS with ``pass_msg``."""
if problems:
self.fail(ar_id, "; ".join(problems))
return False
self.ok(ar_id, pass_msg)
return True
def guard(self, ar_id: str, fn: Callable[[], object]) -> None:
"""Run a check; a network failure becomes an undeterminable SKIP."""
try:
fn()
except Undeterminable as exc:
self.undetermined += 1
self.skip(ar_id, f"undeterminable (network): {exc}")
def exit_code(self) -> int:
"""0 all pass, 1 any fail, 3 nothing failed but something undeterminable."""
if self.fails:
return EXIT_FAIL
return EXIT_NETWORK if self.undetermined else EXIT_OK
def summarize(items: list[str], limit: int = 4) -> str:
"""Join up to ``limit`` items and note how many more there were."""
head = ", ".join(items[:limit])
return head + (f" (+{len(items) - limit} more)" if len(items) > limit else "")

View File

@ -1,136 +0,0 @@
"""Fixtures: generated site trees and loopback HTTP servers that serve them."""
from __future__ import annotations
import socket
import threading
from collections.abc import Callable, Iterator
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import pytest
from agent_ready.tests import sitegen
_MIME = {
".html": "text/html; charset=utf-8",
".md": "text/markdown; charset=utf-8",
".txt": "text/plain; charset=utf-8",
".xml": "application/xml",
".json": "application/json",
}
def _resolve(root: Path, path: str) -> Path | None:
"""Map a URL path to a file the way a static host does."""
rel = path.split("?", 1)[0].lstrip("/")
cand = root / rel
if path.endswith("/"):
cand = cand / "index.html"
return cand if cand.is_file() else None
def make_handler(root: Path, mode: str, block_ua: str) -> type[BaseHTTPRequestHandler]:
"""Build a handler class.
``mode`` is ``good`` (real 404s), ``spa`` (200 index.html for every path,
the kuhy.duckdns.org trap) or ``json404`` (every non-root path is a JSON
404, the popes.kuhy.duckdns.org shape).
"""
class Handler(BaseHTTPRequestHandler):
def log_message(self, *args: object) -> None: # silence test output
return
def _send(
self,
status: int,
ctype: str,
body: bytes,
extra: dict[str, str] | None = None,
) -> None:
self.send_response(status)
self.send_header("Content-Type", ctype)
self.send_header("Content-Length", str(len(body)))
for key, value in (extra or {}).items():
self.send_header(key, value)
self.end_headers()
self.wfile.write(body)
def do_GET(self) -> None:
if block_ua and block_ua in self.headers.get("User-Agent", ""):
self._send(403, "text/html", b"<html>Just a moment...</html>")
return
origin = f"http://{self.headers['Host']}"
if mode == "spa" or (mode == "json404" and self.path == "/"):
body = (root / "index.html").read_bytes()
self._send(200, "text/html; charset=utf-8", body)
return
target = None if mode == "json404" else _resolve(root, self.path)
if target is None:
if mode == "json404":
self._send(404, "application/json", b'{"detail":"Not Found"}')
else:
self._send(404, "text/html", (root / "404.html").read_bytes())
return
if target.name == "ai-catalog.json":
ctype = "application/ai-catalog+json"
else:
ctype = _MIME.get(target.suffix, "application/octet-stream")
body = target.read_bytes().replace(sitegen.ORIGIN.encode(), origin.encode())
extra = {}
if self.path == "/":
extra["Link"] = '</.well-known/ai-catalog.json>; rel="ai-catalog"'
self._send(200, ctype, body, extra)
def do_POST(self) -> None:
"""Minimal MCP: answer initialize at /mcp in good mode only."""
self.rfile.read(int(self.headers.get("Content-Length", 0)))
if mode == "good" and self.path == "/mcp":
self._send(
200, "application/json", b'{"jsonrpc":"2.0","id":1,"result":{}}'
)
else:
self._send(200, "text/html", (root / "index.html").read_bytes())
return Handler
@pytest.fixture
def good_dir(tmp_path: Path) -> Path:
"""A compliant build output."""
return sitegen.build_good(tmp_path / "good")
@pytest.fixture
def bad_dir(tmp_path: Path) -> Path:
"""A build output failing every find/read item."""
return sitegen.build_bad(tmp_path / "bad")
@pytest.fixture
def serve() -> Iterator[Callable[..., str]]:
"""Start loopback servers on demand; return their base URL."""
servers: list[ThreadingHTTPServer] = []
def start(root: Path, mode: str = "good", block_ua: str = "") -> str:
server = ThreadingHTTPServer(
("127.0.0.1", 0), make_handler(root, mode, block_ua)
)
threading.Thread(target=server.serve_forever, daemon=True).start()
servers.append(server)
return f"http://127.0.0.1:{server.server_address[1]}"
yield start
for server in servers:
server.shutdown()
server.server_close()
@pytest.fixture
def dead_url() -> str:
"""A loopback URL with nothing listening (connection refused)."""
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
port = sock.getsockname()[1]
return f"http://127.0.0.1:{port}"

View File

@ -1,80 +0,0 @@
"""In-memory sites and CLI helpers for tests that need exact responses."""
from __future__ import annotations
import io
import pytest
from agent_ready.backends import DEFAULT_UA, HttpSite, Response, Site
from agent_ready.check import main
from agent_ready.context import Ctx
from agent_ready.report import Report
HTML = "text/html"
_MISSING = Response(404, HTML, b"")
def resp(body: str, ctype: str = HTML, status: int = 200, **headers: str) -> Response:
"""Build a Response; ``headers`` keys are already lowercase names."""
return Response(status, ctype, body.encode(), dict(headers))
class FakeSite(Site):
"""A dist-like (``live=False``) site answering from a dict."""
def __init__(self, pages: dict[str, Response]) -> None:
super().__init__("https://fake.test")
self.pages = pages
def _fetch(self, path: str, ua: str) -> Response:
return self.pages.get(path, _MISSING)
class FakeHttp(HttpSite):
"""A live site with no sockets: per-path (and per-UA) canned responses."""
def __init__(
self,
pages: dict[str, Response],
ua_pages: dict[tuple[str, str], Response] | None = None,
mcp: Response | None = None,
) -> None:
super().__init__("https://fake.test")
self.pages = pages
self.ua_pages = ua_pages or {}
self.mcp = mcp or _MISSING
def _fetch(self, path: str, ua: str) -> Response:
return self.ua_pages.get((path, ua)) or self.pages.get(path, _MISSING)
def request(
self,
path: str,
ua: str = DEFAULT_UA,
data: bytes | None = None,
headers: dict[str, str] | None = None,
) -> Response:
"""The canned MCP answer, whatever the request."""
return self.mcp
def make_ctx(site: Site, *, act: bool = False) -> tuple[Ctx, io.StringIO]:
"""A Ctx writing to a StringIO; returns it with that buffer."""
out = io.StringIO()
return Ctx(site=site, report=Report(out=out), act=act), out
def run_main(capsys: pytest.CaptureFixture[str], *argv: str) -> tuple[int, list[str]]:
"""Run the CLI in-process; return exit code and output lines."""
code = main(list(argv))
return code, capsys.readouterr().out.splitlines()
def lines_for(lines: list[str], ar_id: str, status: str = "") -> list[str]:
"""Output lines for ``ar_id`` (optionally only one status)."""
return [
ln
for ln in lines
if ln.split()[1] == ar_id and (not status or ln.startswith(status))
]

View File

@ -1,153 +0,0 @@
"""Build good and bad site trees in a temp dir (no static fixture files).
Generated at test time on purpose: mirror files such as ``index.md`` would
otherwise trip the repo's markdown-naming gate.
"""
from __future__ import annotations
import json
from pathlib import Path
ORIGIN = "https://example.test"
_TEXT = (
"Dice roller for tabletop games. Roll any combination of polyhedral dice, "
"keep the highest, and read the full breakdown of every throw. This text "
"is in the initial HTML so an agent that never runs JavaScript can read it."
)
_LD = json.dumps(
{
"@context": "https://schema.org",
"@type": "WebSite",
"name": "Dice",
"url": ORIGIN,
}
)
CATALOG = {
"specVersion": "1.0",
"host": {"displayName": "kuhy", "identifier": "did:web:example.test"},
"entries": [
{
"identifier": "urn:air:example.test:docs:llms",
"displayName": "llms.txt",
"type": "text/plain",
"url": f"{ORIGIN}/llms.txt",
}
],
}
LLMS = f"""# Dice
> A dice roller for tabletop games.
Plain prose is allowed between sections.
## Pages
- [Home]({ORIGIN}/): what the roller does
- [About]({ORIGIN}/about/): who made it
"""
ROBOTS = f"""# robots.txt
User-agent: *
Disallow: /search/
Sitemap: {ORIGIN}/sitemap.xml
"""
SITEMAP = f"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url><loc>{ORIGIN}/</loc></url>
<url><loc>{ORIGIN}/about/</loc></url>
</urlset>
"""
def page(title: str, mirror: str, *, home: bool = False) -> str:
"""Return a server-rendered HTML page with every read-stage feature."""
head = [
f"<title>{title}</title>",
f'<link rel="alternate" type="text/markdown" href="{mirror}">',
]
if home:
head += [
'<link rel="ai-catalog" href="/.well-known/ai-catalog.json" type="application/ai-catalog+json">',
'<link rel="alternate" type="text/plain" href="/llms.txt">',
f'<script type="application/ld+json">{_LD}</script>',
]
return (
'<!doctype html><html lang="en"><head>' + "".join(head) + "</head><body>"
f"<h1>{title}</h1><p>{_TEXT}</p>"
'<pre><code class="language-python">print("d20")</code></pre>'
'<nav><a href="/">Home</a> <a href="/about/">About</a> <a href="/llms.txt">llms.txt</a></nav>'
"</body></html>"
)
def mirror(title: str) -> str:
"""Return a markdown mirror with a language-tagged fence."""
return f"# {title}\n\n{_TEXT}\n\n```python\nprint('d20')\n```\n"
def write(root: Path, rel: str, text: str) -> None:
"""Write ``text`` to ``root/rel``, creating parent dirs."""
path = root / rel
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
def build_good(root: Path) -> Path:
"""Create a fully compliant build output under ``root``."""
write(root, "index.html", page("Dice", "/index.md", home=True))
write(root, "index.md", mirror("Dice"))
write(root, "about/index.html", page("About", "/about/index.md"))
write(root, "about/index.md", mirror("About"))
write(
root,
"404.html",
"<!doctype html><html><body><h1>Not found</h1><p>No page at this address.</p></body></html>",
)
write(root, "robots.txt", ROBOTS)
write(root, "sitemap.xml", SITEMAP)
write(root, "llms.txt", LLMS)
write(root, ".well-known/ai-catalog.json", json.dumps(CATALOG))
return root
SHELL = '<!doctype html><html><head><title>App</title></head><body><div id="root"></div><script src="/app.js"></script></body></html>'
def build_bad(root: Path) -> Path:
"""Create a build output that violates every find/read item."""
write(root, "index.html", SHELL)
write(root, "about/index.html", SHELL)
write(
root,
"robots.txt",
"User-agent: *\nDisallow:\n\nUser-agent: GPTBot\nDisallow: /\n",
)
write(root, "sitemap.xml", "<html>not xml")
write(root, "llms.txt", "# Dice\n\n## Pages\n- [About](/about/)\n")
write(
root,
".well-known/ai-catalog.json",
json.dumps({"entries": [{"identifier": "x"}]}),
)
return root
def add_pages(root: Path, count: int, *, broken_last: bool = False) -> None:
"""Add ``count`` more sitemap pages; the last one an empty shell if asked."""
urls = []
for i in range(count):
title = f"Post {i}"
html = (
SHELL if broken_last and i == count - 1 else page(title, f"/p{i}/index.md")
)
write(root, f"p{i}/index.html", html)
write(root, f"p{i}/index.md", mirror(title))
urls.append(f" <url><loc>{ORIGIN}/p{i}/</loc></url>\n")
write(
root, "sitemap.xml", SITEMAP.replace("</urlset>", "".join(urls) + "</urlset>")
)

View File

@ -1,75 +0,0 @@
"""Backend edge cases: URL mapping, memoized failures, traversal, deadline."""
from __future__ import annotations
from pathlib import Path
import pytest
from agent_ready.backends import DirSite, HttpSite, Response, Site
from agent_ready.report import Undeterminable
from agent_ready.tests.fakes import FakeHttp, resp
class FlakySite(Site):
"""Raises Undeterminable on every fetch and counts the attempts."""
live = True
def __init__(self) -> None:
super().__init__("https://flaky.test")
self.attempts = 0
def _fetch(self, path: str, ua: str) -> Response:
self.attempts += 1
raise Undeterminable(f"{path}: boom")
def test_to_path_rejects_offsite_and_non_http(tmp_path: Path) -> None:
site = DirSite(tmp_path)
assert site.to_path("https://other.test/x") is None
assert site.to_path("mailto:a@b.test") is None
assert site.to_path("/rel/") == "/rel/"
assert site.to_path("https://dist.invalid") == "/"
def test_failures_are_memoized() -> None:
site = FlakySite()
for _ in range(2):
with pytest.raises(Undeterminable, match="boom"):
site.get("/x")
assert site.attempts == 1
def test_prefetch_swallows_failures_then_get_raises() -> None:
site = FlakySite()
site.prefetch(["/a", "/b", "/a"])
assert site.attempts == 2
with pytest.raises(Undeterminable):
site.get("/a")
def test_base_fetch_is_abstract() -> None:
with pytest.raises(NotImplementedError):
Site("https://x.test")._fetch("/", "ua")
def test_dist_never_serves_outside_root(tmp_path: Path) -> None:
root = tmp_path / "dist"
root.mkdir()
(tmp_path / "secret.txt").write_text("nope")
site = DirSite(root)
assert site.file_for("/../secret.txt") is None
assert site.get("/../secret.txt").status == 404
def test_exhausted_budget_is_undeterminable() -> None:
site = HttpSite("https://budget.test/anything", budget=0)
with pytest.raises(Undeterminable, match="time budget"):
site.get("/")
def test_fake_http_serves_canned_pages() -> None:
site = FakeHttp({"/": resp("hi")})
assert site.get("/").text == "hi"
assert site.get("/nope").status == 404

View File

@ -1,153 +0,0 @@
"""Dist-mode scenarios: one mutation of the good tree per failing branch."""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from agent_ready.tests import sitegen
from agent_ready.tests.fakes import lines_for, run_main
OPENAPI_OP = {"get": {"operationId": "ping"}}
def test_robots_without_wildcard_group(
capsys: pytest.CaptureFixture[str], good_dir: Path
) -> None:
robots = f"User-agent: Foo\nDisallow: /x\n\nSitemap: {sitegen.ORIGIN}/sitemap.xml\n"
sitegen.write(good_dir, "robots.txt", robots)
_, lines = run_main(capsys, "dist", str(good_dir))
assert "no 'User-agent: *' group" in lines_for(lines, "AR-FIND-01", "FAIL")[0]
def test_sitemap_found_without_robots_reference_sets_origin(
capsys: pytest.CaptureFixture[str], good_dir: Path
) -> None:
sitegen.write(good_dir, "robots.txt", "User-agent: *\nDisallow:\n")
code, lines = run_main(capsys, "dist", str(good_dir))
assert code == 1
assert "no 'Sitemap:' line" in lines_for(lines, "AR-FIND-03", "FAIL")[0]
assert lines_for(lines, "AR-FIND-03", "FAIL")[1:] == []
assert lines_for(lines, "AR-READ-01", "PASS")
def test_sitemap_index_follows_children(
capsys: pytest.CaptureFixture[str], good_dir: Path
) -> None:
child = f"{sitegen.ORIGIN}/sitemap-pages.xml"
index = (
'<?xml version="1.0"?><sitemapindex '
'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
f"<sitemap><loc>{child}</loc></sitemap></sitemapindex>"
)
sitegen.write(good_dir, "sitemap-pages.xml", sitegen.SITEMAP)
sitegen.write(good_dir, "sitemap.xml", index)
code, lines = run_main(capsys, "dist", str(good_dir))
assert code == 0, lines
assert "lists 2 pages" in lines_for(lines, "AR-FIND-03", "PASS")[-1]
def test_sitemap_with_offsite_locs_and_none_at_all(
capsys: pytest.CaptureFixture[str], good_dir: Path
) -> None:
urlset = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">{}</urlset>'
off = urlset.format("<url><loc>https://other.test/x/</loc></url>")
sitegen.write(good_dir, "sitemap.xml", off)
_, lines = run_main(capsys, "dist", str(good_dir))
assert "1 <loc> URLs are not on" in lines_for(lines, "AR-FIND-03", "FAIL")[0]
sitegen.write(good_dir, "sitemap.xml", urlset.format(""))
_, lines = run_main(capsys, "dist", str(good_dir))
assert "no <loc> entries" in lines_for(lines, "AR-FIND-03", "FAIL")[0]
assert "sitemap yielded no pages" in lines_for(lines, "AR-READ-01", "FAIL")[0]
def test_sitemap_pages_missing_or_copies_of_the_shell(
capsys: pytest.CaptureFixture[str], good_dir: Path
) -> None:
home = (good_dir / "index.html").read_text()
sitegen.write(good_dir, "dup/index.html", home)
extra = "".join(
f"<url><loc>{sitegen.ORIGIN}/{p}/</loc></url>" for p in ("dup", "gone")
)
sitemap = (good_dir / "sitemap.xml").read_text()
sitegen.write(
good_dir, "sitemap.xml", sitemap.replace("</urlset>", extra + "</urlset>")
)
code, lines = run_main(capsys, "dist", str(good_dir))
assert code == 1
problem = lines_for(lines, "AR-READ-01", "FAIL")[0]
assert "/gone/ (HTTP 404)" in problem
assert "/dup/ (byte-identical to the homepage shell)" in problem
def test_markdown_mirror_with_untagged_fence(
capsys: pytest.CaptureFixture[str], good_dir: Path
) -> None:
sitegen.write(good_dir, "about/index.md", "# About\n\n```\nplain\n```\n")
_, lines = run_main(capsys, "dist", str(good_dir))
assert "about/index.md (1 untagged fences)" in lines_for(lines, "AR-READ-05")[0]
def test_llms_with_broken_link(
capsys: pytest.CaptureFixture[str], good_dir: Path
) -> None:
llms = (good_dir / "llms.txt").read_text() + "- [Gone](/gone/): missing\n"
sitegen.write(good_dir, "llms.txt", llms)
_, lines = run_main(capsys, "dist", str(good_dir))
assert "broken links: /gone/" in lines_for(lines, "AR-READ-07", "FAIL")[0]
def test_homepage_with_unparseable_json_ld(
capsys: pytest.CaptureFixture[str], good_dir: Path
) -> None:
home = (good_dir / "index.html").read_text()
sitegen.write(good_dir, "index.html", home.replace('"@context"', "@context"))
_, lines = run_main(capsys, "dist", str(good_dir))
assert "1 JSON-LD blocks fail" in lines_for(lines, "AR-READ-08", "FAIL")[-1]
def test_webmcp_found_in_linked_script(
capsys: pytest.CaptureFixture[str], good_dir: Path
) -> None:
home = (good_dir / "index.html").read_text()
scripts = '<script src="https://cdn.other.test/x.js"></script><script src="/app.js"></script>'
sitegen.write(good_dir, "index.html", home.replace("</body>", scripts + "</body>"))
sitegen.write(good_dir, "app.js", "document.modelContext.registerTool({});")
_, lines = run_main(capsys, "dist", str(good_dir), "--act")
assert lines_for(lines, "AR-ACT-07", "PASS")
@pytest.mark.parametrize(
("doc", "status", "needle"),
[
({"openapi": "3.1.0", "paths": {"/p": OPENAPI_OP}}, "PASS", "1 operations"),
(
{"openapi": "3.1.0", "paths": {"/p": {"get": {}}}},
"FAIL",
"lack operationId",
),
({"openapi": "3.1.0", "paths": {}}, "FAIL", "no operations in paths"),
({"openapi": "3.1.0", "paths": 5}, "FAIL", "no operations in paths"),
(
{
"openapi": "3.1.0",
"paths": {"/a": "x", "/b": {"parameters": [], "get": 1}},
},
"FAIL",
"1 operations lack",
),
],
)
def test_openapi_document_shapes(
capsys: pytest.CaptureFixture[str],
good_dir: Path,
doc: dict[str, object],
status: str,
needle: str,
) -> None:
sitegen.write(good_dir, "openapi.json", json.dumps(doc))
_, lines = run_main(capsys, "dist", str(good_dir), "--act")
found = lines_for(lines, "AR-ACT-02", status)
assert found and needle in found[0], lines_for(lines, "AR-ACT-02")

View File

@ -1,83 +0,0 @@
"""Live-mode checks driven by canned responses (no sockets, no network)."""
from __future__ import annotations
import pytest
from agent_ready import checks_act, checks_find, checks_read
from agent_ready.backends import Response
from agent_ready.tests import sitegen
from agent_ready.tests.fakes import FakeHttp, make_ctx, resp
LONG = "Readable text for agents that never run JavaScript. " * 4
MD_LINK = '</index.md>; rel="alternate"; type="text/markdown"'
def _home(*, ctype: str = "text/html", **headers: str) -> Response:
return resp(f"<html><body><p>{LONG}</p></body></html>", ctype, **headers)
def test_challenge_page_for_one_agent_is_reported() -> None:
challenge = resp("<html>Just a moment...</html>")
ua = checks_find.AGENT_UAS["GPTBot"]
site = FakeHttp({"/": _home()}, ua_pages={("/", ua): challenge})
ctx, out = make_ctx(site)
checks_find.check_agent_access(ctx)
assert out.getvalue().startswith("FAIL AR-FIND-01 GPTBot got a challenge page")
def test_sitemap_on_another_host_is_a_failure() -> None:
robots = "User-agent: *\nDisallow:\nSitemap: https://other.test/sitemap.xml\n"
site = FakeHttp({"/robots.txt": resp(robots, "text/plain")})
ctx, out = make_ctx(site)
checks_find.check_robots(ctx)
ctx.sitemap_pages = []
assert checks_find._sitemap_paths(ctx) == ["/sitemap.xml"]
assert (
"Sitemap is on another host: https://other.test/sitemap.xml" in out.getvalue()
)
def test_markdown_mirror_advertised_only_by_link_header() -> None:
site = FakeHttp(
{
"/": _home(link=MD_LINK),
"/index.md": resp(sitegen.mirror("Home"), "text/markdown"),
}
)
ctx, out = make_ctx(site)
checks_read.check_pages(ctx)
assert (
"PASS AR-READ-09 1 pages advertise a working markdown mirror" in out.getvalue()
)
def test_page_served_with_wrong_content_type() -> None:
ctx, out = make_ctx(FakeHttp({"/": _home(ctype="text/plain")}))
checks_read.check_pages(ctx)
assert (
"FAIL AR-READ-01 answer not in initial HTML: / (text/plain)" in out.getvalue()
)
def _mcp(status: int, **headers: str) -> Response:
return Response(status, "application/json", b"{}", dict(headers))
@pytest.mark.parametrize(
("answer", "status"),
[
(_mcp(401, **{"www-authenticate": "Bearer"}), "PASS"),
(_mcp(401), "FAIL"),
],
)
def test_mcp_behind_oauth_needs_www_authenticate(answer: Response, status: str) -> None:
ctx, out = make_ctx(FakeHttp({}, mcp=answer), act=True)
checks_act.check_mcp(ctx)
assert f"{status} AR-ACT-03" in out.getvalue()
def test_act_off_reports_a_single_skip() -> None:
ctx, out = make_ctx(FakeHttp({}))
checks_act.run_act(ctx)
assert out.getvalue().startswith("SKIP AR-ACT-10 act stage off")

View File

@ -1,202 +0,0 @@
"""End-to-end tests of ``check.py`` in dist and url mode."""
from __future__ import annotations
import re
import runpy
import subprocess
import sys
from collections.abc import Callable
from pathlib import Path
import pytest
from agent_ready.check import main
from agent_ready.tests import sitegen
LINE = re.compile(r"^(PASS|FAIL|SKIP) AR-(FIND|READ|ACT)-\d\d \S")
PKG = Path(__file__).resolve().parents[1]
def run(capsys: pytest.CaptureFixture[str], *argv: str) -> tuple[int, list[str]]:
"""Run the CLI in-process; return exit code and output lines."""
code = main(list(argv))
lines = capsys.readouterr().out.splitlines()
bad = [ln for ln in lines if not LINE.match(ln)]
assert not bad, bad
return code, lines
def statuses(lines: list[str], ar_id: str) -> set[str]:
"""Return the set of statuses reported for ``ar_id``."""
return {ln.split()[0] for ln in lines if ln.split()[1] == ar_id}
def test_dist_good_passes(capsys: pytest.CaptureFixture[str], good_dir: Path) -> None:
code, lines = run(capsys, "dist", str(good_dir))
assert code == 0, "\n".join(lines)
assert not [ln for ln in lines if ln.startswith("FAIL")]
for ar_id in (
"AR-FIND-01",
"AR-FIND-03",
"AR-FIND-04",
"AR-READ-01",
"AR-READ-02",
"AR-READ-07",
"AR-READ-09",
):
found = statuses(lines, ar_id)
assert "PASS" in found and "FAIL" not in found, ar_id
assert statuses(lines, "AR-READ-11") == {"SKIP"}
assert statuses(lines, "AR-ACT-10") == {"SKIP"}
def test_dist_bad_fails_every_item(
capsys: pytest.CaptureFixture[str], bad_dir: Path
) -> None:
code, lines = run(capsys, "dist", str(bad_dir))
assert code == 1
for ar_id in (
"AR-FIND-01",
"AR-FIND-03",
"AR-FIND-04",
"AR-READ-01",
"AR-READ-02",
"AR-READ-03",
"AR-READ-06",
"AR-READ-07",
"AR-READ-08",
"AR-READ-09",
):
assert "FAIL" in statuses(lines, ar_id), ar_id
assert any("gptbot" in ln for ln in lines if "AR-FIND-01" in ln)
def test_dist_untagged_code_fails(
capsys: pytest.CaptureFixture[str], good_dir: Path
) -> None:
about = good_dir / "about/index.html"
about.write_text(about.read_text().replace(' class="language-python"', ""))
code, lines = run(capsys, "dist", str(good_dir))
assert code == 1
assert "FAIL" in statuses(lines, "AR-READ-05")
def test_dist_act_opt_in(capsys: pytest.CaptureFixture[str], good_dir: Path) -> None:
code, lines = run(capsys, "dist", str(good_dir), "--act")
assert code == 1
assert "FAIL" in statuses(lines, "AR-ACT-02")
def test_usage_errors(capsys: pytest.CaptureFixture[str], tmp_path: Path) -> None:
assert main(["dist", str(tmp_path / "missing")]) == 2
assert main(["url", "not-a-url"]) == 2
with pytest.raises(SystemExit) as exc:
main([])
assert exc.value.code == 2
def test_url_good_passes(
capsys: pytest.CaptureFixture[str], good_dir: Path, serve: Callable[..., str]
) -> None:
code, lines = run(capsys, "url", serve(good_dir))
assert code == 0, "\n".join(lines)
assert statuses(lines, "AR-READ-02") == {"PASS"}
def test_url_spa_fallback_fails(
capsys: pytest.CaptureFixture[str], good_dir: Path, serve: Callable[..., str]
) -> None:
"""Every path 200 text/html: a status-only check would wrongly pass."""
code, lines = run(capsys, "url", serve(good_dir, "spa"))
assert code == 1
for ar_id in (
"AR-FIND-01",
"AR-FIND-03",
"AR-FIND-04",
"AR-READ-01",
"AR-READ-02",
"AR-READ-07",
"AR-READ-09",
):
assert "FAIL" in statuses(lines, ar_id), ar_id
assert any("text/html" in ln for ln in lines if "AR-FIND-01" in ln)
assert any(
"does not exist" in ln for ln in lines if "AR-READ-06" in ln
) # HTML fallback != file
def test_url_json_404(
capsys: pytest.CaptureFixture[str], good_dir: Path, serve: Callable[..., str]
) -> None:
code, lines = run(capsys, "url", serve(good_dir, "json404"))
assert code == 1
assert statuses(lines, "AR-READ-02") == {"PASS"}
assert "FAIL" in statuses(lines, "AR-FIND-01")
def test_url_blocked_agent(
capsys: pytest.CaptureFixture[str], good_dir: Path, serve: Callable[..., str]
) -> None:
code, lines = run(capsys, "url", serve(good_dir, block_ua="GPTBot"))
assert code == 1
assert any(ln.startswith("FAIL AR-FIND-01") and "GPTBot" in ln for ln in lines)
def test_url_unreachable_is_3(
capsys: pytest.CaptureFixture[str], dead_url: str
) -> None:
assert main(["url", dead_url]) == 3
def test_entry_points_from_other_cwd(good_dir: Path, tmp_path: Path) -> None:
for cmd in (
["bash", str(PKG / "check.sh")],
[sys.executable, str(PKG / "check.py")],
):
proc = subprocess.run(
[*cmd, "dist", str(good_dir)],
cwd=tmp_path,
capture_output=True,
text=True,
check=False,
)
assert proc.returncode == 0, proc.stdout + proc.stderr
def test_dist_checks_every_page_url_caps_visibly(
capsys: pytest.CaptureFixture[str], good_dir: Path, serve: Callable[..., str]
) -> None:
"""Dist mode ignores --max-pages; url mode names the cap instead of hiding it."""
sitegen.add_pages(good_dir, 6, broken_last=True)
code, lines = run(capsys, "dist", str(good_dir), "--max-pages", "3")
assert code == 1
assert any(ln.startswith("FAIL AR-READ-01") and "/p5/" in ln for ln in lines)
code, lines = run(capsys, "url", serve(good_dir), "--max-pages", "3")
assert any(ln.startswith("SKIP AR-READ-01 checked 3 of 8 pages") for ln in lines), (
lines
)
def test_url_act_mcp(
capsys: pytest.CaptureFixture[str], good_dir: Path, serve: Callable[..., str]
) -> None:
_, lines = run(capsys, "url", serve(good_dir), "--act")
assert statuses(lines, "AR-ACT-03") == {"PASS"}
assert statuses(lines, "AR-ACT-10") == {"PASS"}
assert "FAIL" in statuses(lines, "AR-ACT-07")
code, lines = run(capsys, "url", serve(good_dir, "spa"), "--act")
assert code == 1
assert statuses(lines, "AR-ACT-03") == {"FAIL"}
assert statuses(lines, "AR-ACT-10") == {"FAIL"}
def test_script_entry_point_in_process(good_dir: Path) -> None:
"""Run check.py as ``__main__`` here so the guard lines count for coverage."""
argv = ["check.py", "dist", str(good_dir)]
with pytest.MonkeyPatch.context() as patch:
patch.setattr(sys, "argv", argv)
patch.setattr(sys, "path", list(sys.path)) # check.py prepends to it
with pytest.raises(SystemExit) as exc:
runpy.run_path(str(PKG / "check.py"), run_name="__main__")
assert exc.value.code == 0

View File

@ -1,186 +0,0 @@
"""Unit tests for the format validators and the HTML scanner."""
from __future__ import annotations
import pytest
from agent_ready import formats, htmlscan
from agent_ready.tests import sitegen
# Verbatim from spec/spec.md "llms.txt shape (informative)".
SPEC_LLMS = """# Acme
> Acme ships a payments API for developers.
## Docs
- [Quickstart](https://example.com/docs/quickstart): first charge in one request
- [Authentication](https://example.com/docs/auth): API keys, scopes, rotation
## API
- [OpenAPI](https://example.com/openapi.json): full request and response contract
"""
# Shape of https://vercel.com/.well-known/ai-catalog.json (2026-10-22), trimmed.
VERCEL_CATALOG = {
"specVersion": "1.0",
"host": {"displayName": "Vercel", "identifier": "https://vercel.com"},
"entries": [
{
"identifier": "urn:air:vercel.com:api:rest",
"displayName": "Vercel REST API",
"type": "application/json",
"url": "https://vercel.com/openapi.json",
"tags": ["api"],
}
],
}
def test_llms_spec_example_is_valid() -> None:
assert formats.llms_problems(SPEC_LLMS) == []
def test_llms_allows_prose_and_plain_bullets() -> None:
text = SPEC_LLMS.replace(
"## API", "Version: v2.0\n\n## Notes\n- plain bullet\n\n## API"
)
assert formats.llms_problems(text) == []
@pytest.mark.parametrize(
("mutate", "needle"),
[
(lambda t: t.replace("# Acme\n", ""), "H1"),
(
lambda t: t.replace("> Acme ships a payments API for developers.\n", ""),
"blockquote",
),
(lambda t: "# Acme\n> summary\n\nno sections\n", "link list"),
(lambda t: "<!doctype html><html></html>", "HTML"),
],
)
def test_llms_rejects(mutate: object, needle: str) -> None:
problems = formats.llms_problems(mutate(SPEC_LLMS)) # type: ignore[operator]
assert any(needle in p for p in problems), problems
def test_llms_links() -> None:
assert formats.llms_links(SPEC_LLMS)[0] == "https://example.com/docs/quickstart"
def test_catalog_valid_examples() -> None:
assert formats.catalog_problems(VERCEL_CATALOG) == []
assert formats.catalog_problems(sitegen.CATALOG) == []
minimal = {
"specVersion": "1.0",
"entries": [{"identifier": "a", "type": "t", "data": {}}],
}
assert formats.catalog_problems(minimal) == []
@pytest.mark.parametrize(
("doc", "needle"),
[
([], "object"),
({"entries": []}, "specVersion"),
({"specVersion": "1", "entries": []}, "Major.Minor"),
({"specVersion": "1.0"}, "entries"),
({"specVersion": "1.0", "entries": [{"type": "t", "url": "u"}]}, "identifier"),
({"specVersion": "1.0", "entries": [{"identifier": "i", "url": "u"}]}, "type"),
(
{
"specVersion": "1.0",
"entries": [{"identifier": "i", "type": "t", "url": "u", "data": 1}],
},
"exactly one",
),
({"specVersion": "1.0", "entries": [], "host": {}}, "host.displayName"),
],
)
def test_catalog_rejects(doc: object, needle: str) -> None:
assert any(needle in p for p in formats.catalog_problems(doc)), (
formats.catalog_problems(doc)
)
def test_robots_blanket_blocks() -> None:
text = "User-agent: *\nDisallow:\n\nUser-Agent: GPTBot\nUser-agent: CCBot\nDisallow: /*\n"
groups = formats.parse_robots(text)
assert formats.has_wildcard_group(groups)
assert formats.blocked_agents(groups) == ["gptbot", "ccbot"]
def test_robots_partial_disallow_and_allow_override() -> None:
text = "User-agent: *\nDisallow: /search/\n\nUser-agent: ClaudeBot\nDisallow: /\nAllow: /\n"
assert formats.blocked_agents(formats.parse_robots(text)) == []
assert formats.robots_sitemaps("Sitemap: https://x.test/s.xml\n") == [
"https://x.test/s.xml"
]
def test_sitemap_parse() -> None:
kind, locs = formats.parse_sitemap(sitegen.SITEMAP.encode())
assert kind == "urlset"
assert locs == ["https://example.test/", "https://example.test/about/"]
with pytest.raises(ValueError):
formats.parse_sitemap(b"<html>not xml")
def test_untagged_fences() -> None:
assert formats.untagged_fences("```python\nx\n```\n") == 0
assert formats.untagged_fences("```\nx\n```\n~~~\ny\n~~~\n") == 2
def test_htmlscan_good_home() -> None:
scan = htmlscan.scan(sitegen.page("Dice", "/index.md", home=True))
assert len(scan.text) > 150
assert scan.code_blocks == 1 and scan.untagged_code == 0
assert scan.link_href("alternate", "text/markdown") == "/index.md"
assert scan.link_href("ai-catalog") == "/.well-known/ai-catalog.json"
assert "/about/" in scan.anchors
assert scan.jsonld_ok == 1
def test_htmlscan_shell_has_no_text() -> None:
scan = htmlscan.scan(sitegen.SHELL)
assert scan.text.strip() == "" # <title> is head, not body text
assert scan.anchors == []
bare = htmlscan.scan(
'<pre><code>x</code></pre><script type="application/ld+json">{bad</script>'
)
assert bare.untagged_code == 1 and bare.jsonld_bad == 1
def test_sitemap_wrong_root_is_rejected() -> None:
with pytest.raises(ValueError, match="not <urlset>"):
formats.parse_sitemap(b"<feed><loc>https://x.test/</loc></feed>")
def test_catalog_entry_must_be_an_object() -> None:
doc = {"specVersion": "1.0", "entries": ["nope"]}
assert formats.catalog_problems(doc) == ["entries[0]: must be an object"]
def test_catalog_accessors_tolerate_invalid_docs() -> None:
for doc in ([], {"entries": "x"}):
assert formats.catalog_urls(doc) == []
assert formats.catalog_types(doc) == []
assert formats.catalog_urls(sitegen.CATALOG) == [f"{sitegen.ORIGIN}/llms.txt"]
def test_fence_inside_fence_is_not_a_new_block() -> None:
nested = "````md\n```\ninner\n```\n````\n"
assert formats.untagged_fences(nested) == 0
assert formats.untagged_fences("```python\n~~~\n```\n```\nz\n```\n") == 1
def test_htmlscan_jsonld_without_context_is_bad() -> None:
scan = htmlscan.scan(
'<script type="application/ld+json">{"name": "x"}</script>'
'<script type="application/ld+json">[]</script>'
)
assert scan.jsonld_ok == 0 and scan.jsonld_bad == 2
def test_htmlscan_unclosed_inner_hidden_tag_closes_with_outer() -> None:
scan = htmlscan.scan("<body><template><noscript>x</template><p>visible</p></body>")
assert scan.text == "visible"

View File

@ -1,54 +0,0 @@
"""Report policy, Link-header parsing and context helpers."""
from __future__ import annotations
import io
from agent_ready.backends import Response
from agent_ready.context import link_header_href
from agent_ready.report import Report, Undeterminable, summarize
from agent_ready.tests.fakes import FakeSite, make_ctx, resp
def _boom() -> None:
raise Undeterminable("refused")
def test_guard_turns_network_failure_into_skip_and_exit_3() -> None:
out = io.StringIO()
report = Report(out=out)
report.guard("AR-FIND-01", _boom)
assert out.getvalue() == "SKIP AR-FIND-01 undeterminable (network): refused\n"
assert report.exit_code() == 3
report.fail("AR-FIND-01", "x")
assert report.exit_code() == 1
def test_summarize_notes_the_overflow() -> None:
assert summarize(["a", "b"]) == "a, b"
assert summarize(list("abcdef")) == "a, b, c, d (+2 more)"
def _linked(link: str) -> Response:
return Response(200, "text/html", b"", {"link": link})
def test_link_header_skips_unusable_values() -> None:
header = (
"garbage, </no-rel>; type=x, </other>; rel=preload, "
'</a.md>; rel="alternate"; type="text/plain", '
'</b.md>; rel="alternate"; type="text/markdown"'
)
assert link_header_href(_linked(header), "alternate", "text/markdown") == "/b.md"
assert link_header_href(_linked(header), "alternate") == "/a.md"
assert link_header_href(_linked(header), "ai-catalog") is None
assert link_header_href(_linked(""), "alternate") is None
def test_exists_rejects_html_fallback_for_files() -> None:
ctx, _ = make_ctx(
FakeSite({"/f.txt": resp("<html><head></head>"), "/p/": resp("<html>")})
)
assert not ctx.exists("/f.txt")
assert ctx.exists("/p/")
assert not ctx.exists("/missing.txt")

View File

@ -1,7 +0,0 @@
"""Deterministic coverage-gap lister: a coverage report in, a task list out.
Replaces "spawn an LLM agent to explore where coverage is missing". Reads lcov,
Cobertura XML, JaCoCo XML or coverage.py JSON and lists, per file, exactly
which lines and branches are uncovered. Exit codes are the verdict (0 covered,
1 gaps, 2 unreadable) so CI and hooks adjudicate without a model.
"""

View File

@ -1,5 +0,0 @@
"""`python3 -m coverage_gaps` -- the entrypoint the `coverage-gaps` wrapper calls."""
from coverage_gaps.cli import main
raise SystemExit(main())

View File

@ -1,16 +0,0 @@
#!/bin/bash
# ============================================================================
# coverage-gaps -- per-file task list of uncovered lines/branches.
# Thin wrapper over `python3 -m coverage_gaps` (see coverage_gaps/cli.py).
# Usage: coverage-gaps <report> [--json] [--max-files N] [--min-missed N]
# [--path-prefix P] Exit: 0 covered | 1 gaps | 2 bad input
# ============================================================================
set -euo pipefail
BIN_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd -P)"
readonly BIN_DIR
PYTHONPATH="$BIN_DIR/../.." python3 -m coverage_gaps "$@"
exit $?

View File

@ -1,63 +0,0 @@
"""Command line: `coverage-gaps REPORT [--json] [--max-files N] ...`."""
from __future__ import annotations
import argparse
import sys
from collections.abc import Sequence
from pathlib import Path
from coverage_gaps.models import ReportError
from coverage_gaps.parsers import load_report
from coverage_gaps.render import select, to_json, to_markdown
EXIT_COVERED = 0
EXIT_GAPS = 1
EXIT_UNREADABLE = 2
def build_parser() -> argparse.ArgumentParser:
"""Argument parser (format is auto-detected from file content)."""
parser = argparse.ArgumentParser(
prog="coverage-gaps",
description="List uncovered lines/branches from lcov, Cobertura, JaCoCo "
"or coverage.py JSON reports. Exit 0 covered, 1 gaps, 2 unreadable.",
)
parser.add_argument("report", type=Path, help="coverage report file")
parser.add_argument("--json", action="store_true", help="emit JSON, not markdown")
parser.add_argument(
"--max-files",
type=int,
default=0,
metavar="N",
help="list only the N worst files (0 = all)",
)
parser.add_argument(
"--min-missed",
type=int,
default=1,
metavar="N",
help="skip files with fewer than N missed lines+branches",
)
parser.add_argument(
"--path-prefix",
default="",
metavar="PREFIX",
help="only files whose path starts with PREFIX",
)
return parser
def main(argv: Sequence[str] | None = None) -> int:
"""Run the tool and return the process exit code."""
args = build_parser().parse_args(argv)
try:
fmt, gaps = load_report(args.report)
except ReportError as exc:
print(f"coverage-gaps: {exc}", file=sys.stderr)
return EXIT_UNREADABLE
files = select(gaps, path_prefix=args.path_prefix, min_missed=args.min_missed)
shown = args.max_files if args.max_files > 0 else len(files)
render = to_json if args.json else to_markdown
sys.stdout.write(render(fmt, files, shown) + ("\n" if args.json else ""))
return EXIT_GAPS if files else EXIT_COVERED

View File

@ -1,69 +0,0 @@
"""Format-neutral data model shared by every parser and renderer."""
from __future__ import annotations
from dataclasses import dataclass, field
class ReportError(Exception):
"""The input is unreadable, unrecognised or malformed (exit code 2)."""
@dataclass
class FileAccumulator:
"""Mutable per-file collector that merges repeated records for one path.
Reports may list a file more than once (lcov re-runs, Cobertura inner
classes). A line or branch counts as covered if ANY record covered it.
"""
lines: dict[int, bool] = field(default_factory=dict)
branches: dict[tuple[int, int, int], bool] = field(default_factory=dict)
def add_line(self, line: int, covered: bool) -> None:
"""Record a line, OR-merging with earlier records of the same line."""
self.lines[line] = self.lines.get(line, False) or covered
def add_branch(self, line: int, block: int, index: int, covered: bool) -> None:
"""Record one branch outcome, OR-merging on (line, block, index)."""
key = (line, block, index)
self.branches[key] = self.branches.get(key, False) or covered
@dataclass(frozen=True)
class FileGap:
"""Coverage state of one source file."""
path: str
total_lines: int
missed_lines: tuple[int, ...]
total_branches: int
missed_branch_lines: tuple[int, ...]
missed_branches: int
@property
def missed(self) -> int:
"""Missed lines plus missed branches: the sort key."""
return len(self.missed_lines) + self.missed_branches
@property
def percent(self) -> float:
"""Percent covered over lines and branches combined (100 if empty)."""
total = self.total_lines + self.total_branches
if total == 0:
return 100.0
return 100.0 * (total - self.missed) / total
def build_gap(path: str, acc: FileAccumulator) -> FileGap:
"""Freeze an accumulator into a FileGap."""
missed_lines = tuple(sorted(n for n, ok in acc.lines.items() if not ok))
missed_br = [key for key, ok in acc.branches.items() if not ok]
return FileGap(
path=path,
total_lines=len(acc.lines),
missed_lines=missed_lines,
total_branches=len(acc.branches),
missed_branch_lines=tuple(sorted({key[0] for key in missed_br})),
missed_branches=len(missed_br),
)

View File

@ -1,132 +0,0 @@
"""Parsers for lcov, Cobertura XML, JaCoCo XML and coverage.py JSON."""
from __future__ import annotations
import json
import re
import xml.etree.ElementTree as ET
from collections.abc import Callable
from pathlib import Path
from coverage_gaps.models import FileAccumulator, FileGap, ReportError, build_gap
_CONDITION = re.compile(r"\((\d+)/(\d+)\)")
def _finish(files: dict[str, FileAccumulator]) -> list[FileGap]:
return [build_gap(path, acc) for path, acc in files.items()]
def parse_lcov(text: str) -> list[FileGap]:
"""Parse lcov.info: SF / DA / BRDA records, `end_of_record` optional."""
files: dict[str, FileAccumulator] = {}
acc: FileAccumulator | None = None
for raw in text.splitlines():
line = raw.strip()
if line.startswith("SF:"):
acc = files.setdefault(line[3:], FileAccumulator())
elif acc is not None and line.startswith("DA:"):
num, hits, *_ = line[3:].split(",")
acc.add_line(int(num), int(hits) > 0)
elif acc is not None and line.startswith("BRDA:"):
num, block, branch, taken = line[5:].split(",")
covered = taken not in ("-", "0")
acc.add_branch(int(num), int(block), _branch_id(branch), covered)
elif line == "end_of_record":
acc = None
return _finish(files)
def _branch_id(branch: str) -> int:
return int(branch) if branch.isdigit() else 0
def parse_cobertura(text: str) -> list[FileGap]:
"""Parse Cobertura `coverage.xml` (coverage.py, gcovr, ...)."""
root = ET.fromstring(text)
files: dict[str, FileAccumulator] = {}
for cls in root.iter("class"):
acc = files.setdefault(cls.get("filename", ""), FileAccumulator())
for node in cls.iter("line"):
num = int(node.get("number", "0"))
acc.add_line(num, int(node.get("hits", "0")) > 0)
match = _CONDITION.search(node.get("condition-coverage", ""))
if match:
hit, total = int(match[1]), int(match[2])
for idx in range(total):
acc.add_branch(num, 0, idx, idx < hit)
return _finish(files)
def parse_jacoco(text: str) -> list[FileGap]:
"""Parse JaCoCo XML: per `sourcefile` line records (mi/ci/mb/cb)."""
root = ET.fromstring(text)
files: dict[str, FileAccumulator] = {}
for package in root.iter("package"):
base = package.get("name", "")
for source in package.iter("sourcefile"):
name = source.get("name", "")
acc = files.setdefault(
f"{base}/{name}" if base else name, FileAccumulator()
)
for node in source.iter("line"):
num = int(node.get("nr", "0"))
acc.add_line(num, int(node.get("ci", "0")) > 0)
missed_b, covered_b = int(node.get("mb", "0")), int(node.get("cb", "0"))
for idx in range(missed_b + covered_b):
acc.add_branch(num, 0, idx, idx >= missed_b)
return _finish(files)
def parse_coverage_json(text: str) -> list[FileGap]:
"""Parse `coverage json` output (with or without branch data)."""
files: dict[str, FileAccumulator] = {}
for path, data in json.loads(text).get("files", {}).items():
acc = files.setdefault(path, FileAccumulator())
for num in data.get("executed_lines", []):
acc.add_line(num, True)
for num in data.get("missing_lines", []):
acc.add_line(num, False)
missing = data.get("missing_branches", [])
for idx, (src, _dst) in enumerate(missing):
acc.add_branch(abs(src), 1, idx, False)
covered = data.get("summary", {}).get("covered_branches", 0)
for idx in range(covered):
acc.add_branch(0, 2, idx, True)
return _finish(files)
def detect_format(text: str) -> str:
"""Return 'lcov' | 'cobertura' | 'jacoco' | 'json' from file content."""
head = text.lstrip()[:4000]
if head.startswith("{"):
return "json"
if head.startswith("<"):
if "<report" in head:
return "jacoco"
if "<coverage" in head:
return "cobertura"
elif re.search(r"^(TN|SF):", head, re.MULTILINE):
return "lcov"
raise ReportError("unrecognised coverage report format")
_PARSERS: dict[str, Callable[[str], list[FileGap]]] = {
"lcov": parse_lcov,
"cobertura": parse_cobertura,
"jacoco": parse_jacoco,
"json": parse_coverage_json,
}
def load_report(path: Path) -> tuple[str, list[FileGap]]:
"""Read and parse a report; raise ReportError on any failure."""
try:
text = path.read_text(encoding="utf-8", errors="replace")
except OSError as exc:
raise ReportError(f"cannot read {path}: {exc.strerror}") from exc
fmt = detect_format(text)
try:
return fmt, _PARSERS[fmt](text)
except (ET.ParseError, ValueError, TypeError, AttributeError) as exc:
raise ReportError(f"malformed {fmt} report {path}: {exc}") from exc

View File

@ -1,102 +0,0 @@
"""Filtering and rendering: markdown (agent task list) and JSON."""
from __future__ import annotations
import json
from collections.abc import Iterable, Sequence
from coverage_gaps.models import FileGap
def compact_ranges(lines: Iterable[int]) -> str:
"""Collapse line numbers into '12-18, 40'."""
nums = sorted(set(lines))
parts: list[str] = []
idx = 0
while idx < len(nums):
end = idx
while end + 1 < len(nums) and nums[end + 1] == nums[end] + 1:
end += 1
parts.append(_span(nums[idx], nums[end]))
idx = end + 1
return ", ".join(parts)
def _span(start: int, end: int) -> str:
return str(start) if start == end else f"{start}-{end}"
def select(
gaps: Sequence[FileGap], *, path_prefix: str = "", min_missed: int = 1
) -> list[FileGap]:
"""Keep files with gaps under the prefix, worst first (ties by path)."""
kept = [
gap
for gap in gaps
if gap.path.startswith(path_prefix) and gap.missed >= max(min_missed, 1)
]
return sorted(kept, key=lambda gap: (-gap.missed, gap.path))
def _totals(files: Sequence[FileGap]) -> dict[str, int]:
return {
"files": len(files),
"missed_lines": sum(len(gap.missed_lines) for gap in files),
"missed_branches": sum(gap.missed_branches for gap in files),
}
def to_json(fmt: str, files: Sequence[FileGap], shown: int) -> str:
"""Machine-readable output; `files` is already sorted and filtered."""
entries = [
{
"path": gap.path,
"percent_covered": round(gap.percent, 2),
"missed": gap.missed,
"missed_lines": compact_ranges(gap.missed_lines),
"missed_line_numbers": list(gap.missed_lines),
"missed_branch_count": gap.missed_branches,
"missed_branch_lines": list(gap.missed_branch_lines),
}
for gap in files[:shown]
]
doc = {
"format": fmt,
"total": _totals(files),
"shown": len(entries),
"files": entries,
}
return json.dumps(doc, indent=2)
def to_markdown(fmt: str, files: Sequence[FileGap], shown: int) -> str:
"""Markdown task list meant to be pasted to an agent verbatim."""
total = _totals(files)
out = [
"# Coverage gaps",
"",
(
f"Format: {fmt}. Total: {total['files']} files with gaps, "
f"{total['missed_lines']} missed lines, "
f"{total['missed_branches']} missed branches."
),
]
if shown < len(files):
out.append(f"Showing the {shown} worst files.")
if not files:
return "\n".join(out) + "\n\nNo coverage gaps.\n"
out += ["", "Write tests that cover every item below, then re-run the report.", ""]
for rank, gap in enumerate(files[:shown], start=1):
out.append(f"## {rank}. `{gap.path}` - {gap.percent:.1f}% covered")
if gap.missed_lines:
out.append(
f"- missed lines ({len(gap.missed_lines)}): "
f"{compact_ranges(gap.missed_lines)}"
)
if gap.missed_branches:
out.append(
f"- missed branches ({gap.missed_branches}) at lines: "
f"{compact_ranges(gap.missed_branch_lines)}"
)
out.append("")
return "\n".join(out).rstrip() + "\n"

View File

@ -1,89 +0,0 @@
"""Small fixture reports, one per supported format, all describing a.py/b.py.
a.py: lines 1-5 with 2,3 and 5 missed + one missed branch on line 4.
b.py: fully covered (must never be listed).
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
LCOV = """TN:
SF:a.py
DA:1,1
DA:2,0
DA:3,0
DA:4,1
DA:5,0
BRDA:4,0,0,1
BRDA:4,0,1,-
end_of_record
SF:b.py
DA:1,3
end_of_record
SF:a.py
DA:1,1
DA:2,0
end_of_record
"""
COBERTURA = """<?xml version="1.0" ?>
<coverage version="7"><packages><package name="p"><classes>
<class filename="a.py"><lines>
<line number="1" hits="1"/><line number="2" hits="0"/><line number="3" hits="0"/>
<line number="4" hits="1" branch="true" condition-coverage="50% (1/2)"/>
<line number="5" hits="0"/></lines></class>
<class filename="b.py"><lines><line number="1" hits="2"/></lines></class>
<class filename="a.py"><lines><line number="1" hits="1"/></lines></class>
</classes></package></packages></coverage>
"""
JACOCO = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE report PUBLIC "-//JACOCO//DTD Report 1.1//EN" "report.dtd">
<report name="x"><package name="com/x">
<sourcefile name="A.kt">
<line nr="1" mi="0" ci="3"/><line nr="2" mi="2" ci="0"/><line nr="3" mi="1" ci="0"/>
<line nr="4" mi="0" ci="4" mb="1" cb="1"/><line nr="5" mi="1" ci="0"/></sourcefile>
<sourcefile name="B.kt"><line nr="1" mi="0" ci="1"/></sourcefile>
</package><package name=""><sourcefile name="Top.kt">
<line nr="1" mi="0" ci="1"/></sourcefile></package></report>
"""
COVERAGE_JSON = json.dumps(
{
"files": {
"a.py": {
"executed_lines": [1, 4],
"missing_lines": [2, 3, 5],
"missing_branches": [[4, 5]],
"summary": {"covered_branches": 1},
},
"b.py": {
"executed_lines": [1],
"missing_lines": [],
"missing_branches": [],
},
}
}
)
@pytest.fixture
def reports(tmp_path: Path) -> dict[str, Path]:
"""Write each fixture report to disk, keyed by format name."""
out = {}
for name, body in {
"lcov": LCOV,
"cobertura": COBERTURA,
"jacoco": JACOCO,
"json": COVERAGE_JSON,
}.items():
out[name] = tmp_path / f"{name}.rpt"
out[name].write_text(body, encoding="utf-8")
return out

View File

@ -1,73 +0,0 @@
"""Parser and format-detection tests."""
from __future__ import annotations
from pathlib import Path
import pytest
from coverage_gaps.models import FileGap, ReportError
from coverage_gaps.parsers import detect_format, load_report
def _by_path(gaps: list[FileGap]) -> dict[str, FileGap]:
return {gap.path: gap for gap in gaps}
@pytest.mark.parametrize("fmt", ["lcov", "cobertura", "json"])
def test_a_py_gaps_match_across_formats(reports: dict[str, Path], fmt: str) -> None:
detected, gaps = load_report(reports[fmt])
assert detected == fmt
a = _by_path(gaps)["a.py"]
assert a.missed_lines == (2, 3, 5)
assert a.missed_branch_lines == (4,)
assert a.missed_branches == 1
assert a.total_lines == 5
assert a.total_branches == 2
assert _by_path(gaps)["b.py"].missed == 0
def test_jacoco(reports: dict[str, Path]) -> None:
fmt, gaps = load_report(reports["jacoco"])
assert fmt == "jacoco"
by = _by_path(gaps)
assert by["com/x/A.kt"].missed_lines == (2, 3, 5)
assert by["com/x/A.kt"].missed_branch_lines == (4,)
assert by["com/x/A.kt"].total_branches == 2
assert "Top.kt" in by
def test_percent_and_empty_file() -> None:
gap = FileGap("x", 4, (1,), 4, (2,), 1)
assert gap.percent == 75.0
assert FileGap("y", 0, (), 0, (), 0).percent == 100.0
def test_lcov_ignores_records_outside_a_file_and_dash_branch_ids() -> None:
from coverage_gaps.parsers import parse_lcov
gaps = parse_lcov(
"DA:1,0\nBRDA:1,0,0,0\nSF:z\nBRDA:2,0,-,0\nend_of_record\nDA:9,0\n"
)
assert len(gaps) == 1
assert gaps[0].missed_lines == ()
assert gaps[0].missed_branch_lines == (2,)
def test_detect_unknown_and_xml_without_known_root() -> None:
for text in ("hello", "<other/>", ""):
with pytest.raises(ReportError):
detect_format(text)
def test_load_errors(tmp_path: Path) -> None:
with pytest.raises(ReportError, match="cannot read"):
load_report(tmp_path / "missing")
bad_xml = tmp_path / "bad.xml"
bad_xml.write_text("<coverage><oops", encoding="utf-8")
with pytest.raises(ReportError, match="malformed cobertura"):
load_report(bad_xml)
bad_json = tmp_path / "bad.json"
bad_json.write_text("{nope", encoding="utf-8")
with pytest.raises(ReportError, match="malformed json"):
load_report(bad_json)

View File

@ -1,97 +0,0 @@
"""Rendering, filtering and CLI exit-code tests."""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from coverage_gaps.cli import main
from coverage_gaps.models import FileGap
from coverage_gaps.render import compact_ranges, select, to_markdown
def test_compact_ranges() -> None:
assert compact_ranges([]) == ""
assert compact_ranges([40, 12, 13, 14, 15, 16, 17, 18, 18]) == "12-18, 40"
assert compact_ranges([1, 3, 5, 6]) == "1, 3, 5-6"
def test_select_sorts_and_filters() -> None:
small = FileGap("src/b.py", 5, (1,), 0, (), 0)
big = FileGap("lib/a.py", 5, (1, 2), 2, (3,), 1)
clean = FileGap("src/c.py", 5, (), 0, (), 0)
assert [g.path for g in select([small, clean, big])] == ["lib/a.py", "src/b.py"]
assert [g.path for g in select([small, big], path_prefix="src")] == ["src/b.py"]
assert [g.path for g in select([small, big], min_missed=2)] == ["lib/a.py"]
def test_markdown_branch_only_gap() -> None:
out = to_markdown("lcov", [FileGap("z.py", 2, (), 2, (7,), 1)], 1)
assert "- missed lines" not in out
assert "missed branches (1) at lines: 7" in out
def test_module_entrypoint(
reports: dict[str, Path], monkeypatch: pytest.MonkeyPatch
) -> None:
import runpy
monkeypatch.setattr("sys.argv", ["coverage_gaps", str(reports["lcov"])])
with pytest.raises(SystemExit) as exc:
runpy.run_module("coverage_gaps", run_name="__main__")
assert exc.value.code == 1
def test_markdown_no_gaps() -> None:
assert "No coverage gaps." in to_markdown("lcov", [], 0)
@pytest.mark.parametrize("fmt", ["lcov", "cobertura", "jacoco", "json"])
def test_cli_markdown_exit_1(
reports: dict[str, Path], fmt: str, capsys: pytest.CaptureFixture[str]
) -> None:
assert main([str(reports[fmt])]) == 1
out = capsys.readouterr().out
assert "2-3, 5" in out
assert "missed branches (1) at lines: 4" in out
assert "b.py" not in out.replace("A.kt", "")
def test_cli_json_and_max_files(
reports: dict[str, Path], capsys: pytest.CaptureFixture[str]
) -> None:
assert main([str(reports["jacoco"]), "--json", "--max-files", "1"]) == 1
doc = json.loads(capsys.readouterr().out)
assert doc["shown"] == 1
assert doc["total"] == {"files": 1, "missed_lines": 3, "missed_branches": 1}
assert doc["files"][0]["missed_lines"] == "2-3, 5"
def test_cli_markdown_truncation_note(
reports: dict[str, Path], capsys: pytest.CaptureFixture[str], tmp_path: Path
) -> None:
two = tmp_path / "two.info"
two.write_text("SF:x\nDA:1,0\nend_of_record\nSF:y\nDA:1,0\nDA:2,0\nend_of_record\n")
assert main([str(two), "--max-files", "1"]) == 1
out = capsys.readouterr().out
assert "Showing the 1 worst files." in out
assert "`y`" in out
assert "`x`" not in out
def test_cli_exit_0_when_filtered_clean(
reports: dict[str, Path], capsys: pytest.CaptureFixture[str]
) -> None:
assert main([str(reports["lcov"]), "--path-prefix", "b."]) == 0
assert main([str(reports["lcov"]), "--json", "--min-missed", "99"]) == 0
assert "No coverage gaps." in capsys.readouterr().out
def test_cli_exit_2(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
assert main([str(tmp_path / "nope")]) == 2
junk = tmp_path / "junk"
junk.write_text("junk")
assert main([str(junk)]) == 2
assert "coverage-gaps:" in capsys.readouterr().err

View File

@ -0,0 +1,19 @@
name: pre-commit
on:
push:
branches: [main]
pull_request:
branches: [main]
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install dependencies
run: pip install -r requirements.txt
- uses: pre-commit/action@v3.0.1

View File

@ -0,0 +1,28 @@
name: Tests
on:
push:
branches: [main]
pull_request:
branches: [main]
jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: pip install -r requirements.txt
- name: Run tests with coverage
run: python -m pytest -v

12
crdt-sync/.gitignore vendored Normal file
View File

@ -0,0 +1,12 @@
__pycache__/
*.pyc
.venv/
.mypy_cache/
.ruff_cache/
.pytest_cache/
.coverage
coverage.lcov
htmlcov/
dist/
build/
*.egg-info/

View File

@ -3,14 +3,6 @@
# Run: pre-commit run --all-files
# Update: pre-commit autoupdate
# NOTE: this subproject lives inside the ~/src/utils monorepo, whose git root is
# one level up. pre-commit always runs hooks from the git root, so every path
# below is written relative to THAT, not to this directory, and the top-level
# `files` key keeps these hooks off the sibling subprojects. CI invokes this
# config exactly the same way:
# pre-commit run --config crdt-sync/.pre-commit-config.yaml --all-files
files: ^crdt-sync/
default_language_version:
python: python3
default_stages: [pre-commit]
@ -40,19 +32,6 @@ repos:
- repo: local
hooks:
- id: file-length-cap
name: file length <= 250 lines
entry: bash scripts/check_file_length.sh
language: system
pass_filenames: true
- id: subproject-scope
name: subproject scope regex still matches
entry: scripts/check_subproject_scope.sh crdt-sync
language: script
# always_run so the top-level `files` filter cannot skip the one hook
# whose job is to notice that filter has stopped matching anything.
always_run: true
pass_filenames: false
- id: no-noqa
name: Block noqa comments
entry: '(?i)#\s*(noqa|type:\s*ignore)'
@ -65,7 +44,7 @@ repos:
types: [python]
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.17.2
rev: v0.15.2
hooks:
- id: ruff
args: [--fix, --unsafe-fixes, --exit-non-zero-on-fix, --show-fixes]
@ -74,7 +53,7 @@ repos:
types_or: [python, pyi]
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v2.4.0
rev: v1.13.0
hooks:
- id: mypy
additional_dependencies:
@ -103,36 +82,36 @@ repos:
- --disable-error-code=list-item
- repo: https://github.com/pylint-dev/pylint
rev: v4.0.8
rev: v3.3.2
hooks:
- id: pylint
args:
- --rcfile=crdt-sync/pyproject.toml
- --fail-under=10
- --rcfile=pyproject.toml
- --fail-under=8.0
- --jobs=4
additional_dependencies:
- pytest
- requests
- repo: https://github.com/PyCQA/bandit
rev: 1.9.4
rev: 1.7.10
hooks:
- id: bandit
args:
- -c
- crdt-sync/pyproject.toml
- pyproject.toml
- --severity-level=high
- --confidence-level=medium
- --skip=B113
additional_dependencies: ["bandit[toml]"]
exclude: ^(crdt-sync/crdt_sync/tests/|.*test.*\.py$)
exclude: ^(tests/|.*test.*\.py$)
- repo: local
hooks:
- id: pytest-coverage
name: pytest with coverage enforcement
entry: scripts/run_subproject_tests.sh crdt-sync
language: script
entry: python -m pytest
language: system
types: [python]
pass_filenames: false
require_serial: true

View File

@ -1,98 +1,28 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""Shared CRDT merge scheme + pluggable remote sync transports."""
"""Shared CRDT merge scheme + GitHub-Contents-API sync transport."""
from __future__ import annotations
from crdt_sync._config import (
CONFIG_DIR,
CONFIG_FILE,
PASSWORD_FILE,
ConfigError,
FirebaseConfig,
credential_store_for,
firebase_client_for,
mirror_client_for,
)
from crdt_sync._device import DeviceIdentity, load_device_identity
from crdt_sync._firebase import (
DatabaseNotFoundError,
FirebaseSyncClient,
FirebaseSyncError,
)
from crdt_sync._firebase_auth import (
CredentialStore,
FileCredentialStore,
FirebaseAuthError,
FirebaseCredentials,
FirebaseTokenProvider,
MemoryCredentialStore,
)
from crdt_sync._github import GitHubSyncClient, GitHubSyncError, RepoNotFoundError
from crdt_sync._hlc import Hlc
from crdt_sync._log import Log, merge_logs
from crdt_sync._mirror import MirrorSyncClient
from crdt_sync._record import Field, Record, merge_field, merge_record
from crdt_sync._remote import RemoteNotFoundError, RemoteStore, RemoteSyncError
from crdt_sync._store import dump_log, load_log, read_log, write_log
from crdt_sync._sync import (
FileSyncStateStore,
LogCodec,
MemorySyncStateStore,
RevisionTracking,
SyncState,
SyncStateStore,
SyncTarget,
default_revs_path,
revision_of,
sync_log,
)
from crdt_sync._sync import sync_log
__all__ = [
"CONFIG_DIR",
"CONFIG_FILE",
"PASSWORD_FILE",
"ConfigError",
"CredentialStore",
"DatabaseNotFoundError",
"DeviceIdentity",
"Field",
"FileCredentialStore",
"FileSyncStateStore",
"FirebaseAuthError",
"FirebaseConfig",
"FirebaseCredentials",
"FirebaseSyncClient",
"FirebaseSyncError",
"FirebaseTokenProvider",
"GitHubSyncClient",
"GitHubSyncError",
"Hlc",
"Log",
"LogCodec",
"MemoryCredentialStore",
"MemorySyncStateStore",
"MirrorSyncClient",
"Record",
"RemoteNotFoundError",
"RemoteStore",
"RemoteSyncError",
"RepoNotFoundError",
"RevisionTracking",
"SyncState",
"SyncStateStore",
"SyncTarget",
"credential_store_for",
"default_revs_path",
"dump_log",
"firebase_client_for",
"load_device_identity",
"load_log",
"merge_field",
"merge_logs",
"merge_record",
"mirror_client_for",
"read_log",
"revision_of",
"sync_log",
"write_log",
]

View File

@ -1,71 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""Turning a Firebase auth response into a message, and reading its verdict.
Split from :mod:`crdt_sync._firebase_auth`, which keeps the token provider.
identitytoolkit reports failures as a nested JSON error whose useful part is
buried; these turn that into something a log line can carry, and decide
whether the failure is permanent (the refresh token is dead and the user must
sign in again) or worth retrying.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from crdt_sync._remote import RemoteSyncError
if TYPE_CHECKING:
import requests as _requests
class FirebaseAuthError(RemoteSyncError):
"""Raised for an authentication failure the caller must not ignore.
A :class:`crdt_sync.RemoteSyncError` so callers that only care about
"sync is broken" can catch one type, while a settings screen can single
this out to say "your password is wrong" rather than "the network is
down".
"""
def _reason(response: _requests.Response) -> str:
"""Return Google's machine-readable error reason, in parentheses.
Pulls ``INVALID_PASSWORD`` / ``TOKEN_EXPIRED`` / ``USER_DISABLED`` out of
the body so the raised message says what actually went wrong rather than
just reporting a status code.
"""
try:
data = response.json()
except ValueError:
# A non-JSON body (a proxy error page, say): the status code is all
# the detail there is.
return ""
if not isinstance(data, dict):
return ""
error = data.get("error")
if isinstance(error, dict):
return f" ({error.get('message')})"
if isinstance(error, str):
return f" ({error})"
return ""
#: Reasons that mean the refresh token is permanently dead. Only these clear
#: the session: a network error or a 5xx must not, because a device that signs
#: itself out whenever the wifi drops needs a manual sign-in to recover, which
#: is a worse failure than the stale banner this guards against.
_TERMINAL_AUTH_REASONS = (
"TOKEN_EXPIRED",
"USER_DISABLED",
"USER_NOT_FOUND",
"INVALID_REFRESH_TOKEN",
"INVALID_GRANT_TYPE",
"MISSING_REFRESH_TOKEN",
)
def _is_revoked(error: FirebaseAuthError) -> bool:
"""Return whether ``error`` means the refresh token is permanently dead."""
message = str(error)
return any(reason in message for reason in _TERMINAL_AUTH_REASONS)

View File

@ -1,199 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""The one place every app learns how to reach Firebase.
Before this module each caller re-read ``~/.config/crdt-sync/firebase.json``
by hand, so a change to the schema meant editing every consumer, and each
consumer got to invent its own half of the credential-cache convention. The
apps share one Firebase project and one account, so they should share one
loader.
Two things live here:
* :class:`FirebaseConfig` -- the parsed, validated config file.
* :func:`firebase_client_for` -- the whole "config file to usable client"
path, including the per-app credential cache that keeps a fresh process
from re-authenticating. ``wake_alarm``'s PC side runs every minute and
``diet_guard``'s every 15, so caching the refresh token is what makes the
sign-in cost negligible rather than constant.
Deliberately *not* here: any choice about which backend an app uses. That
stays a constructor change at the call site, per :mod:`crdt_sync._remote`.
"""
from __future__ import annotations
from dataclasses import dataclass
import json
from pathlib import Path
from typing import TYPE_CHECKING
from crdt_sync._firebase import _DEFAULT_TIMEOUT_SECONDS, FirebaseSyncClient
from crdt_sync._firebase_auth import FileCredentialStore, FirebaseTokenProvider
from crdt_sync._mirror import MirrorSyncClient
if TYPE_CHECKING:
from crdt_sync._remote import RemoteStore
CONFIG_DIR = Path.home() / ".config" / "crdt-sync"
CONFIG_FILE = CONFIG_DIR / "firebase.json"
PASSWORD_FILE = CONFIG_DIR / "password"
# The scaffold ships "_comment_*" keys explaining where each value comes
# from. They are documentation for a human filling the file in, not config.
_COMMENT_PREFIX = "_comment"
class ConfigError(Exception):
"""The shared Firebase config is missing, malformed or unfilled."""
@dataclass(frozen=True)
class FirebaseConfig:
"""Everything needed to reach the shared Firebase project.
``api_key`` is not a secret -- it ships inside the Android APKs, and the
security rules are what actually protect the data. The password is.
"""
api_key: str
database_url: str
project_id: str
uid: str
email: str
@classmethod
def load(cls, path: Path | None = None) -> FirebaseConfig:
"""Return the config at ``path``, defaulting to :data:`CONFIG_FILE`.
Raises:
ConfigError: If the file is absent, unparsable, missing a key, or
still holding a scaffold placeholder -- each reported with the
specific field at fault, because the alternative is a
confusing auth failure much later.
"""
source = path if path is not None else CONFIG_FILE
try:
raw = json.loads(source.read_text(encoding="utf-8"))
except FileNotFoundError as exc:
msg = f"{source} does not exist; see utils/firebase/README.md"
raise ConfigError(msg) from exc
except OSError as exc:
msg = f"{source} could not be read: {exc}"
raise ConfigError(msg) from exc
except ValueError as exc:
msg = f"{source} is not valid JSON: {exc}"
raise ConfigError(msg) from exc
if not isinstance(raw, dict):
msg = f"{source} must contain a JSON object"
raise ConfigError(msg)
data = {k: v for k, v in raw.items() if not k.startswith(_COMMENT_PREFIX)}
fields = {
"api_key": "apiKey",
"database_url": "databaseUrl",
"project_id": "projectId",
"uid": "uid",
"email": "email",
}
missing = [key for key in fields.values() if not str(data.get(key, "")).strip()]
if missing:
msg = f"{source} is missing or has empty: {', '.join(missing)}"
raise ConfigError(msg)
unfilled = [key for key in fields.values() if "PASTE_" in str(data[key])]
if unfilled:
msg = f"{source} still holds the placeholder for: {', '.join(unfilled)}"
raise ConfigError(msg)
return cls(**{attr: str(data[key]) for attr, key in fields.items()})
def read_password(self, path: Path | None = None) -> str:
"""Return the sync account's password.
Stripped, because a text editor that appends a trailing newline would
otherwise turn into an authentication failure with no visible cause.
"""
source = path if path is not None else PASSWORD_FILE
try:
password = source.read_text(encoding="utf-8").strip()
except OSError as exc:
msg = f"{source} could not be read: {exc}"
raise ConfigError(msg) from exc
if not password or "PASTE_" in password:
msg = f"{source} is empty or still holds the scaffold placeholder"
raise ConfigError(msg)
return password
def credential_store_for(app_name: str) -> FileCredentialStore:
"""Return the credential cache for ``app_name``.
Per app rather than shared: two apps signing in as the same account still
refresh tokens on their own schedules, and a single file written by
several processes at once is a corruption waiting to happen.
"""
return FileCredentialStore(
Path.home() / ".config" / app_name / "firebase_auth.json"
)
def firebase_client_for(
app_name: str,
*,
config: FirebaseConfig | None = None,
timeout_seconds: float = _DEFAULT_TIMEOUT_SECONDS,
) -> FirebaseSyncClient:
"""Return a signed-in client for ``app_name``.
Reuses the cached refresh token when there is one and signs in with the
password only when there is not, so the common path costs no
authentication round trip at all.
Args:
app_name: Names the credential cache, e.g. ``"diet_guard"``.
config: Overrides the on-disk config; for tests.
timeout_seconds: Per-request network timeout. Without this the
client fell back to its own default and a caller's tighter
budget reached the GitHub backend only.
Raises:
ConfigError: If the shared config is unusable.
FirebaseAuthError: If the credentials are rejected.
"""
resolved = config if config is not None else FirebaseConfig.load()
auth = FirebaseTokenProvider(resolved.api_key, credential_store_for(app_name))
if not auth.has_session():
auth.sign_in(resolved.email, resolved.read_password())
return FirebaseSyncClient(
resolved.database_url, auth, timeout_seconds=timeout_seconds
)
def mirror_client_for(
app_name: str,
github_client: RemoteStore,
*,
config: FirebaseConfig | None = None,
) -> MirrorSyncClient:
"""Return a Firebase-primary client that still mirrors to ``github_client``.
This is what an app uses *during* the cutover. Firebase is authoritative,
so its failures fail the tick; GitHub is kept in step but never allowed
to. Reads union both, which is what lets devices cut over one at a time
without a migrated PC losing sight of an un-migrated phone.
Rolling back is deleting the call: pass the GitHub client straight to
:func:`crdt_sync.sync_log` again. Retiring the mirror is swapping this for
:func:`firebase_client_for`. Both are one-line changes at the call site,
which is the whole point of the :class:`~crdt_sync.RemoteStore` seam.
Args:
app_name: Names the credential cache, e.g. ``"diet_guard"``.
github_client: The existing GitHub client, kept as the fallback.
config: Overrides the on-disk config; for tests.
"""
return MirrorSyncClient(
primary=firebase_client_for(app_name, config=config),
mirror=github_client,
)

View File

@ -1,158 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""Firebase credentials and the stores that persist them.
Split from :mod:`crdt_sync._firebase_auth`, which keeps the token provider.
A credential set is an ID token, a refresh token and an expiry; the stores
decide where it survives a restart -- memory for tests, a mode-0600 file for
real installs.
Re-exported from :mod:`crdt_sync._firebase_auth`, so existing imports work.
"""
from __future__ import annotations
from dataclasses import dataclass
import datetime as dt
import json
import logging
import os
from pathlib import Path
import tempfile
from typing import Protocol
_logger = logging.getLogger(__name__)
# Refresh this long before the token actually expires. An ID token lives ~1h;
# refreshing early means a tick that starts just under the wire still finishes
# with a valid token rather than 401ing halfway through a multi-file push.
_REFRESH_SKEW = dt.timedelta(minutes=5)
@dataclass(frozen=True)
class FirebaseCredentials:
"""One device's session: a short-lived ID token plus its refresh token.
Attributes:
----------
id_token:
The bearer credential for Realtime Database requests. Expires fast.
refresh_token:
The long-lived credential. **This is the secret worth protecting.**
expires_at:
When ``id_token`` stops being accepted, in UTC.
"""
id_token: str
refresh_token: str
expires_at: dt.datetime
def is_expired_at(self, now: dt.datetime) -> bool:
"""Return whether the token is expired, or close enough to it."""
return now + _REFRESH_SKEW >= self.expires_at
def to_json(self) -> dict[str, str]:
"""Return a JSON-serializable form."""
return {
"id_token": self.id_token,
"refresh_token": self.refresh_token,
"expires_at": self.expires_at.isoformat(),
}
@classmethod
def from_json(cls, data: dict[str, str]) -> FirebaseCredentials:
"""Rebuild credentials from :meth:`to_json` output."""
return cls(
id_token=data["id_token"],
refresh_token=data["refresh_token"],
expires_at=dt.datetime.fromisoformat(data["expires_at"]),
)
class CredentialStore(Protocol):
"""Where a device keeps its credentials between runs."""
def load(self) -> FirebaseCredentials | None:
"""Return the stored credentials, or ``None`` if not signed in."""
def save(self, credentials: FirebaseCredentials) -> None:
"""Persist ``credentials``."""
def clear(self) -> None:
"""Forget any stored credentials."""
class MemoryCredentialStore:
"""A :class:`CredentialStore` for tests and one-shot scripts."""
def __init__(self, credentials: FirebaseCredentials | None = None) -> None:
"""Start holding ``credentials`` (or nothing)."""
self._credentials = credentials
def load(self) -> FirebaseCredentials | None:
"""Return the held credentials."""
return self._credentials
def save(self, credentials: FirebaseCredentials) -> None:
"""Replace the held credentials."""
self._credentials = credentials
def clear(self) -> None:
"""Drop the held credentials."""
self._credentials = None
class FileCredentialStore:
"""A :class:`CredentialStore` backed by a ``0600`` JSON file.
Persistence is not an optimisation here: ``wake_alarm``'s PC side is a
fresh process every minute and ``diet_guard``'s every 15 minutes, so an
in-memory store would re-authenticate thousands of times a day.
"""
def __init__(self, path: Path) -> None:
"""Store credentials at ``path``, creating parent dirs as needed."""
self._path = path
def load(self) -> FirebaseCredentials | None:
"""Return the stored credentials, or ``None`` if the file is absent.
A corrupt or truncated file (an interrupted write) reads as "not
signed in" rather than raising: the caller's next step is to sign in
again, which repairs it.
"""
try:
data = json.loads(self._path.read_text(encoding="utf-8"))
except OSError, ValueError:
return None
try:
return FirebaseCredentials.from_json(data)
except KeyError, TypeError, ValueError:
return None
def save(self, credentials: FirebaseCredentials) -> None:
"""Write ``credentials`` atomically, readable only by this user."""
self._path.parent.mkdir(parents=True, exist_ok=True)
# A unique temp name per write, not a fixed ``.tmp``: two processes
# (a long-running stream and a periodic sync) share one cache, and a
# fixed name let one rename the other's half-written file into place.
# mkstemp creates it 0600 from the outset -- writing then chmod'ing
# would leave the refresh token world-readable for the gap between.
fd, name = tempfile.mkstemp(
dir=self._path.parent, prefix=f".{self._path.name}.", suffix=".tmp"
)
temp = Path(name)
try:
with os.fdopen(fd, "w", encoding="utf-8") as handle:
handle.write(json.dumps(credentials.to_json()))
temp.replace(self._path)
except BaseException:
temp.unlink(missing_ok=True)
raise
def clear(self) -> None:
"""Delete the credentials file if it exists."""
self._path.unlink(missing_ok=True)
def _utcnow() -> dt.datetime:
return dt.datetime.now(dt.UTC)

View File

@ -1,88 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""This device's stable sync identity.
Every device that pushes into a namespace needs an id that is *its own*: it
is the directory segment its log is pushed under
(``<path_prefix>/<device_id>/<filename>``), the revision key, and the node
component baked into every HLC stamp it writes (``<iso>-<counter>-<id>``).
Two devices sharing an id overwrite each other's pushed file on every tick,
so the id must be per-*install*, not per-role: a fixed ``"pc"``/``"phone"``
constant silently collides the moment a second machine takes the same role,
and a reinstall inherits the previous install's CRDT identity.
A migration from a role constant to a persisted uuid cannot rewrite history.
Stamps already written keep the old id, and the old path keeps the log that
was pushed under it -- so a device that has just switched ids must still
recognise its *former* id as itself. That is what [legacy_id] carries, and
why [DeviceIdentity.own_ids] exists: skip-own-writes has to test membership
of that set rather than equality with one id, or the device re-downloads and
re-merges its own pre-migration history as though a peer had written it.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING
import uuid
if TYPE_CHECKING:
from pathlib import Path
@dataclass(frozen=True)
class DeviceIdentity:
"""This device's current sync id, plus the id it used to push under.
Attributes:
device_id: The id to push, stamp and key revisions under from now on.
legacy_id: The id this device pushed under before the migration to
persisted uuids, or ``None`` once its old path has been reclaimed.
"""
device_id: str
legacy_id: str | None = None
@property
def own_ids(self) -> frozenset[str]:
"""Every id that means "this device", for skip-own-writes checks."""
if self.legacy_id is None:
return frozenset({self.device_id})
return frozenset({self.device_id, self.legacy_id})
def is_own(self, other_device_id: str) -> bool:
"""Return whether [other_device_id] is one of this device's own ids."""
return other_device_id in self.own_ids
def load_device_identity(path: Path, *, legacy_id: str | None = None) -> DeviceIdentity:
"""Return the id persisted at [path], creating it on first call.
The file holds exactly one uuid4 and is written once; every later call
returns the same value, which is what makes the id stable across restarts
(and is why it must live in durable state, not a temp dir).
Args:
path: File holding this device's uuid. Parent dirs are created.
legacy_id: The role constant this device pushed under before
migrating, recorded on the returned identity so skip-own-writes
still recognises the old path as this device's own. Pass ``None``
once that path has been reclaimed.
Returns:
This device's [DeviceIdentity].
"""
try:
existing = path.read_text(encoding="utf-8").strip()
except FileNotFoundError:
existing = ""
if existing:
return DeviceIdentity(device_id=existing, legacy_id=legacy_id)
# Deliberately not guarded: if the state directory cannot be created the
# id cannot be persisted, and returning an unpersisted one would mint a
# fresh uuid on every restart, stranding a device directory each time.
minted = str(uuid.uuid4())
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(f"{minted}\n", encoding="utf-8")
return DeviceIdentity(device_id=minted, legacy_id=legacy_id)

View File

@ -1,26 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""The errors the Firebase sync backend raises.
Split from :mod:`crdt_sync._firebase` for the 250-line cap, mirroring
:mod:`crdt_sync._gherrors` on the GitHub side.
Re-exported from :mod:`crdt_sync._firebase`, so existing imports keep working.
"""
from __future__ import annotations
from crdt_sync._remote import RemoteNotFoundError, RemoteSyncError
class FirebaseSyncError(RemoteSyncError):
"""Raised for an RTDB failure the caller must not silently ignore."""
class DatabaseNotFoundError(FirebaseSyncError, RemoteNotFoundError):
"""Raised when the database is unreachable or the uid is not allowed.
The Firebase counterpart of :class:`crdt_sync.RepoNotFoundError`: "the
database URL is wrong, or the security rules reject this account", as
opposed to "nothing has been pushed to that path yet", which is benign
and surfaces as ``None``.
"""

View File

@ -1,101 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""Escaping arbitrary strings into Firebase Realtime Database keys.
Split from :mod:`crdt_sync._firebase`, which keeps the client. RTDB forbids
``.``, ``$``, ``#``, ``[``, ``]`` and ``/`` in a key, but the app stores
record ids and file paths that contain them, so every key is escaped on the
way in and unescaped on the way out. The pair has to round-trip exactly or a
device silently stops seeing its own writes.
Re-exported from :mod:`crdt_sync._firebase`, so existing imports keep working.
"""
from __future__ import annotations
# Characters Realtime Database forbids in a key, plus ``~`` itself because it
# is this module's escape character. ``/`` is absent deliberately: it is the
# path separator, handled by splitting into segments before escaping.
_ESCAPES = {
"~": "~7E",
".": "~2E",
"$": "~24",
"#": "~23",
"[": "~5B",
"]": "~5D",
}
_UNESCAPES = {escape: char for char, escape in _ESCAPES.items()}
_ESCAPE_LENGTH = 3
def encode_key(segment: str) -> str:
"""Escape one path segment into a legal Realtime Database key.
RTDB rejects ``. $ # [ ] /`` in keys, and the REST API's trailing
``.json`` is a *format suffix* rather than part of the path -- so a
filename like ``log.json`` cannot be stored verbatim. The mapping is a
reversible ``~XX`` escape (``log.json`` -> ``log~2Ejson``) rather than a
lossy "strip the extension", because callers see these names again:
todo-app lists *filenames*, not device directories, and must keep getting
``<uuid>.json`` back.
``~`` is the escape character because it is legal in RTDB keys and
unreserved in URLs, so the escaped form needs no percent-encoding --
which the server would otherwise decode back into the illegal character.
Parameters
----------
segment : str
One path segment, with no ``/`` in it.
Returns:
-------
str
The escaped key.
"""
return "".join(_ESCAPES.get(char, char) for char in segment)
def decode_key(key: str) -> str:
"""Reverse :func:`encode_key`.
Parameters
----------
key : str
An escaped Realtime Database key.
Returns:
-------
str
The original segment.
"""
out: list[str] = []
index = 0
while index < len(key):
escape = key[index : index + _ESCAPE_LENGTH]
char = _UNESCAPES.get(escape)
if char is None:
out.append(key[index])
index += 1
else:
out.append(char)
index += _ESCAPE_LENGTH
return "".join(out)
def encode_path(path: str) -> str:
"""Escape every segment of a ``/``-separated logical path.
Parameters
----------
path : str
A logical path such as ``"todo-sync/notes/abc.json"``.
Returns:
-------
str
The path with each segment escaped, empty segments dropped.
"""
return "/".join(encode_key(part) for part in path.split("/") if part)

View File

@ -1,248 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""Firebase Realtime Database as dumb keyed storage, over its REST API.
The RTDB REST endpoints are plain HTTPS, so this works identically under
systemd, in a venv, and alongside the Dart client on Android -- and the two
write byte-identical payloads, so a device on either implementation reads the
other's.
Why RTDB rather than Firestore: on the Spark (free) plan RTDB bills only
storage and bandwidth, with **no per-operation quota**, so a misbehaving sync
loop can never exhaust a daily budget and silently stop working mid-day.
Mirrors ``crdt_sync_dart``'s ``lib/src/firebase_client.dart``; keep the two in
step.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import requests as _requests
from crdt_sync._fberrors import DatabaseNotFoundError, FirebaseSyncError
from crdt_sync._fbkeys import decode_key, encode_key, encode_path
# Named explicitly so the autofixer cannot prune an import that exists for
# its re-export: the tests import encode_key/decode_key from this module.
__all__ = [
"DatabaseNotFoundError",
"FirebaseSyncClient",
"FirebaseSyncError",
"decode_key",
"encode_key",
"encode_path",
]
from crdt_sync._http import new_session
from crdt_sync._remote import RemoteSyncError
# Bound to the name ``requests`` so the call sites below -- and the tests
# that patch ``<module>.requests.<verb>`` -- are unchanged by pooling.
requests = new_session()
if TYPE_CHECKING:
from crdt_sync._firebase_auth import FirebaseTokenProvider
_DEFAULT_TIMEOUT_SECONDS = 15
_HTTP_UNAUTHORIZED = 401
_HTTP_FORBIDDEN = 403
class FirebaseSyncClient:
"""Realtime Database seen through the same contract as GitHub.
Reads and writes UTF-8 text blobs stored as JSON string leaves, so the
payload is byte-identical to what the GitHub backend stored and the two
can be mirrored against each other during migration.
"""
def __init__(
self,
database_url: str,
auth: FirebaseTokenProvider,
*,
timeout_seconds: float = _DEFAULT_TIMEOUT_SECONDS,
) -> None:
"""Create a client for ``database_url`` authenticated by ``auth``.
Args:
database_url: The database's HTTPS origin.
auth: Mints the ID token every request carries.
timeout_seconds: Per-request network timeout.
"""
self._database_url = database_url.rstrip("/")
self.auth = auth
self._timeout_seconds = timeout_seconds
def _url(self, path: str) -> str:
return f"{self._database_url}/{encode_path(path)}.json"
def _params(self, **extra: str) -> dict[str, str]:
return {"auth": self.auth.id_token(), **extra}
def _raise(self, what: str, response: _requests.Response) -> None:
"""Raise the right error type for a non-2xx response.
401/403 mean the rules rejected this uid or the token is bad.
Everything else -- including a Spark quota exhaustion, which answers
with an error rather than billing you -- is a plain
:class:`FirebaseSyncError`. Nothing is ever swallowed: a
quota-exhausted database that silently returned "no data" would look
exactly like "nothing to sync".
"""
detail = response.text.strip()
if response.status_code in (_HTTP_UNAUTHORIZED, _HTTP_FORBIDDEN):
msg = (
f"{what} rejected: HTTP {response.status_code} {detail} -- "
f"the database URL or the security rules do not allow this "
f"account"
)
raise DatabaseNotFoundError(msg)
msg = f"{what} failed: HTTP {response.status_code} {detail}"
raise FirebaseSyncError(msg)
def _get(self, path: str, **params: str) -> _requests.Response:
try:
return requests.get(
self._url(path),
params=self._params(**params),
timeout=self._timeout_seconds,
)
except _requests.RequestException as exc:
msg = f"network error reading {path}"
raise FirebaseSyncError(msg) from exc
def list_directory(self, path: str) -> list[str]:
"""Return the entry names directly under ``path``.
Uses ``shallow=true``, which returns ``{key: true}`` without any
values -- so a listing costs bytes rather than the whole subtree.
That is the difference between a sync tick costing hundreds of bytes
and hundreds of kilobytes.
Raises:
DatabaseNotFoundError: If the rules reject this account.
FirebaseSyncError: For any other non-2xx response or network
error.
"""
response = self._get(path, shallow="true")
if not response.ok:
self._raise(f"listing {path}", response)
data = response.json()
if not isinstance(data, dict):
return []
return [decode_key(key) for key in data]
def get_file_text(self, path: str) -> str | None:
"""Return the text stored at ``path``, or ``None`` if absent.
Raises:
DatabaseNotFoundError: If the rules reject this account.
FirebaseSyncError: If the value is not a text blob, or for any
other non-2xx response or network error.
"""
response = self._get(path)
if not response.ok:
self._raise(f"reading {path}", response)
data = response.json()
if data is None:
return None
if not isinstance(data, str):
msg = (
f"value at {path} is {type(data).__name__}, not the expected text blob"
)
raise FirebaseSyncError(msg)
return data
def put_file_text(self, path: str, text: str, *, message: str) -> None:
"""Write ``text`` at ``path``.
Args:
path: A logical path such as ``"ns/devices/pc/log.json"``.
text: The full new content.
message: Ignored -- RTDB has no commit log to attach a reason to.
Present so this matches the ``RemoteStore`` contract.
Raises:
FirebaseSyncError: On any non-2xx response or network error.
"""
del message
try:
response = requests.put(
self._url(path),
params=self._params(),
json=text,
timeout=self._timeout_seconds,
)
except _requests.RequestException as exc:
msg = f"network error writing {path}"
raise FirebaseSyncError(msg) from exc
if not response.ok:
self._raise(f"writing {path}", response)
def get_string_map(self, path: str) -> dict[str, str]:
"""Return the map at ``path`` as ``key -> text``, empty if absent.
Tolerates a non-map or malformed value rather than raising: this
backs the revision cache, which is an *optimisation*. A corrupt revs
node must degrade into "fetch everything", never into a failed sync.
Raises:
FirebaseSyncError: Only for a transport-level failure.
"""
response = self._get(path)
if not response.ok:
self._raise(f"reading {path}", response)
data = response.json()
if not isinstance(data, dict):
return {}
return {
decode_key(key): value
for key, value in data.items()
if isinstance(value, str)
}
def delete_file(self, path: str, *, message: str = "crdt_sync: delete") -> None:
"""Delete ``path``. A no-op if it does not exist.
Args:
path: A logical path.
message: Ignored, as for :meth:`put_file_text`.
Raises:
FirebaseSyncError: On any non-2xx response or network error.
"""
del message
try:
response = requests.delete(
self._url(path),
params=self._params(),
timeout=self._timeout_seconds,
)
except _requests.RequestException as exc:
msg = f"network error deleting {path}"
raise FirebaseSyncError(msg) from exc
if not response.ok:
self._raise(f"deleting {path}", response)
def can_access_remote(self) -> bool:
"""Return whether the credential can reach the database.
Never raises -- a bad URL, a rejected token, a missing session or a
network failure all report ``False``, so a settings "Test connection"
button cannot blow up.
"""
try:
response = requests.get(
self._url(""),
params=self._params(shallow="true"),
timeout=self._timeout_seconds,
)
except _requests.RequestException:
return False
except RemoteSyncError:
# Includes FirebaseAuthError: "cannot get a token" is exactly
# "cannot access the remote".
return False
return response.ok

View File

@ -1,245 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""Firebase Authentication over its REST API, with no SDK dependency.
The Firebase Admin SDK authenticates with a service-account key, which
bypasses security rules entirely and would have to be distributed to every
device. These are single-user personal apps, so devices instead sign in as
one ordinary user and hold a refresh token -- the same path the Flutter side
takes, so both platforms behave identically.
Mirrors ``crdt_sync_dart``'s ``lib/src/firebase_auth_rest.dart``; keep the
two in step.
"""
from __future__ import annotations
import datetime as dt
import requests as _requests
from crdt_sync._autherrors import FirebaseAuthError, _is_revoked, _reason
from crdt_sync._credentials import (
CredentialStore,
FileCredentialStore,
FirebaseCredentials,
MemoryCredentialStore,
_utcnow,
)
from crdt_sync._http import new_session
__all__ = [
"CredentialStore",
"FileCredentialStore",
"FirebaseAuthError",
"FirebaseCredentials",
"FirebaseTokenProvider",
"MemoryCredentialStore",
]
# Bound to the name ``requests`` so the call sites below -- and the tests
# that patch ``<module>.requests.<verb>`` -- are unchanged by pooling.
requests = new_session()
_SIGN_IN_BASE = "https://identitytoolkit.googleapis.com/v1"
_REFRESH_BASE = "https://securetoken.googleapis.com/v1"
_DEFAULT_TIMEOUT_SECONDS = 15
class FirebaseTokenProvider:
"""Signs in and keeps a valid ID token available, refreshing as needed.
``api_key`` is the project's public Web API key -- not a secret, and safe
to ship inside an APK. The actual credential is the refresh token held by
the store.
"""
def __init__(
self,
api_key: str,
store: CredentialStore,
*,
timeout_seconds: float = _DEFAULT_TIMEOUT_SECONDS,
clock: object = None,
) -> None:
"""Create a provider for ``api_key`` persisting through ``store``.
Args:
api_key: The project's public Web API key.
store: Where the refresh token lives between runs.
timeout_seconds: Per-request network timeout.
clock: A zero-arg callable returning "now" (tests inject one).
"""
self._api_key = api_key
self._store = store
self._timeout_seconds = timeout_seconds
self._clock = clock if callable(clock) else _utcnow
self._cached: FirebaseCredentials | None = None
def sign_in(self, email: str, password: str) -> None:
"""Exchange an email/password for a session and persist it.
Called once per device from a setup command; every later run reuses
the stored refresh token.
Raises:
FirebaseAuthError: If the credentials are rejected.
"""
body = self._post(
f"{_SIGN_IN_BASE}/accounts:signInWithPassword?key={self._api_key}",
{"email": email, "password": password, "returnSecureToken": True},
"sign in",
)
self._adopt(
id_token=body["idToken"],
refresh_token=body["refreshToken"],
expires_in_seconds=body["expiresIn"],
)
def sign_in_with_google(self, id_token: str, expected_uid: str = "") -> str:
"""Exchange a Google ID token for a session and persist it.
Mirrors ``crdt_sync_dart``'s ``signInWithGoogle``. Used to reseed a
machine's session after the sync account moved to Google-only sign-in:
the refresh token this stores is what keeps a headless job working
between runs, so the browser flow is needed once per machine, not once
per tick.
Deliberately *not* a service-account key. That would authenticate as
an admin and bypass the security rules entirely, so the pinned-uid
rule that protects this data would stop applying to every PC write.
Args:
id_token: A Google ID token whose audience is this project.
expected_uid: When given, asserted against the uid Firebase
returns. A mismatch raises without storing anything -- an
unlinked Google identity is *signed up* rather than refused,
and that session would then be denied every read and write.
Returns:
The email Firebase reports for the account.
Raises:
FirebaseAuthError: If the token is rejected, or resolves to a uid
other than ``expected_uid``.
"""
body = self._post(
f"{_SIGN_IN_BASE}/accounts:signInWithIdp?key={self._api_key}",
{
# The IdP credential travels form-encoded, not as JSON -- an
# identitytoolkit quirk. Sending it as a field returns
# INVALID_IDP_RESPONSE, which reads like a bad token.
"postBody": f"id_token={id_token}&providerId=google.com",
"requestUri": "http://localhost",
"returnSecureToken": True,
},
"sign in with Google",
)
uid = body.get("localId", "")
if expected_uid and uid != expected_uid:
msg = (
f"signed in as the wrong account: Google resolved to uid "
f"{uid!r}, but this data belongs to {expected_uid!r}"
)
raise FirebaseAuthError(msg)
self._adopt(
id_token=body["idToken"],
refresh_token=body["refreshToken"],
expires_in_seconds=body["expiresIn"],
)
return str(body.get("email", ""))
def id_token(self) -> str:
"""Return a currently-valid ID token, refreshing if needed.
Never returns a stale token and never silently no-ops: a sync that
quietly stops syncing is the failure mode this design exists to
prevent.
Raises:
FirebaseAuthError: If no session is stored, or the refresh token
has been revoked.
"""
credentials = self._load()
if credentials is None:
msg = "not signed in: no stored refresh token for this device"
raise FirebaseAuthError(msg)
if not credentials.is_expired_at(self._clock()):
return credentials.id_token
return self._refresh(credentials.refresh_token)
def has_session(self) -> bool:
"""Return whether this device has a stored session at all.
Distinguishes "sync is not configured" -- a normal state for
``screen-locker`` -- from "sync is configured and broken", an error.
"""
return self._load() is not None
def sign_out(self) -> None:
"""Forget the stored session, so the next token request fails."""
self._cached = None
self._store.clear()
def _load(self) -> FirebaseCredentials | None:
if self._cached is None:
self._cached = self._store.load()
return self._cached
def _refresh(self, refresh_token: str) -> str:
try:
body = self._post(
f"{_REFRESH_BASE}/token?key={self._api_key}",
{"grant_type": "refresh_token", "refresh_token": refresh_token},
"refresh the session",
)
except FirebaseAuthError as error:
# A revoked refresh token never becomes valid again, so keeping it
# is what let a dead device report "connected" and then fail every
# sync with TOKEN_EXPIRED. Drop it so has_session() is honest.
if _is_revoked(error):
self.sign_out()
raise
return self._adopt(
id_token=body["id_token"],
# A refresh may hand back a rotated token; keep the new one.
refresh_token=body["refresh_token"],
expires_in_seconds=body["expires_in"],
)
def _adopt(
self,
*,
id_token: str,
refresh_token: str,
expires_in_seconds: str,
) -> str:
credentials = FirebaseCredentials(
id_token=id_token,
refresh_token=refresh_token,
expires_at=self._clock() + dt.timedelta(seconds=int(expires_in_seconds)),
)
self._cached = credentials
self._store.save(credentials)
return id_token
def _post(
self,
url: str,
payload: dict[str, object],
what: str,
) -> dict[str, str]:
try:
response = requests.post(
url,
json=payload,
timeout=self._timeout_seconds,
)
except _requests.RequestException as exc:
msg = f"network error trying to {what}"
raise FirebaseAuthError(msg) from exc
if not response.ok:
reason = _reason(response)
msg = f"failed to {what}: HTTP {response.status_code}{reason}"
raise FirebaseAuthError(msg)
return response.json()

View File

@ -1,31 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""The errors the GitHub sync backend raises.
Split from :mod:`crdt_sync._github` for the 250-line cap. Kept separate so a
caller can catch them without importing the client -- and so the client file
is the client, not its taxonomy.
Re-exported from :mod:`crdt_sync._github`, so existing imports keep working.
"""
from __future__ import annotations
from crdt_sync._remote import RemoteNotFoundError, RemoteSyncError
class GitHubSyncError(RemoteSyncError):
"""Raised for a GitHub API failure the caller must not silently ignore."""
class RepoNotFoundError(GitHubSyncError, RemoteNotFoundError):
"""Raised when the configured repo itself is unreachable.
Distinguished from a path-404 (nothing pushed to that path yet, which is
benign -- it just means no other device has synced before) so the caller
can tell "the repo name is wrong or the token isn't scoped to it" apart
from "no other device has synced yet".
Derives from both so existing ``except GitHubSyncError`` handlers still
cover it *and* backend-neutral callers can catch "the remote itself is
missing" without naming GitHub.
"""

View File

@ -1,4 +1,3 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""Minimal GitHub Contents API client used as dumb file storage.
GitHub is used purely as file storage via the REST Contents API, not a git
@ -12,21 +11,27 @@ from __future__ import annotations
import base64
import requests as _requests
from crdt_sync._gherrors import GitHubSyncError, RepoNotFoundError
from crdt_sync._http import new_session
# Bound to the name ``requests`` so the call sites below -- and the tests
# that patch ``<module>.requests.<verb>`` -- are unchanged by pooling.
requests = new_session()
import requests
_API_BASE = "https://api.github.com"
_HTTP_NOT_FOUND = 404
_DEFAULT_TIMEOUT_SECONDS = 15
class GitHubSyncError(Exception):
"""Raised for a GitHub API failure the caller must not silently ignore."""
class RepoNotFoundError(GitHubSyncError):
"""Raised when the configured repo itself is unreachable.
Distinguished from a path-404 (nothing pushed to that path yet, which is
benign -- it just means no other device has synced before) so the caller
can tell "the repo name is wrong or the token isn't scoped to it" apart
from "no other device has synced yet".
"""
class GitHubSyncClient:
"""Thin wrapper around the subset of the Contents API sync needs."""
@ -57,14 +62,14 @@ class GitHubSyncClient:
def _contents_url(self, path: str) -> str:
return f"{_API_BASE}/repos/{self._owner}/{self._repo}/contents/{path}"
def _get(self, path: str) -> _requests.Response:
def _get(self, path: str) -> requests.Response:
try:
return requests.get(
self._contents_url(path),
headers=self._headers,
timeout=self._timeout_seconds,
)
except _requests.RequestException as exc:
except requests.RequestException as exc:
msg = f"network error reading {path}"
raise GitHubSyncError(msg) from exc
@ -75,7 +80,7 @@ class GitHubSyncClient:
headers=self._headers,
timeout=self._timeout_seconds,
)
except _requests.RequestException:
except requests.RequestException:
return False
return response.ok
@ -155,7 +160,7 @@ class GitHubSyncClient:
json=payload,
timeout=self._timeout_seconds,
)
except _requests.RequestException as exc:
except requests.RequestException as exc:
msg = f"network error pushing {path}"
raise GitHubSyncError(msg) from exc
if not response.ok:
@ -194,17 +199,6 @@ class GitHubSyncClient:
Hits the bare repo endpoint, so it succeeds even before any file has
been pushed. Never raises -- a network failure or missing repo is
reported as ``False``.
The GitHub-era name for :meth:`can_access_remote`; kept because app
settings screens call it. Both do the same probe.
"""
return self._repo_exists()
def can_access_remote(self) -> bool:
"""Return whether the token can reach the remote (a connection test).
The :class:`crdt_sync.RemoteStore` spelling of
:meth:`can_access_repo`.
"""
return self._repo_exists()
@ -230,7 +224,7 @@ class GitHubSyncClient:
json={"message": message, "sha": sha},
timeout=self._timeout_seconds,
)
except _requests.RequestException as exc:
except requests.RequestException as exc:
msg = f"network error deleting {path}"
raise GitHubSyncError(msg) from exc
if not response.ok:

View File

@ -1,4 +1,3 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""Hybrid Logical Clock: a totally-ordered, monotonic per-node timestamp.
An HLC combines wall-clock time with a logical counter so that two ticks
@ -10,11 +9,11 @@ ties), without requiring synchronized clocks across devices.
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime
from datetime import datetime, timezone
import time
_ISO_PREFIX_LEN = 23 # len("YYYY-MM-DDTHH:MM:SS.mmm")
_EPOCH = datetime(1970, 1, 1, tzinfo=UTC)
_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
@dataclass(frozen=True, order=True)
@ -67,7 +66,7 @@ class Hlc:
comparison for the wall-clock component.
"""
seconds, millis = divmod(self.wall_time_ms, 1000)
dt = datetime.fromtimestamp(seconds, tz=UTC)
dt = datetime.fromtimestamp(seconds, tz=timezone.utc)
iso = dt.strftime("%Y-%m-%dT%H:%M:%S")
return f"{iso}.{millis:03d}Z-{self.counter:04x}-{self.node_id}"
@ -86,7 +85,9 @@ class Hlc:
if not sep:
msg = f"not a valid Hlc string: {text!r}"
raise ValueError(msg)
dt = datetime.strptime(iso_part, "%Y-%m-%dT%H:%M:%S.%f").replace(tzinfo=UTC)
dt = datetime.strptime(iso_part, "%Y-%m-%dT%H:%M:%S.%f").replace(
tzinfo=timezone.utc
)
delta = dt - _EPOCH
wall_time_ms = (
delta.days * 86_400_000 + delta.seconds * 1000 + delta.microseconds // 1000

View File

@ -1,58 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""Pooled HTTP sessions, so every request does not re-handshake.
Every call site in this package used the module-level ``requests.get`` /
``put`` / ``post`` / ``delete`` helpers, each of which opens a throwaway
connection: fresh DNS, TCP and TLS on every single request. Measured against
the two real backends on 2026-08-21 that costs 82ms per request to
``api.github.com`` and 51ms to Realtime Database -- on a 27-request sync tick,
most of the wall clock.
A :class:`requests.Session` keeps the connection alive between requests, and
is a drop-in for the module-level helpers: same verb names, same signatures.
So each transport module binds one of these to the name ``requests`` and its
call sites are unchanged.
**One session per module, deliberately not one global.** The tests patch the
verb on the module attribute (``patch.object(fb.requests, "get", ...)``), and
several use sequence-consuming ``side_effect`` lists. Were all three modules
to share a single object, a call from one would consume another's queued
response and the failure would look like a logic bug. Per-module also happens
to be per-host here -- GitHub, Realtime Database, and the token endpoint are
three different origins -- so pooling loses nothing by the split.
Not thread-safe, in that :class:`requests.Session` is not documented to be;
this package spawns no threads and its consumers drive sync from one.
"""
from __future__ import annotations
import requests
from requests.adapters import HTTPAdapter, Retry
_RETRY_TOTAL = 3
_RETRY_BACKOFF_SECONDS = 0.3
_RETRY_STATUSES = (429, 500, 502, 503, 504)
def new_session() -> requests.Session:
"""Return a connection-pooling session that retries transient GET failures.
Retries are restricted to ``GET``. A ``PUT`` here carries a Contents-API
``sha`` (and a Realtime Database write is a full-node replace); replaying
one whose response was merely lost would push a second write built on a
now-stale precondition, against a remote another device reads. Reads are
idempotent, so only they are replayed.
"""
session = requests.Session()
retry = Retry(
total=_RETRY_TOTAL,
backoff_factor=_RETRY_BACKOFF_SECONDS,
status_forcelist=_RETRY_STATUSES,
allowed_methods=frozenset({"GET"}),
raise_on_status=False,
)
adapter = HTTPAdapter(max_retries=retry)
session.mount("https://", adapter)
session.mount("http://", adapter)
return session

View File

@ -1,4 +1,3 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""Log: a collection of Records keyed by id, with a union-merge."""
from __future__ import annotations

View File

@ -1,231 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""A remote store that writes to two backends and reads from both.
Exists for the GitHub -> Firebase cutover: each app can move to Firebase
while still mirroring to the old repo, so rolling back is a constructor
change rather than a data recovery.
Mirrors ``crdt_sync_dart``'s ``lib/src/mirror_store.dart``; keep the two in
step.
"""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, TypeVar
from crdt_sync._remote import RemoteSyncError
if TYPE_CHECKING:
from collections.abc import Callable
from crdt_sync._remote import RemoteStore
_logger = logging.getLogger(__name__)
# PEP 695 type parameters need Python 3.12; this package targets 3.10.
_T = TypeVar("_T")
class MirrorSyncClient:
"""Writes to both backends; reads from both, preferring the primary.
**Reads consult both on purpose.** Each app spans two devices, and they
cut over one at a time. If reads were primary-only, a migrated PC would
never see an un-migrated phone's writes while still mirroring its own
back -- silent one-directional convergence, with no error raised. Because
:func:`crdt_sync.merge_logs` is commutative and idempotent, a union read
costs nothing semantically and makes the cutover order irrelevant.
Write asymmetry is deliberate too: the primary is authoritative, so a
primary failure fails the tick (fail-closed), while a mirror failure is
reported and otherwise tolerated. Once the mirror is retired, the old
backend going away must not break sync.
**Reads are resilient on BOTH sides; writes are not.** A read is only
fail-closed when neither backend can answer. Reads used to call the
primary unguarded, so a Firebase outage produced no data at all and the
mirror was never consulted -- the union read silently degraded to nothing,
contradicting the paragraph above. Writes keep the asymmetry: a write the
primary did not accept has not happened, so it must fail the tick.
"""
def __init__(
self,
primary: RemoteStore,
mirror: RemoteStore,
*,
on_mirror_failure: Callable[[str, Exception], None] | None = None,
) -> None:
"""Wrap ``primary`` (authoritative) and ``mirror`` (best effort).
Args:
primary: The backend whose failures fail the tick.
mirror: The backend kept in step but never allowed to fail a tick.
on_mirror_failure: Notified when the mirror misbehaves. Defaults
to a warning log -- a silent mirror failure would let the
fallback rot unnoticed until the day it was needed.
"""
self.primary = primary
self.mirror = mirror
self._on_mirror_failure = on_mirror_failure or _log_mirror_failure
def list_directory(self, path: str) -> list[str]:
"""Return the union of both backends' entries under ``path``.
A reachable primary that lists nothing is a real answer; a primary
that *raises* is not. Reads are the half of this class that exists to
be resilient, so a primary failure degrades to the mirror's entries
rather than failing the read -- returning nothing because one side is
down breaks the union promise exactly when the fallback is most
needed. Only a failure on BOTH sides propagates, since then there is
genuinely no answer to give.
"""
primary_names = self._try_primary_read(
f"list_directory {path}", lambda: self.primary.list_directory(path)
)
mirror_names = self._try_mirror(
f"list_directory {path}", lambda: self.mirror.list_directory(path)
)
if primary_names is None and mirror_names is None:
self._raise_both_failed(f"list_directory {path}")
names = list(primary_names or [])
seen = set(names)
for name in mirror_names or []:
if name not in seen:
seen.add(name)
names.append(name)
return names
def get_file_text(self, path: str) -> str | None:
"""Return ``path``'s text from the primary, else from the mirror.
Absent from the primary means "this device has not migrated yet",
not "no data" -- so it falls through rather than reporting nothing.
A primary that *raises* falls through for the same reason (see
:meth:`list_directory`); only a failure on both sides propagates.
"""
try:
text = self.primary.get_file_text(path)
except RemoteSyncError as error:
self._on_primary_read_failure(f"get_file_text {path}", error)
mirror_text = self._try_mirror(
f"get_file_text {path}", lambda: self.mirror.get_file_text(path)
)
if mirror_text is None:
self._raise_both_failed(f"get_file_text {path}")
return mirror_text
if text is not None:
return text
return self._try_mirror(
f"get_file_text {path}", lambda: self.mirror.get_file_text(path)
)
def put_file_text(self, path: str, text: str, *, message: str) -> None:
"""Write to the primary, then best-effort to the mirror.
Raises:
RemoteSyncError: If the *primary* write fails.
"""
self.primary.put_file_text(path, text, message=message)
self._try_mirror(
f"put_file_text {path}",
lambda: self.mirror.put_file_text(path, text, message=message),
)
def delete_file(self, path: str, *, message: str = "crdt_sync: delete") -> None:
"""Delete from the primary, then best-effort from the mirror.
Raises:
RemoteSyncError: If the *primary* delete fails.
"""
self.primary.delete_file(path, message=message)
self._try_mirror(
f"delete_file {path}",
lambda: self.mirror.delete_file(path, message=message),
)
def can_access_remote(self) -> bool:
"""Return whether the **primary** is reachable.
Only the primary: a settings "Test connection" button must not report
success because the backend being retired happens to answer.
"""
return self.primary.can_access_remote()
def get_string_map(self, path: str) -> dict[str, str]:
"""Merge both backends' revision maps, primary winning on conflict.
A device that has not migrated publishes revisions only to the
mirror; without this it would look revision-less and be
re-downloaded every tick for the whole trial period.
Like the other reads, a primary failure degrades to the mirror rather
than failing the tick: revisions are a cache, and losing them re-reads
data rather than losing it.
"""
merged: dict[str, str] = {}
mirror_read = getattr(self.mirror, "get_string_map", None)
# A mirror that CANNOT do bulk reads contributes nothing, but that is
# not a failure -- it has no revisions to give, which is a real answer.
# Only a mirror that tried and threw leaves the read with no answer,
# and only then can a failing primary make this raise.
mirror_answered = mirror_read is None
if mirror_read is not None:
mirror_map = self._try_mirror(
f"get_string_map {path}", lambda: mirror_read(path)
)
mirror_answered = mirror_map is not None
merged.update(mirror_map or {})
primary_read = getattr(self.primary, "get_string_map", None)
if primary_read is not None:
primary_map = self._try_primary_read(
f"get_string_map {path}", lambda: primary_read(path)
)
if primary_map is None and not mirror_answered:
self._raise_both_failed(f"get_string_map {path}")
merged.update(primary_map or {})
return merged
def _try_mirror(self, operation: str, run: Callable[[], _T]) -> _T | None:
"""Run a mirror operation, reporting rather than propagating failure."""
try:
return run()
except RemoteSyncError as error:
self._on_mirror_failure(operation, error)
return None
def _try_primary_read(self, operation: str, run: Callable[[], _T]) -> _T | None:
"""Run a primary READ, reporting rather than propagating failure.
Reads only -- writes stay fail-closed, since a write the primary did
not accept has not happened and must fail the tick.
"""
try:
return run()
except RemoteSyncError as error:
self._on_primary_read_failure(operation, error)
return None
def _on_primary_read_failure(self, operation: str, error: Exception) -> None:
"""Warn that a read fell back to the mirror."""
_logger.warning(
"Primary backend failed during %s: %s — falling back to the mirror",
operation,
error,
)
def _raise_both_failed(self, operation: str) -> None:
"""Raise when neither backend could answer a read.
Fail closed rather than returning an empty result: an empty read is
indistinguishable from "no data", and callers act on that by treating
remote state as absent.
"""
msg = f"both primary and mirror backends failed during {operation}"
raise RemoteSyncError(msg)
def _log_mirror_failure(operation: str, error: Exception) -> None:
"""Warn loudly, so a rotting fallback is visible before it is needed."""
_logger.warning("Mirror backend failed during %s: %s", operation, error)

View File

@ -1,109 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""Downloading peers' logs, skipping the ones we already have.
Split from :mod:`crdt_sync._sync`, which keeps the push half and the
orchestration. The revision map is the optimisation that makes a tick cheap:
a peer whose published revision matches what we last merged is not fetched at
all, so a quiet device costs one small read instead of one read per peer.
"""
from __future__ import annotations
from dataclasses import dataclass
import logging
from typing import TYPE_CHECKING
from crdt_sync._revisions import revision_of
if TYPE_CHECKING:
from collections.abc import Callable
from crdt_sync._log import Log
from crdt_sync._remote import RemoteStore
from crdt_sync._syncstate import SyncState
_logger = logging.getLogger(__name__)
def _remote_revs(client: RemoteStore, revs_path: str) -> dict[str, str]:
"""Return every peer's published revision, cheaply where possible.
Degrades to an empty map -- meaning "fetch everything", the old behaviour
-- on any backend without a bulk-map read, so correctness never depends
on the optimisation being available.
"""
get_string_map = getattr(client, "get_string_map", None)
if get_string_map is None:
return {}
return get_string_map(revs_path)
@dataclass(frozen=True)
class _PullContext:
"""Everything :func:`_pull_remote_logs` needs for one tick's pull.
A bundle rather than eight positional parameters: they are always passed
together, from one call site, and several share a type -- exactly the
shape where a positional mix-up is silent.
"""
client: RemoteStore
own_ids: frozenset[str]
path_prefix: str
filename: str
decode: Callable[[str], Log]
remote_revs: dict[str, str]
state: SyncState
def _pull_remote_logs(
ctx: _PullContext,
seen_revs: dict[str, str],
) -> list[Log]:
"""Return every other device's last-pushed log, skipping this one.
A device whose pushed file is corrupt or unparsable (e.g. an interrupted
push) is logged and skipped, same as one that has never pushed at all --
the remote is an external system boundary, and one bad device's file must
not stall merging in every other device's.
``own_ids`` is a *set* rather than a single id because a device that has
migrated from a role constant to a persisted uuid still owns the log it
pushed under the old id. Matching only the current id would pull that
file back and re-merge this device's own pre-migration history as though
a peer had written it -- idempotent, but pure wasted transfer.
Records into ``seen_revs`` which peers are now merged in, so the next
tick can skip them.
"""
remote_logs: list[Log] = []
for other_device_id in ctx.client.list_directory(ctx.path_prefix):
if other_device_id in ctx.own_ids:
continue
remote_rev = ctx.remote_revs.get(other_device_id)
if remote_rev is not None and remote_rev == ctx.state.peer_revs.get(
other_device_id
):
# Unchanged since we last merged it, and that merge is already
# part of local_log -- so the (potentially hundreds of KB)
# download is pure waste. Carry the revision forward so it stays
# skipped next tick.
seen_revs[other_device_id] = remote_rev
continue
text = ctx.client.get_file_text(
f"{ctx.path_prefix}/{other_device_id}/{ctx.filename}"
)
if text is None:
continue
try:
remote_logs.append(ctx.decode(text))
except ValueError, KeyError, TypeError:
_logger.warning(
"Unparsable log pushed by device %r, skipping",
other_device_id,
)
# Deliberately not recorded as seen: a corrupt push must be
# retried next tick, not remembered as merged.
continue
seen_revs[other_device_id] = remote_rev or revision_of(text)
return remote_logs

View File

@ -1,4 +1,3 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""Record: a per-field LWW map with a sticky (monotonic) delete flag.
This is the unit CRDT merges operate on. It's deliberately domain-agnostic --

View File

@ -1,83 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""Backend-neutral remote-storage contract that sync talks through.
Deliberately tiny: :func:`crdt_sync.sync_log` and every app-level sync
service talk only to :class:`RemoteStore`, so swapping the storage backend
(GitHub Contents API, Firebase Realtime Database, a dual-writing mirror of
both) is a constructor change at the call site and nothing more.
Mirrors ``crdt_sync_dart``'s ``lib/src/remote_store.dart``; keep the two in
step.
"""
from __future__ import annotations
from typing import Protocol, runtime_checkable
class RemoteSyncError(Exception):
"""Raised for a remote-storage failure the caller must not ignore.
The backend-neutral base of the error hierarchy: catch this to handle
"the sync transport failed" regardless of which backend is configured.
Backends narrow it -- :class:`crdt_sync.GitHubSyncError` and
:class:`crdt_sync.FirebaseSyncError` both derive from it -- so existing
``except GitHubSyncError`` handlers keep working while a caller that has
been migrated can catch the base type instead.
"""
class RemoteNotFoundError(RemoteSyncError):
"""Raised when the configured remote itself is unreachable.
Distinguished from a missing *path* (nothing pushed there yet, which is
benign -- it just means no other device has synced before) so the caller
can tell "the repo/database is wrong or the credential isn't scoped to
it" apart from "no other device has synced yet".
"""
@runtime_checkable
class RemoteStore(Protocol):
"""Dumb keyed storage of UTF-8 text blobs, with directory listing.
Implementations raise :class:`RemoteSyncError` (or a subclass) for any
failure the caller must not silently ignore, and treat a missing path as
a benign ``None`` / empty list rather than an error.
A ``Protocol`` rather than an ABC so the existing
:class:`crdt_sync.GitHubSyncClient` satisfies it structurally, with no
base class to add and no change to its own tests.
"""
def list_directory(self, path: str) -> list[str]:
"""Return the entry names directly under ``path``.
Returns an empty list when nothing has been written there yet.
Includes both files and subdirectories: the
``<path_prefix>/<device_id>/<filename>`` layout needs to discover
device *directories*, not just files.
"""
def get_file_text(self, path: str) -> str | None:
"""Return the text stored at ``path``, or ``None`` if absent."""
def put_file_text(self, path: str, text: str, *, message: str) -> None:
"""Create or replace the text at ``path``.
``message`` is a human-readable reason for the write. Backends that
record one (GitHub commits) use it; backends that do not (Firebase)
ignore it.
"""
def delete_file(self, path: str, *, message: str = ...) -> None:
"""Delete ``path``. A no-op when ``path`` does not exist."""
def can_access_remote(self) -> bool:
"""Return whether the configured credential can reach the remote.
A lightweight connection test for a settings "Test connection"
button: it probes the remote root, so it succeeds even before any
file has been pushed. Never raises -- a network failure or missing
remote returns ``False``.
"""

View File

@ -1,38 +0,0 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""The revision string a device publishes alongside its log.
Split from :mod:`crdt_sync._sync` so both the push and pull halves can import
it without importing each other. Pure: a revision is a function of the encoded
log and nothing else, which is what lets two devices agree on whether they are
looking at the same bytes.
Re-exported from :mod:`crdt_sync._sync`, so existing imports keep working.
"""
from __future__ import annotations
import hashlib
def revision_of(encoded_log: str) -> str:
"""Return the revision of a serialized log: a content hash.
A hash rather than the log's maximum HLC because merging a peer's *older*
record changes the content without raising the maximum -- with three
devices in one namespace (diet-guard has ``pc``, ``phone`` and
``desktop``) a clock-based revision can miss a peer's merged state. It is
also the same value used to suppress no-op pushes, so the two
optimisations share one mechanism.
Parameters
----------
encoded_log : str
The serialized log, exactly as it would be pushed.
Returns:
-------
str
A hex SHA-256 digest.
"""
return hashlib.sha256(encoded_log.encode("utf-8")).hexdigest()

View File

@ -1,4 +1,3 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""Local JSON persistence for a ``Log``.
Mirrors ``crdt_sync_dart``'s ``logToJson``/``FileLogPersistence`` serialization
@ -54,7 +53,7 @@ def read_log(path: Path) -> Log:
return {}
try:
return load_log(text)
except ValueError, KeyError, TypeError:
except (ValueError, KeyError, TypeError):
return {}

View File

@ -1,4 +1,3 @@
# Copyright (c) 2026 Krzysztof Rudnicki
"""Domain-agnostic pull/merge/push sync orchestration.
Generalizes diet_guard's original ``_sync.py`` (pull every other device's
@ -6,179 +5,102 @@ pushed log, merge with the local one, push this device's own merged result
back up) so any app can reuse the loop while keeping its own on-disk JSON
shape via the ``encode``/``decode`` callbacks -- this module has no opinion
on what a ``Record``'s fields actually mean.
Mirrors ``crdt_sync_dart``'s ``lib/src/sync.dart``; keep the two in step.
"""
from __future__ import annotations
from dataclasses import dataclass
import logging
from typing import TYPE_CHECKING
from crdt_sync._log import merge_logs
from crdt_sync._pull import _pull_remote_logs, _PullContext, _remote_revs
from crdt_sync._revisions import revision_of
from crdt_sync._syncargs import LogCodec, RevisionTracking, SyncTarget
from crdt_sync._syncstate import (
FileSyncStateStore,
MemorySyncStateStore,
SyncState,
SyncStateStore,
)
if TYPE_CHECKING:
from crdt_sync._log import Log
from collections.abc import Callable
# Named explicitly so the autofixer cannot prune an import that exists for its
# re-export: crdt_sync/__init__.py imports all of these from here.
__all__ = [
"FileSyncStateStore",
"LogCodec",
"MemorySyncStateStore",
"RevisionTracking",
"SyncState",
"SyncStateStore",
"SyncTarget",
"default_revs_path",
"revision_of",
"sync_log",
]
from crdt_sync._github import GitHubSyncClient
from crdt_sync._log import Log
_logger = logging.getLogger(__name__)
_DEFAULT_FILENAME = "log.json"
def default_revs_path(path_prefix: str) -> str:
"""Return where revisions live for ``path_prefix``.
A ``revs`` sibling of the device directory, e.g.
``diet-guard-sync/devices`` -> ``diet-guard-sync/revs`` and
``todo-sync/notes`` -> ``todo-sync/revs``.
Parameters
----------
path_prefix : str
The directory holding one subdirectory per device.
Returns:
-------
str
The revisions directory.
def _pull_remote_logs(
client: GitHubSyncClient,
device_id: str,
path_prefix: str,
filename: str,
decode: Callable[[str], Log],
) -> list[Log]:
"""Return every other device's last-pushed log, skipping this one.
A device whose pushed file is corrupt or unparsable (e.g. an interrupted
push) is logged and skipped, same as one that has never pushed at all --
GitHub is an external system boundary, and one bad device's file must not
stall merging in every other device's.
"""
head, separator, _ = path_prefix.rpartition("/")
return f"{head}/revs" if separator else f"{path_prefix}/revs"
remote_logs: list[Log] = []
for other_device_id in client.list_directory(path_prefix):
if other_device_id == device_id:
continue
text = client.get_file_text(f"{path_prefix}/{other_device_id}/{filename}")
if text is None:
continue
try:
remote_logs.append(decode(text))
except (ValueError, KeyError, TypeError):
_logger.warning(
"Unparsable log pushed by device %r, skipping",
other_device_id,
)
return remote_logs
def sync_log(
target: SyncTarget,
*,
client: GitHubSyncClient,
device_id: str,
path_prefix: str,
local_log: Log,
codec: LogCodec,
revisions: RevisionTracking | None = None,
encode: Callable[[Log], str],
decode: Callable[[str], Log],
filename: str = _DEFAULT_FILENAME,
commit_message: str = "crdt_sync: update log",
) -> Log:
"""Run one full sync tick: pull every other device's log, merge, push.
Pulls from ``<path_prefix>/<other-device-id>/<filename>`` for every
device directory the remote reports under ``path_prefix``, merges each
into ``local_log`` with :func:`crdt_sync.merge_logs`, then pushes this
device directory GitHub reports under ``path_prefix``, merges each into
``local_log`` with :func:`crdt_sync.merge_logs`, then pushes this
device's own merged result to ``<path_prefix>/<device_id>/<filename>``.
Args:
target: Which remote and device this tick is for.
client: An authenticated :class:`GitHubSyncClient`.
device_id: This device's identifier; also the directory name its own
log is pushed under.
path_prefix: The repo-relative directory holding one subdirectory
per device (e.g. ``"devices"``).
local_log: This device's current full log (including tombstones).
codec: How to serialize and parse a log.
revisions: Pass one to skip downloading peers that have not changed
and skip pushing a log that has not changed. ``None`` fetches
everything and pushes unconditionally.
encode: Serializes a merged log for pushing.
decode: Parses a remote device's pushed text back into a log.
Raising ``ValueError``, ``KeyError``, or ``TypeError`` is treated
as a corrupt/unparsable push, and that device is skipped for
this tick rather than aborting the whole sync.
filename: The file name each device pushes its log as.
commit_message: The commit message used for this device's push.
Returns:
The merged log, as pushed.
Notes:
Revisions live at ``<revs_path>/<device_id>`` -- one small text node
per device, each written only by its owner. Deliberately *not* one
shared map per namespace: a whole-map write would erase every other
device's entry, after which those peers would look permanently
unchanged and never be fetched again. Per-device keys make that
failure impossible rather than merely avoided.
"""
tracking = revisions if revisions is not None else RevisionTracking()
state_store = tracking.state_store
revs = (
tracking.revs_path
if tracking.revs_path is not None
else default_revs_path(target.path_prefix)
)
state = state_store.load() if state_store is not None else SyncState()
remote_revs = _remote_revs(target.client, revs) if state_store is not None else {}
seen_revs: dict[str, str] = {}
merged = _merge_peers(
_PullContext(
client=target.client,
own_ids=target.own_ids,
path_prefix=target.path_prefix,
filename=codec.filename,
decode=codec.decode,
remote_revs=remote_revs,
state=state,
),
local_log,
seen_revs,
)
encoded = codec.encode(merged)
revision = revision_of(encoded)
if not (state_store is not None and revision == state.pushed_rev):
_push(
target,
codec,
encoded,
_RevisionMarker(revs, revision) if state_store is not None else None,
)
if state_store is not None:
state_store.save(SyncState(pushed_rev=revision, peer_revs=seen_revs))
return merged
def _merge_peers(
ctx: _PullContext,
local_log: Log,
seen_revs: dict[str, str],
) -> Log:
"""Return ``local_log`` with every peer's pushed log merged in."""
merged = dict(local_log)
for remote_log in _pull_remote_logs(ctx, seen_revs):
for remote_log in _pull_remote_logs(
client, device_id, path_prefix, filename, decode
):
merged = merge_logs(merged, remote_log)
return merged
@dataclass(frozen=True)
class _RevisionMarker:
"""The revision node a tracked push publishes after its log."""
revs_path: str
revision: str
def _push(
target: SyncTarget,
codec: LogCodec,
encoded: str,
marker: _RevisionMarker | None,
) -> None:
"""Push this device's merged log, then its revision marker."""
target.client.put_file_text(
f"{target.path_prefix}/{target.device_id}/{codec.filename}",
encoded,
message=codec.commit_message,
client.put_file_text(
f"{path_prefix}/{device_id}/{filename}",
encode(merged),
message=commit_message,
)
if marker is not None:
# Published after the log, never before: a peer that cached
# "seen rev X" against a log it never received would skip it
# forever.
target.client.put_file_text(
f"{marker.revs_path}/{target.device_id}",
marker.revision,
message=codec.commit_message,
)
return merged

Some files were not shown because too many files have changed in this diff Show More