mirror of
https://github.com/kuhyx/utils.git
synced 2026-11-10 21:47:14 +01:00
Compare commits
No commits in common. "main" and "gatelock-v0.1.1" have entirely different histories.
main
...
gatelock-v
48
.github/workflows/crdt-sync-dart-tests.yml
vendored
48
.github/workflows/crdt-sync-dart-tests.yml
vendored
@ -1,48 +0,0 @@
|
||||
name: crdt_sync_dart tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['crdt_sync_dart/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['crdt_sync_dart/**']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: crdt_sync_dart
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
# Pure Dart, no Flutter dependency -- so setup-dart, not flutter-action.
|
||||
- name: Set up Dart
|
||||
uses: dart-lang/setup-dart@v1
|
||||
with:
|
||||
sdk: stable
|
||||
|
||||
- name: Install dependencies
|
||||
run: dart pub get
|
||||
|
||||
- name: Analyze
|
||||
run: dart analyze --fatal-infos
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: dart test --coverage=coverage
|
||||
|
||||
# Unlike `flutter test --coverage`, `dart test --coverage` emits raw VM
|
||||
# JSON and no lcov. Skipping this step leaves a stale lcov.info on disk
|
||||
# that the gate below happily "passes" -- which is exactly how this
|
||||
# package drifted to 96% while believing it was at 100%.
|
||||
- name: Convert coverage to lcov
|
||||
run: |
|
||||
dart pub global activate coverage
|
||||
dart pub global run coverage:format_coverage \
|
||||
--lcov --in=coverage/test --out=coverage/lcov.info \
|
||||
--report-on=lib --packages=.dart_tool/package_config.json
|
||||
|
||||
- name: Enforce 100% line coverage
|
||||
run: scripts/check_coverage.sh coverage/lcov.info 100
|
||||
36
.github/workflows/crdt-sync-flutter-tests.yml
vendored
36
.github/workflows/crdt-sync-flutter-tests.yml
vendored
@ -1,36 +0,0 @@
|
||||
name: crdt_sync_flutter tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['crdt_sync_flutter/**', 'crdt_sync_dart/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['crdt_sync_flutter/**', 'crdt_sync_dart/**']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: crdt_sync_flutter
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Flutter
|
||||
uses: subosito/flutter-action@v2
|
||||
with:
|
||||
channel: stable
|
||||
|
||||
- name: Install dependencies
|
||||
run: flutter pub get
|
||||
|
||||
- name: Analyze
|
||||
run: flutter analyze
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: flutter test --coverage
|
||||
|
||||
- name: Enforce 100% line coverage
|
||||
run: scripts/check_coverage.sh coverage/lcov.info 100
|
||||
62
.github/workflows/crdt-sync-precommit.yml
vendored
62
.github/workflows/crdt-sync-precommit.yml
vendored
@ -3,10 +3,10 @@ name: crdt-sync pre-commit
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['crdt-sync/**', '.github/workflows/crdt-sync-precommit.yml']
|
||||
paths: ['crdt-sync/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['crdt-sync/**', '.github/workflows/crdt-sync-precommit.yml']
|
||||
paths: ['crdt-sync/**']
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
@ -15,56 +15,10 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
# Only pre-commit itself. Every linting hook runs in its own isolated
|
||||
# env -- ruff, mypy, pylint, bandit and codespell each come from their
|
||||
# own repo+rev, and the local hooks need nothing beyond bash and
|
||||
# python3 -- so installing the full requirements.txt spent 9s a run on
|
||||
# packages no pre-commit-stage hook imports. Verified by running all
|
||||
# three configs green in a venv holding pre-commit and nothing else.
|
||||
# The pin is read out of requirements.txt rather than repeated here, so
|
||||
# it cannot drift from what the dependency-freshness gate governs.
|
||||
- name: Install pre-commit
|
||||
run: |
|
||||
pip install "$(grep '^pre-commit==' crdt-sync/requirements.txt)"
|
||||
|
||||
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
|
||||
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
|
||||
# package jobs shared one key, raced to overwrite it with hook sets built
|
||||
# from four different configs, and editing a per-package config did not
|
||||
# invalidate it. The cache was therefore silently wrong rather than
|
||||
# missing, and the hooks reinstalled from scratch on every run (38-49s).
|
||||
- uses: actions/cache@v4
|
||||
python-version: "3.12"
|
||||
- name: Install dependencies
|
||||
working-directory: crdt-sync
|
||||
run: pip install -r requirements.txt
|
||||
- uses: pre-commit/action@v3.0.1
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: >-
|
||||
pre-commit-crdt-sync-${{ env.pythonLocation }}-${{
|
||||
hashFiles('crdt-sync/.pre-commit-config.yaml') }}
|
||||
# Prefix fallback, or this fix would evaporate on the very
|
||||
# event that motivated it: bumping a hook rev changes the
|
||||
# exact key, and a total miss means reinstalling every hook
|
||||
# again. pre-commit's store is keyed internally by hook
|
||||
# repo+rev, so a superseded entry is ignored disk rather
|
||||
# than a wrong hook, and cache still re-saves under the new
|
||||
# exact key after a prefix-only hit.
|
||||
restore-keys: pre-commit-crdt-sync-${{ env.pythonLocation }}-
|
||||
# mypy re-analyses the stdlib and every third-party stub from scratch
|
||||
# without this, which measured 10s of the hook step in CI against 0.25s
|
||||
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
|
||||
# the config, not the sources: mypy's own cache metadata records the
|
||||
# interpreter, the flag set and a hash per source file, so it re-checks
|
||||
# what changed and cannot pass stale code. Deliberately NOT keyed on
|
||||
# github.sha -- that re-saves 19MB x3 every commit and would evict the
|
||||
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
|
||||
# the expensive part, the dependency analysis, which is what costs 10s.
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: >-
|
||||
mypy-crdt-sync-${{ env.pythonLocation }}-${{
|
||||
hashFiles('crdt-sync/.pre-commit-config.yaml') }}
|
||||
restore-keys: mypy-crdt-sync-${{ env.pythonLocation }}-
|
||||
- name: Run the hooks
|
||||
run: >-
|
||||
pre-commit run --config crdt-sync/.pre-commit-config.yaml
|
||||
--all-files --show-diff-on-failure
|
||||
extra_args: --config crdt-sync/.pre-commit-config.yaml --all-files
|
||||
|
||||
14
.github/workflows/crdt-sync-tests.yml
vendored
14
.github/workflows/crdt-sync-tests.yml
vendored
@ -3,10 +3,10 @@ name: crdt-sync tests
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['crdt-sync/**', '.github/workflows/crdt-sync-tests.yml']
|
||||
paths: ['crdt-sync/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['crdt-sync/**', '.github/workflows/crdt-sync-tests.yml']
|
||||
paths: ['crdt-sync/**']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
@ -14,15 +14,17 @@ jobs:
|
||||
defaults:
|
||||
run:
|
||||
working-directory: crdt-sync
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: crdt-sync/requirements.txt
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt
|
||||
|
||||
@ -1,36 +0,0 @@
|
||||
name: data_transfer_flutter tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['data_transfer_flutter/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['data_transfer_flutter/**']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: data_transfer_flutter
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Flutter
|
||||
uses: subosito/flutter-action@v2
|
||||
with:
|
||||
channel: stable
|
||||
|
||||
- name: Install dependencies
|
||||
run: flutter pub get
|
||||
|
||||
- name: Analyze
|
||||
run: flutter analyze
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: flutter test --coverage
|
||||
|
||||
- name: Enforce 100% line coverage
|
||||
run: scripts/check_coverage.sh coverage/lcov.info 100
|
||||
61
.github/workflows/dependency-freshness.yml
vendored
61
.github/workflows/dependency-freshness.yml
vendored
@ -1,61 +0,0 @@
|
||||
name: dependency freshness
|
||||
|
||||
# Every dependency on its ecosystem's newest STABLE release, enforced on push
|
||||
# as well as at commit time. The local pre-commit hook only sees the manifests
|
||||
# in a commit and can be skipped entirely by a fresh clone that never ran
|
||||
# `pre-commit install`; this runs `--all` over the whole repo.
|
||||
#
|
||||
# `--strict` is CI-only. Offline the gate degrades to its cached answer and
|
||||
# passes, because `--no-verify` is banned locally and a hook that hard-failed
|
||||
# on a flaky connection would leave no way to commit. A runner is always
|
||||
# online, so here an undeterminable answer is a real failure.
|
||||
#
|
||||
# Expect this to go red for reasons this repo's commit did not cause: an
|
||||
# upstream release overnight is enough. That is the chosen behaviour -- the
|
||||
# alternative is a gate that only notices staleness when someone remembers to
|
||||
# look.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
dependency-freshness:
|
||||
name: dependencies on newest stable
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
# The gate lives in the shared utils repo so every repo resolves the
|
||||
# same "latest stable" the same way. Checking it out beats vendoring a
|
||||
# copy, which is exactly how a gate and its survey start disagreeing.
|
||||
- name: Check out the shared gate
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: kuhyx/utils
|
||||
path: .utils
|
||||
|
||||
# Any recent Python runs the gate. The Python TARGET is python.org's
|
||||
# newest release (registries/toolchain.python_latest), not this
|
||||
# interpreter: when it was, "3.x" resolved to 3.15.0 on one runner and
|
||||
# 3.14.8 on the next, and spellcode's allowlist flipped dead/required.
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
|
||||
- name: Install the gate's two runtime dependencies
|
||||
run: python -m pip install --upgrade pyyaml packaging
|
||||
|
||||
- name: Cache registry lookups
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/dep-freshness
|
||||
key: dep-freshness-${{ hashFiles('**/pubspec.yaml', '**/pyproject.toml',
|
||||
'**/requirements*.txt', '**/package.json', '**/Cargo.toml', '**/go.mod') }}
|
||||
restore-keys: dep-freshness-
|
||||
|
||||
- name: Every dependency is on newest stable
|
||||
run: bash .utils/scripts/check_dependency_freshness.sh --all --strict
|
||||
36
.github/workflows/design-system-tests.yml
vendored
36
.github/workflows/design-system-tests.yml
vendored
@ -1,36 +0,0 @@
|
||||
name: design_system tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['design_system/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['design_system/**']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: design_system
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Flutter
|
||||
uses: subosito/flutter-action@v2
|
||||
with:
|
||||
channel: stable
|
||||
|
||||
- name: Install dependencies
|
||||
run: flutter pub get
|
||||
|
||||
- name: Analyze
|
||||
run: flutter analyze
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: flutter test --coverage
|
||||
|
||||
- name: Enforce 100% line coverage
|
||||
run: scripts/check_coverage.sh coverage/lcov.info 100
|
||||
73
.github/workflows/earned-time-precommit.yml
vendored
73
.github/workflows/earned-time-precommit.yml
vendored
@ -1,73 +0,0 @@
|
||||
name: earned_time pre-commit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as earned_time: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['earned_time/**', '.github/workflows/earned-time-precommit.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['earned_time/**', '.github/workflows/earned-time-precommit.yml']
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
# Only pre-commit itself. Every linting hook runs in its own isolated
|
||||
# env -- ruff, mypy, pylint, bandit and codespell each come from their
|
||||
# own repo+rev, and the local hooks need nothing beyond bash and
|
||||
# python3 -- so installing the full requirements.txt spent 9s a run on
|
||||
# packages no pre-commit-stage hook imports. Verified by running all
|
||||
# three configs green in a venv holding pre-commit and nothing else.
|
||||
# The pin is read out of requirements.txt rather than repeated here, so
|
||||
# it cannot drift from what the dependency-freshness gate governs.
|
||||
- name: Install pre-commit
|
||||
run: |
|
||||
pip install "$(grep '^pre-commit==' earned_time/requirements.txt)"
|
||||
|
||||
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
|
||||
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
|
||||
# package jobs shared one key, raced to overwrite it with hook sets built
|
||||
# from four different configs, and editing a per-package config did not
|
||||
# invalidate it. The cache was therefore silently wrong rather than
|
||||
# missing, and the hooks reinstalled from scratch on every run (38-49s).
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: >-
|
||||
pre-commit-earned_time-${{ env.pythonLocation }}-${{
|
||||
hashFiles('earned_time/.pre-commit-config.yaml') }}
|
||||
# Prefix fallback, or this fix would evaporate on the very
|
||||
# event that motivated it: bumping a hook rev changes the
|
||||
# exact key, and a total miss means reinstalling every hook
|
||||
# again. pre-commit's store is keyed internally by hook
|
||||
# repo+rev, so a superseded entry is ignored disk rather
|
||||
# than a wrong hook, and cache still re-saves under the new
|
||||
# exact key after a prefix-only hit.
|
||||
restore-keys: pre-commit-earned_time-${{ env.pythonLocation }}-
|
||||
# mypy re-analyses the stdlib and every third-party stub from scratch
|
||||
# without this, which measured 10s of the hook step in CI against 0.25s
|
||||
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
|
||||
# the config, not the sources: mypy's own cache metadata records the
|
||||
# interpreter, the flag set and a hash per source file, so it re-checks
|
||||
# what changed and cannot pass stale code. Deliberately NOT keyed on
|
||||
# github.sha -- that re-saves 19MB x3 every commit and would evict the
|
||||
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
|
||||
# the expensive part, the dependency analysis, which is what costs 10s.
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: >-
|
||||
mypy-earned_time-${{ env.pythonLocation }}-${{
|
||||
hashFiles('earned_time/.pre-commit-config.yaml') }}
|
||||
restore-keys: mypy-earned_time-${{ env.pythonLocation }}-
|
||||
- name: Run the hooks
|
||||
run: >-
|
||||
pre-commit run --config earned_time/.pre-commit-config.yaml
|
||||
--all-files --show-diff-on-failure
|
||||
36
.github/workflows/earned-time-tests.yml
vendored
36
.github/workflows/earned-time-tests.yml
vendored
@ -1,36 +0,0 @@
|
||||
name: earned_time tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as earned_time: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['earned_time/**', '.github/workflows/earned-time-tests.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['earned_time/**', '.github/workflows/earned-time-tests.yml']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: earned_time
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: earned_time/requirements.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt && pip install --no-deps -e .
|
||||
|
||||
# No display needed: earned_time is stdlib-only policy, imported
|
||||
# headless by the root gaming daemon.
|
||||
- name: Run tests with coverage
|
||||
run: python -m pytest -v
|
||||
34
.github/workflows/file-length.yml
vendored
34
.github/workflows/file-length.yml
vendored
@ -1,34 +0,0 @@
|
||||
name: file length
|
||||
|
||||
# The 250-line cap, enforced on push as well as in pre-commit
|
||||
# (refactor_claude_todo.md, "make it permanent").
|
||||
#
|
||||
# Unlike the pre-commit hook, which only sees the files in a commit, this runs
|
||||
# over the whole tree: the hook keeps a violation from being *introduced*, and
|
||||
# this keeps one from surviving a path the hook never saw (a merge, a push
|
||||
# from a machine without hooks installed, a rebase that resurrects a file).
|
||||
#
|
||||
# No paths filter on purpose. The cap applies to every file in the repo, so
|
||||
# scoping this job to a subdirectory would be the one way to reintroduce the
|
||||
# problem it exists to catch.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
file-length:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
|
||||
- name: Every file is at most 250 lines
|
||||
run: bash scripts/check_file_length.sh --all
|
||||
73
.github/workflows/focus-project-precommit.yml
vendored
73
.github/workflows/focus-project-precommit.yml
vendored
@ -1,73 +0,0 @@
|
||||
name: focus-project pre-commit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as focus_project: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['focus_project/**', '.github/workflows/focus-project-precommit.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['focus_project/**', '.github/workflows/focus-project-precommit.yml']
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
# Only pre-commit itself. Every linting hook runs in its own isolated
|
||||
# env -- ruff, mypy, pylint, bandit and codespell each come from their
|
||||
# own repo+rev, and the local hooks need nothing beyond bash and
|
||||
# python3 -- so installing the full requirements.txt spent 9s a run on
|
||||
# packages no pre-commit-stage hook imports. Verified by running all
|
||||
# three configs green in a venv holding pre-commit and nothing else.
|
||||
# The pin is read out of requirements.txt rather than repeated here, so
|
||||
# it cannot drift from what the dependency-freshness gate governs.
|
||||
- name: Install pre-commit
|
||||
run: |
|
||||
pip install "$(grep '^pre-commit==' focus_project/requirements.txt)"
|
||||
|
||||
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
|
||||
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
|
||||
# package jobs shared one key, raced to overwrite it with hook sets built
|
||||
# from four different configs, and editing a per-package config did not
|
||||
# invalidate it. The cache was therefore silently wrong rather than
|
||||
# missing, and the hooks reinstalled from scratch on every run (38-49s).
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: >-
|
||||
pre-commit-focus-project-${{ env.pythonLocation }}-${{
|
||||
hashFiles('focus_project/.pre-commit-config.yaml') }}
|
||||
# Prefix fallback, or this fix would evaporate on the very
|
||||
# event that motivated it: bumping a hook rev changes the
|
||||
# exact key, and a total miss means reinstalling every hook
|
||||
# again. pre-commit's store is keyed internally by hook
|
||||
# repo+rev, so a superseded entry is ignored disk rather
|
||||
# than a wrong hook, and cache still re-saves under the new
|
||||
# exact key after a prefix-only hit.
|
||||
restore-keys: pre-commit-focus-project-${{ env.pythonLocation }}-
|
||||
# mypy re-analyses the stdlib and every third-party stub from scratch
|
||||
# without this, which measured 10s of the hook step in CI against 0.25s
|
||||
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
|
||||
# the config, not the sources: mypy's own cache metadata records the
|
||||
# interpreter, the flag set and a hash per source file, so it re-checks
|
||||
# what changed and cannot pass stale code. Deliberately NOT keyed on
|
||||
# github.sha -- that re-saves 19MB x3 every commit and would evict the
|
||||
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
|
||||
# the expensive part, the dependency analysis, which is what costs 10s.
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: >-
|
||||
mypy-focus-project-${{ env.pythonLocation }}-${{
|
||||
hashFiles('focus_project/.pre-commit-config.yaml') }}
|
||||
restore-keys: mypy-focus-project-${{ env.pythonLocation }}-
|
||||
- name: Run the hooks
|
||||
run: >-
|
||||
pre-commit run --config focus_project/.pre-commit-config.yaml
|
||||
--all-files --show-diff-on-failure
|
||||
34
.github/workflows/focus-project-tests.yml
vendored
34
.github/workflows/focus-project-tests.yml
vendored
@ -1,34 +0,0 @@
|
||||
name: focus-project tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as focus_project: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['focus_project/**', '.github/workflows/focus-project-tests.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['focus_project/**', '.github/workflows/focus-project-tests.yml']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: focus_project
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: focus_project/requirements.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt && pip install --no-deps -e .
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: python -m pytest -v
|
||||
73
.github/workflows/freedays-precommit.yml
vendored
73
.github/workflows/freedays-precommit.yml
vendored
@ -1,73 +0,0 @@
|
||||
name: freedays pre-commit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as freedays: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['freedays/**', '.github/workflows/freedays-precommit.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['freedays/**', '.github/workflows/freedays-precommit.yml']
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
# Only pre-commit itself. Every linting hook runs in its own isolated
|
||||
# env -- ruff, mypy, pylint, bandit and codespell each come from their
|
||||
# own repo+rev, and the local hooks need nothing beyond bash and
|
||||
# python3 -- so installing the full requirements.txt spent 9s a run on
|
||||
# packages no pre-commit-stage hook imports. Verified by running all
|
||||
# three configs green in a venv holding pre-commit and nothing else.
|
||||
# The pin is read out of requirements.txt rather than repeated here, so
|
||||
# it cannot drift from what the dependency-freshness gate governs.
|
||||
- name: Install pre-commit
|
||||
run: |
|
||||
pip install "$(grep '^pre-commit==' freedays/requirements.txt)"
|
||||
|
||||
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
|
||||
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
|
||||
# package jobs shared one key, raced to overwrite it with hook sets built
|
||||
# from four different configs, and editing a per-package config did not
|
||||
# invalidate it. The cache was therefore silently wrong rather than
|
||||
# missing, and the hooks reinstalled from scratch on every run (38-49s).
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: >-
|
||||
pre-commit-freedays-${{ env.pythonLocation }}-${{
|
||||
hashFiles('freedays/.pre-commit-config.yaml') }}
|
||||
# Prefix fallback, or this fix would evaporate on the very
|
||||
# event that motivated it: bumping a hook rev changes the
|
||||
# exact key, and a total miss means reinstalling every hook
|
||||
# again. pre-commit's store is keyed internally by hook
|
||||
# repo+rev, so a superseded entry is ignored disk rather
|
||||
# than a wrong hook, and cache still re-saves under the new
|
||||
# exact key after a prefix-only hit.
|
||||
restore-keys: pre-commit-freedays-${{ env.pythonLocation }}-
|
||||
# mypy re-analyses the stdlib and every third-party stub from scratch
|
||||
# without this, which measured 10s of the hook step in CI against 0.25s
|
||||
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
|
||||
# the config, not the sources: mypy's own cache metadata records the
|
||||
# interpreter, the flag set and a hash per source file, so it re-checks
|
||||
# what changed and cannot pass stale code. Deliberately NOT keyed on
|
||||
# github.sha -- that re-saves 19MB x3 every commit and would evict the
|
||||
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
|
||||
# the expensive part, the dependency analysis, which is what costs 10s.
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: >-
|
||||
mypy-freedays-${{ env.pythonLocation }}-${{
|
||||
hashFiles('freedays/.pre-commit-config.yaml') }}
|
||||
restore-keys: mypy-freedays-${{ env.pythonLocation }}-
|
||||
- name: Run the hooks
|
||||
run: >-
|
||||
pre-commit run --config freedays/.pre-commit-config.yaml
|
||||
--all-files --show-diff-on-failure
|
||||
44
.github/workflows/freedays-tests.yml
vendored
44
.github/workflows/freedays-tests.yml
vendored
@ -1,44 +0,0 @@
|
||||
name: freedays tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as freedays: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['freedays/**', '.github/workflows/freedays-tests.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['freedays/**', '.github/workflows/freedays-tests.yml']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: freedays
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: freedays/requirements.txt
|
||||
|
||||
# crdt-sync is a sibling in this repo, so it installs from the checkout
|
||||
# rather than from a tag -- CI must test what is on this branch, not
|
||||
# what was last released. Deliberately NOT gatelock: importing it pulls
|
||||
# in tkinter, and this job has no display precisely because freedays
|
||||
# must work without one.
|
||||
- name: Install the sibling package
|
||||
run: pip install ../crdt-sync
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt && pip install --no-deps -e .
|
||||
|
||||
# No display needed: freedays draws nothing. It is policy, a JSON file
|
||||
# and a CLI.
|
||||
- name: Run tests with coverage
|
||||
run: python -m pytest -v
|
||||
64
.github/workflows/gatelock-precommit.yml
vendored
64
.github/workflows/gatelock-precommit.yml
vendored
@ -3,12 +3,10 @@ name: gatelock pre-commit
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as gatelock: editing only this file used
|
||||
# to trigger nothing, so a fix to the job itself could never prove itself.
|
||||
paths: ['gatelock/**', '.github/workflows/gatelock-precommit.yml']
|
||||
paths: ['gatelock/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['gatelock/**', '.github/workflows/gatelock-precommit.yml']
|
||||
paths: ['gatelock/**']
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
@ -17,56 +15,10 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
# Only pre-commit itself. Every linting hook runs in its own isolated
|
||||
# env -- ruff, mypy, pylint, bandit and codespell each come from their
|
||||
# own repo+rev, and the local hooks need nothing beyond bash and
|
||||
# python3 -- so installing the full requirements.txt spent 9s a run on
|
||||
# packages no pre-commit-stage hook imports. Verified by running all
|
||||
# three configs green in a venv holding pre-commit and nothing else.
|
||||
# The pin is read out of requirements.txt rather than repeated here, so
|
||||
# it cannot drift from what the dependency-freshness gate governs.
|
||||
- name: Install pre-commit
|
||||
run: |
|
||||
pip install "$(grep '^pre-commit==' gatelock/requirements.txt)"
|
||||
|
||||
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
|
||||
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
|
||||
# package jobs shared one key, raced to overwrite it with hook sets built
|
||||
# from four different configs, and editing a per-package config did not
|
||||
# invalidate it. The cache was therefore silently wrong rather than
|
||||
# missing, and the hooks reinstalled from scratch on every run (38-49s).
|
||||
- uses: actions/cache@v4
|
||||
python-version: "3.12"
|
||||
- name: Install dependencies
|
||||
working-directory: gatelock
|
||||
run: pip install -r requirements.txt
|
||||
- uses: pre-commit/action@v3.0.1
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: >-
|
||||
pre-commit-gatelock-${{ env.pythonLocation }}-${{
|
||||
hashFiles('gatelock/.pre-commit-config.yaml') }}
|
||||
# Prefix fallback, or this fix would evaporate on the very
|
||||
# event that motivated it: bumping a hook rev changes the
|
||||
# exact key, and a total miss means reinstalling every hook
|
||||
# again. pre-commit's store is keyed internally by hook
|
||||
# repo+rev, so a superseded entry is ignored disk rather
|
||||
# than a wrong hook, and cache still re-saves under the new
|
||||
# exact key after a prefix-only hit.
|
||||
restore-keys: pre-commit-gatelock-${{ env.pythonLocation }}-
|
||||
# mypy re-analyses the stdlib and every third-party stub from scratch
|
||||
# without this, which measured 10s of the hook step in CI against 0.25s
|
||||
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
|
||||
# the config, not the sources: mypy's own cache metadata records the
|
||||
# interpreter, the flag set and a hash per source file, so it re-checks
|
||||
# what changed and cannot pass stale code. Deliberately NOT keyed on
|
||||
# github.sha -- that re-saves 19MB x3 every commit and would evict the
|
||||
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
|
||||
# the expensive part, the dependency analysis, which is what costs 10s.
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: >-
|
||||
mypy-gatelock-${{ env.pythonLocation }}-${{
|
||||
hashFiles('gatelock/.pre-commit-config.yaml') }}
|
||||
restore-keys: mypy-gatelock-${{ env.pythonLocation }}-
|
||||
- name: Run the hooks
|
||||
run: >-
|
||||
pre-commit run --config gatelock/.pre-commit-config.yaml
|
||||
--all-files --show-diff-on-failure
|
||||
extra_args: --config gatelock/.pre-commit-config.yaml --all-files
|
||||
|
||||
26
.github/workflows/gatelock-tests.yml
vendored
26
.github/workflows/gatelock-tests.yml
vendored
@ -3,12 +3,10 @@ name: gatelock tests
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as gatelock: editing only this file used
|
||||
# to trigger nothing, so a fix to the job itself could never prove itself.
|
||||
paths: ['gatelock/**', '.github/workflows/gatelock-tests.yml']
|
||||
paths: ['gatelock/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['gatelock/**', '.github/workflows/gatelock-tests.yml']
|
||||
paths: ['gatelock/**']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
@ -16,26 +14,20 @@ jobs:
|
||||
defaults:
|
||||
run:
|
||||
working-directory: gatelock
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: gatelock/requirements.txt
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt
|
||||
|
||||
# The viewport, fit-check and keyboard tests drive real Tk: focus
|
||||
# delivery, canvas scrolling and widget heights are Tk's own behaviour,
|
||||
# and mocking them is what let a viewport that scrolled itself pass
|
||||
# every check for a week. Headless they skip, silently taking coverage
|
||||
# with them -- so the runner gets a display instead.
|
||||
- name: Install a virtual X display
|
||||
run: sudo apt-get update && sudo apt-get install -y xvfb
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: xvfb-run -a -s "-screen 0 1600x1200x24" python -m pytest -v
|
||||
run: python -m pytest -v
|
||||
|
||||
36
.github/workflows/github-device-auth-tests.yml
vendored
36
.github/workflows/github-device-auth-tests.yml
vendored
@ -1,36 +0,0 @@
|
||||
name: github_device_auth tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['github_device_auth/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['github_device_auth/**']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: github_device_auth
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Flutter
|
||||
uses: subosito/flutter-action@v2
|
||||
with:
|
||||
channel: stable
|
||||
|
||||
- name: Install dependencies
|
||||
run: flutter pub get
|
||||
|
||||
- name: Analyze
|
||||
run: flutter analyze
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: flutter test --coverage
|
||||
|
||||
- name: Enforce 100% line coverage
|
||||
run: scripts/check_coverage.sh coverage/lcov.info 100
|
||||
36
.github/workflows/google-account-token-tests.yml
vendored
36
.github/workflows/google-account-token-tests.yml
vendored
@ -1,36 +0,0 @@
|
||||
name: google_account_token tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['google_account_token/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['google_account_token/**']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: google_account_token
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Flutter
|
||||
uses: subosito/flutter-action@v2
|
||||
with:
|
||||
channel: stable
|
||||
|
||||
- name: Install dependencies
|
||||
run: flutter pub get
|
||||
|
||||
- name: Analyze
|
||||
run: flutter analyze
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: flutter test --coverage
|
||||
|
||||
- name: Enforce 100% line coverage
|
||||
run: scripts/check_coverage.sh coverage/lcov.info 100
|
||||
73
.github/workflows/holds-precommit.yml
vendored
73
.github/workflows/holds-precommit.yml
vendored
@ -1,73 +0,0 @@
|
||||
name: holds pre-commit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as holds: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['holds/**', '.github/workflows/holds-precommit.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['holds/**', '.github/workflows/holds-precommit.yml']
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
# Only pre-commit itself. Every linting hook runs in its own isolated
|
||||
# env -- ruff, mypy, pylint, bandit and codespell each come from their
|
||||
# own repo+rev, and the local hooks need nothing beyond bash and
|
||||
# python3 -- so installing the full requirements.txt spent 9s a run on
|
||||
# packages no pre-commit-stage hook imports. Verified by running all
|
||||
# three configs green in a venv holding pre-commit and nothing else.
|
||||
# The pin is read out of requirements.txt rather than repeated here, so
|
||||
# it cannot drift from what the dependency-freshness gate governs.
|
||||
- name: Install pre-commit
|
||||
run: |
|
||||
pip install "$(grep '^pre-commit==' holds/requirements.txt)"
|
||||
|
||||
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
|
||||
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
|
||||
# package jobs shared one key, raced to overwrite it with hook sets built
|
||||
# from four different configs, and editing a per-package config did not
|
||||
# invalidate it. The cache was therefore silently wrong rather than
|
||||
# missing, and the hooks reinstalled from scratch on every run (38-49s).
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: >-
|
||||
pre-commit-holds-${{ env.pythonLocation }}-${{
|
||||
hashFiles('holds/.pre-commit-config.yaml') }}
|
||||
# Prefix fallback, or this fix would evaporate on the very
|
||||
# event that motivated it: bumping a hook rev changes the
|
||||
# exact key, and a total miss means reinstalling every hook
|
||||
# again. pre-commit's store is keyed internally by hook
|
||||
# repo+rev, so a superseded entry is ignored disk rather
|
||||
# than a wrong hook, and cache still re-saves under the new
|
||||
# exact key after a prefix-only hit.
|
||||
restore-keys: pre-commit-holds-${{ env.pythonLocation }}-
|
||||
# mypy re-analyses the stdlib and every third-party stub from scratch
|
||||
# without this, which measured 10s of the hook step in CI against 0.25s
|
||||
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
|
||||
# the config, not the sources: mypy's own cache metadata records the
|
||||
# interpreter, the flag set and a hash per source file, so it re-checks
|
||||
# what changed and cannot pass stale code. Deliberately NOT keyed on
|
||||
# github.sha -- that re-saves 19MB x3 every commit and would evict the
|
||||
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
|
||||
# the expensive part, the dependency analysis, which is what costs 10s.
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: >-
|
||||
mypy-holds-${{ env.pythonLocation }}-${{
|
||||
hashFiles('holds/.pre-commit-config.yaml') }}
|
||||
restore-keys: mypy-holds-${{ env.pythonLocation }}-
|
||||
- name: Run the hooks
|
||||
run: >-
|
||||
pre-commit run --config holds/.pre-commit-config.yaml
|
||||
--all-files --show-diff-on-failure
|
||||
34
.github/workflows/holds-tests.yml
vendored
34
.github/workflows/holds-tests.yml
vendored
@ -1,34 +0,0 @@
|
||||
name: holds tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as holds: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['holds/**', '.github/workflows/holds-tests.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['holds/**', '.github/workflows/holds-tests.yml']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: holds
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: holds/requirements.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt && pip install --no-deps -e .
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: python -m pytest -v
|
||||
73
.github/workflows/leetcode-coach-precommit.yml
vendored
73
.github/workflows/leetcode-coach-precommit.yml
vendored
@ -1,73 +0,0 @@
|
||||
name: leetcode-coach pre-commit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as leetcode_coach: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['leetcode_coach/**', '.github/workflows/leetcode-coach-precommit.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['leetcode_coach/**', '.github/workflows/leetcode-coach-precommit.yml']
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
# Only pre-commit itself. Every linting hook runs in its own isolated
|
||||
# env -- ruff, mypy, pylint, bandit and codespell each come from their
|
||||
# own repo+rev, and the local hooks need nothing beyond bash and
|
||||
# python3 -- so installing the full requirements.txt spent 9s a run on
|
||||
# packages no pre-commit-stage hook imports. Verified by running all
|
||||
# three configs green in a venv holding pre-commit and nothing else.
|
||||
# The pin is read out of requirements.txt rather than repeated here, so
|
||||
# it cannot drift from what the dependency-freshness gate governs.
|
||||
- name: Install pre-commit
|
||||
run: |
|
||||
pip install "$(grep '^pre-commit==' leetcode_coach/requirements.txt)"
|
||||
|
||||
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
|
||||
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
|
||||
# package jobs shared one key, raced to overwrite it with hook sets built
|
||||
# from four different configs, and editing a per-package config did not
|
||||
# invalidate it. The cache was therefore silently wrong rather than
|
||||
# missing, and the hooks reinstalled from scratch on every run (38-49s).
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: >-
|
||||
pre-commit-leetcode-coach-${{ env.pythonLocation }}-${{
|
||||
hashFiles('leetcode_coach/.pre-commit-config.yaml') }}
|
||||
# Prefix fallback, or this fix would evaporate on the very
|
||||
# event that motivated it: bumping a hook rev changes the
|
||||
# exact key, and a total miss means reinstalling every hook
|
||||
# again. pre-commit's store is keyed internally by hook
|
||||
# repo+rev, so a superseded entry is ignored disk rather
|
||||
# than a wrong hook, and cache still re-saves under the new
|
||||
# exact key after a prefix-only hit.
|
||||
restore-keys: pre-commit-leetcode-coach-${{ env.pythonLocation }}-
|
||||
# mypy re-analyses the stdlib and every third-party stub from scratch
|
||||
# without this, which measured 10s of the hook step in CI against 0.25s
|
||||
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
|
||||
# the config, not the sources: mypy's own cache metadata records the
|
||||
# interpreter, the flag set and a hash per source file, so it re-checks
|
||||
# what changed and cannot pass stale code. Deliberately NOT keyed on
|
||||
# github.sha -- that re-saves 19MB x3 every commit and would evict the
|
||||
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
|
||||
# the expensive part, the dependency analysis, which is what costs 10s.
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: >-
|
||||
mypy-leetcode-coach-${{ env.pythonLocation }}-${{
|
||||
hashFiles('leetcode_coach/.pre-commit-config.yaml') }}
|
||||
restore-keys: mypy-leetcode-coach-${{ env.pythonLocation }}-
|
||||
- name: Run the hooks
|
||||
run: >-
|
||||
pre-commit run --config leetcode_coach/.pre-commit-config.yaml
|
||||
--all-files --show-diff-on-failure
|
||||
34
.github/workflows/leetcode-coach-tests.yml
vendored
34
.github/workflows/leetcode-coach-tests.yml
vendored
@ -1,34 +0,0 @@
|
||||
name: leetcode-coach tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as leetcode_coach: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['leetcode_coach/**', '.github/workflows/leetcode-coach-tests.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['leetcode_coach/**', '.github/workflows/leetcode-coach-tests.yml']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: leetcode_coach
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: leetcode_coach/requirements.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt && pip install --no-deps -e .
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: python -m pytest -v
|
||||
33
.github/workflows/md-naming.yml
vendored
33
.github/workflows/md-naming.yml
vendored
@ -1,33 +0,0 @@
|
||||
name: md-naming
|
||||
|
||||
# The markdown namespaces (README / CLAUDE* / DOCS*, no TODO*), enforced on push as well as at commit time.
|
||||
# The local pre-commit hook can be skipped (a fresh clone that never ran
|
||||
# `pre-commit install` has no hooks at all), so the convention needs a second
|
||||
# gate that no one can forget to install.
|
||||
#
|
||||
# Deliberately a SEPARATE workflow from any release job: a badly named doc
|
||||
# should fail lint, not block publishing an artifact.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
md-naming:
|
||||
name: markdown naming convention
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
# The gate and its exemption list live in the shared utils repo so every
|
||||
# repo enforces the same namespaces. Checking it out beats vendoring a
|
||||
# copy here, which is exactly how a gate and a survey start disagreeing.
|
||||
- name: Check out the shared gate
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: kuhyx/utils
|
||||
path: .utils
|
||||
|
||||
- name: Enforce the naming convention
|
||||
run: bash .utils/scripts/check_md_naming.sh --all
|
||||
31
.github/workflows/music-theory-tests.yml
vendored
31
.github/workflows/music-theory-tests.yml
vendored
@ -1,31 +0,0 @@
|
||||
name: music_theory tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as music_theory: editing only this file used
|
||||
# to trigger nothing, so a fix to the job itself could never prove itself.
|
||||
paths: ['music_theory/**', '.github/workflows/music-theory-tests.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['music_theory/**', '.github/workflows/music-theory-tests.yml']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: music_theory
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: python -m pytest -v --cov=music_theory
|
||||
@ -1,73 +0,0 @@
|
||||
name: omniroute_offload pre-commit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as omniroute_offload: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['omniroute_offload/**', '.github/workflows/omniroute_offload-precommit.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['omniroute_offload/**', '.github/workflows/omniroute_offload-precommit.yml']
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
# Only pre-commit itself. Every linting hook runs in its own isolated
|
||||
# env -- ruff, mypy, pylint, bandit and codespell each come from their
|
||||
# own repo+rev, and the local hooks need nothing beyond bash and
|
||||
# python3 -- so installing the full requirements.txt spent 9s a run on
|
||||
# packages no pre-commit-stage hook imports. Verified by running all
|
||||
# three configs green in a venv holding pre-commit and nothing else.
|
||||
# The pin is read out of requirements.txt rather than repeated here, so
|
||||
# it cannot drift from what the dependency-freshness gate governs.
|
||||
- name: Install pre-commit
|
||||
run: |
|
||||
pip install "$(grep '^pre-commit==' omniroute_offload/requirements.txt)"
|
||||
|
||||
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
|
||||
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
|
||||
# package jobs shared one key, raced to overwrite it with hook sets built
|
||||
# from four different configs, and editing a per-package config did not
|
||||
# invalidate it. The cache was therefore silently wrong rather than
|
||||
# missing, and the hooks reinstalled from scratch on every run (38-49s).
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: >-
|
||||
pre-commit-omniroute_offload-${{ env.pythonLocation }}-${{
|
||||
hashFiles('omniroute_offload/.pre-commit-config.yaml') }}
|
||||
# Prefix fallback, or this fix would evaporate on the very
|
||||
# event that motivated it: bumping a hook rev changes the
|
||||
# exact key, and a total miss means reinstalling every hook
|
||||
# again. pre-commit's store is keyed internally by hook
|
||||
# repo+rev, so a superseded entry is ignored disk rather
|
||||
# than a wrong hook, and cache still re-saves under the new
|
||||
# exact key after a prefix-only hit.
|
||||
restore-keys: pre-commit-omniroute_offload-${{ env.pythonLocation }}-
|
||||
# mypy re-analyses the stdlib and every third-party stub from scratch
|
||||
# without this, which measured 10s of the hook step in CI against 0.25s
|
||||
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
|
||||
# the config, not the sources: mypy's own cache metadata records the
|
||||
# interpreter, the flag set and a hash per source file, so it re-checks
|
||||
# what changed and cannot pass stale code. Deliberately NOT keyed on
|
||||
# github.sha -- that re-saves 19MB x3 every commit and would evict the
|
||||
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
|
||||
# the expensive part, the dependency analysis, which is what costs 10s.
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: >-
|
||||
mypy-omniroute_offload-${{ env.pythonLocation }}-${{
|
||||
hashFiles('omniroute_offload/.pre-commit-config.yaml') }}
|
||||
restore-keys: mypy-omniroute_offload-${{ env.pythonLocation }}-
|
||||
- name: Run the hooks
|
||||
run: >-
|
||||
pre-commit run --config omniroute_offload/.pre-commit-config.yaml
|
||||
--all-files --show-diff-on-failure
|
||||
38
.github/workflows/palette-drift.yml
vendored
38
.github/workflows/palette-drift.yml
vendored
@ -1,38 +0,0 @@
|
||||
name: palette drift
|
||||
|
||||
# Deliberately NOT path-filtered. Every other workflow here scopes itself to
|
||||
# one package, which is right for tests but wrong for this check: drift is a
|
||||
# disagreement *between* packages, so editing design_system alone has to be
|
||||
# able to fail it. A `paths:` filter would let exactly the edit that causes
|
||||
# drift be the one that skips the check.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
palette:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
|
||||
# Compares tokens.md (prose freeze), web_ui/src/tokens.css,
|
||||
# design_system tokens.dart and gatelock's LockConfig. Fails on a hex
|
||||
# that disagrees across stacks, and on a colour token that no stack's
|
||||
# map accounts for.
|
||||
- name: One palette across all four token sources
|
||||
run: python3 unified-design-system/scripts/palette_check.py
|
||||
|
||||
- name: Categorical ramp still passes its constraints
|
||||
run: python3 unified-design-system/scripts/ramp_check.py
|
||||
|
||||
# Spacing, radius, type, shadow and motion: palette_check.py parses
|
||||
# hex only, so these scales were unguarded until structural_check.py.
|
||||
- name: Structural scales agree across all token sources
|
||||
run: python3 unified-design-system/scripts/structural_check.py
|
||||
19
.github/workflows/repo-contract.yml
vendored
19
.github/workflows/repo-contract.yml
vendored
@ -1,19 +0,0 @@
|
||||
name: repo-contract
|
||||
|
||||
# utils owns the shared gate, so it runs its own copy directly (no shim, no
|
||||
# second checkout). Other repos use repo_contract/templates/repo-contract.yml.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
repo-contract:
|
||||
name: repo contract
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Enforce the repo contract
|
||||
run: bash scripts/check_repo_contract.sh
|
||||
82
.github/workflows/shared-gates-tests.yml
vendored
82
.github/workflows/shared-gates-tests.yml
vendored
@ -1,82 +0,0 @@
|
||||
name: shared gate tests
|
||||
|
||||
# The gates in this repo are enforced in ~40 other repos, so a bug here is a
|
||||
# bug everywhere at once. Until now `md_naming` had a test suite that no
|
||||
# workflow ran; `dep_freshness` ships with one from day one.
|
||||
#
|
||||
# The dependency-freshness suite never touches the network -- every registry
|
||||
# answer is a fixture, because the gate's worst failure mode (accepting a
|
||||
# pre-release as "latest stable") fails GREEN and only a fixture can pin it.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
shared-gate-tests:
|
||||
name: gate unit tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
|
||||
- name: Install test dependencies
|
||||
run: python -m pip install --upgrade pytest pytest-cov pyyaml packaging
|
||||
|
||||
- name: dep_freshness — 100% branch coverage on the gate itself
|
||||
run: |
|
||||
PYTHONPATH=. python -m pytest dep_freshness/tests -q \
|
||||
--cov=dep_freshness --cov-branch \
|
||||
--cov-report=term-missing \
|
||||
--cov-fail-under=99
|
||||
|
||||
- name: md_naming
|
||||
run: PYTHONPATH=. python -m pytest md_naming/tests -q
|
||||
|
||||
# Whole package at 100% branch: the gate is enforced in every repo and
|
||||
# now reads a per-repo `.file-length-exempt`, whose fail-closed rule is
|
||||
# exactly the kind of branch a partial suite would leave green by accident.
|
||||
- name: file_length — 100% branch coverage on the gate itself
|
||||
run: |
|
||||
PYTHONPATH=. python -m pytest file_length/tests -q \
|
||||
--cov=file_length --cov-branch \
|
||||
--cov-report=term-missing \
|
||||
--cov-fail-under=100
|
||||
|
||||
- name: repo_contract — 100% branch coverage on the gate itself
|
||||
run: |
|
||||
PYTHONPATH=. python -m pytest repo_contract/tests -q \
|
||||
--cov=repo_contract --cov-branch \
|
||||
--cov-report=term-missing \
|
||||
--cov-fail-under=100
|
||||
|
||||
# Every web repo's pre-commit and CI run this checker against its dist/,
|
||||
# so a false green here ships an un-agent-ready site fleet-wide.
|
||||
- name: agent_ready — 100% branch coverage on the checker
|
||||
run: |
|
||||
PYTHONPATH=. python -m pytest agent_ready/tests -q \
|
||||
--cov=agent_ready --cov-branch \
|
||||
--cov-report=term-missing \
|
||||
--cov-fail-under=100
|
||||
|
||||
- name: coverage_gaps — 100% branch coverage on the lister
|
||||
run: |
|
||||
PYTHONPATH=. python -m pytest coverage_gaps/tests -q \
|
||||
--cov=coverage_gaps --cov-branch \
|
||||
--cov-report=term-missing \
|
||||
--cov-fail-under=100
|
||||
|
||||
# gate_stamp.sh is the pre-push gate in every repo. Its tests drive real
|
||||
# pre-commit in throwaway repos, and the hook-id helper runs under
|
||||
# /usr/bin/python3 (the deployment interpreter), hence apt's PyYAML.
|
||||
- name: gate_stamp (step / full --list)
|
||||
run: |
|
||||
sudo apt-get update -q && sudo apt-get install -y python3-yaml
|
||||
python -m pip install --upgrade pre-commit
|
||||
PYTHONPATH=. python -m pytest scripts/tests -q
|
||||
73
.github/workflows/sickdays-precommit.yml
vendored
73
.github/workflows/sickdays-precommit.yml
vendored
@ -1,73 +0,0 @@
|
||||
name: sickdays pre-commit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as sickdays: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['sickdays/**', '.github/workflows/sickdays-precommit.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['sickdays/**', '.github/workflows/sickdays-precommit.yml']
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
# Only pre-commit itself. Every linting hook runs in its own isolated
|
||||
# env -- ruff, mypy, pylint, bandit and codespell each come from their
|
||||
# own repo+rev, and the local hooks need nothing beyond bash and
|
||||
# python3 -- so installing the full requirements.txt spent 9s a run on
|
||||
# packages no pre-commit-stage hook imports. Verified by running all
|
||||
# three configs green in a venv holding pre-commit and nothing else.
|
||||
# The pin is read out of requirements.txt rather than repeated here, so
|
||||
# it cannot drift from what the dependency-freshness gate governs.
|
||||
- name: Install pre-commit
|
||||
run: |
|
||||
pip install "$(grep '^pre-commit==' sickdays/requirements.txt)"
|
||||
|
||||
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
|
||||
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
|
||||
# package jobs shared one key, raced to overwrite it with hook sets built
|
||||
# from four different configs, and editing a per-package config did not
|
||||
# invalidate it. The cache was therefore silently wrong rather than
|
||||
# missing, and the hooks reinstalled from scratch on every run (38-49s).
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: >-
|
||||
pre-commit-sickdays-${{ env.pythonLocation }}-${{
|
||||
hashFiles('sickdays/.pre-commit-config.yaml') }}
|
||||
# Prefix fallback, or this fix would evaporate on the very
|
||||
# event that motivated it: bumping a hook rev changes the
|
||||
# exact key, and a total miss means reinstalling every hook
|
||||
# again. pre-commit's store is keyed internally by hook
|
||||
# repo+rev, so a superseded entry is ignored disk rather
|
||||
# than a wrong hook, and cache still re-saves under the new
|
||||
# exact key after a prefix-only hit.
|
||||
restore-keys: pre-commit-sickdays-${{ env.pythonLocation }}-
|
||||
# mypy re-analyses the stdlib and every third-party stub from scratch
|
||||
# without this, which measured 10s of the hook step in CI against 0.25s
|
||||
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
|
||||
# the config, not the sources: mypy's own cache metadata records the
|
||||
# interpreter, the flag set and a hash per source file, so it re-checks
|
||||
# what changed and cannot pass stale code. Deliberately NOT keyed on
|
||||
# github.sha -- that re-saves 19MB x3 every commit and would evict the
|
||||
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
|
||||
# the expensive part, the dependency analysis, which is what costs 10s.
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: >-
|
||||
mypy-sickdays-${{ env.pythonLocation }}-${{
|
||||
hashFiles('sickdays/.pre-commit-config.yaml') }}
|
||||
restore-keys: mypy-sickdays-${{ env.pythonLocation }}-
|
||||
- name: Run the hooks
|
||||
run: >-
|
||||
pre-commit run --config sickdays/.pre-commit-config.yaml
|
||||
--all-files --show-diff-on-failure
|
||||
49
.github/workflows/sickdays-tests.yml
vendored
49
.github/workflows/sickdays-tests.yml
vendored
@ -1,49 +0,0 @@
|
||||
name: sickdays tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as sickdays: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['sickdays/**', '.github/workflows/sickdays-tests.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['sickdays/**', '.github/workflows/sickdays-tests.yml']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: sickdays
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: sickdays/requirements.txt
|
||||
|
||||
# crdt-sync and freedays are siblings in this repo, so they install from
|
||||
# the checkout rather than from a tag -- CI must test what is on this
|
||||
# branch, not what was last released. Deliberately NOT gatelock:
|
||||
# importing it pulls in tkinter, and this job has no display precisely
|
||||
# because sickdays must work without one.
|
||||
- name: Install the sibling packages
|
||||
run: pip install ../crdt-sync && pip install --no-deps ../freedays
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt && pip install --no-deps -e .
|
||||
|
||||
# The library draws nothing, but sickdays.gui (0.3.0) does, and its
|
||||
# tests drive real Tk: option names and Checkbutton/variable wiring are
|
||||
# Tk's own behaviour. Headless they skip, taking coverage with them --
|
||||
# so the runner gets a virtual display, as gatelock's does.
|
||||
- name: Install a virtual X display
|
||||
run: sudo apt-get update && sudo apt-get install -y xvfb
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: xvfb-run -a -s "-screen 0 1600x1200x24" python -m pytest -v
|
||||
73
.github/workflows/sql-drills-precommit.yml
vendored
73
.github/workflows/sql-drills-precommit.yml
vendored
@ -1,73 +0,0 @@
|
||||
name: sql-drills pre-commit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as sql_drills: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['sql_drills/**', '.github/workflows/sql-drills-precommit.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['sql_drills/**', '.github/workflows/sql-drills-precommit.yml']
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
# Only pre-commit itself. Every linting hook runs in its own isolated
|
||||
# env -- ruff, mypy, pylint, bandit and codespell each come from their
|
||||
# own repo+rev, and the local hooks need nothing beyond bash and
|
||||
# python3 -- so installing the full requirements.txt spent 9s a run on
|
||||
# packages no pre-commit-stage hook imports. Verified by running all
|
||||
# three configs green in a venv holding pre-commit and nothing else.
|
||||
# The pin is read out of requirements.txt rather than repeated here, so
|
||||
# it cannot drift from what the dependency-freshness gate governs.
|
||||
- name: Install pre-commit
|
||||
run: |
|
||||
pip install "$(grep '^pre-commit==' sql_drills/requirements.txt)"
|
||||
|
||||
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
|
||||
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
|
||||
# package jobs shared one key, raced to overwrite it with hook sets built
|
||||
# from four different configs, and editing a per-package config did not
|
||||
# invalidate it. The cache was therefore silently wrong rather than
|
||||
# missing, and the hooks reinstalled from scratch on every run (38-49s).
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: >-
|
||||
pre-commit-sql-drills-${{ env.pythonLocation }}-${{
|
||||
hashFiles('sql_drills/.pre-commit-config.yaml') }}
|
||||
# Prefix fallback, or this fix would evaporate on the very
|
||||
# event that motivated it: bumping a hook rev changes the
|
||||
# exact key, and a total miss means reinstalling every hook
|
||||
# again. pre-commit's store is keyed internally by hook
|
||||
# repo+rev, so a superseded entry is ignored disk rather
|
||||
# than a wrong hook, and cache still re-saves under the new
|
||||
# exact key after a prefix-only hit.
|
||||
restore-keys: pre-commit-sql-drills-${{ env.pythonLocation }}-
|
||||
# mypy re-analyses the stdlib and every third-party stub from scratch
|
||||
# without this, which measured 10s of the hook step in CI against 0.25s
|
||||
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
|
||||
# the config, not the sources: mypy's own cache metadata records the
|
||||
# interpreter, the flag set and a hash per source file, so it re-checks
|
||||
# what changed and cannot pass stale code. Deliberately NOT keyed on
|
||||
# github.sha -- that re-saves 19MB x3 every commit and would evict the
|
||||
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
|
||||
# the expensive part, the dependency analysis, which is what costs 10s.
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: >-
|
||||
mypy-sql-drills-${{ env.pythonLocation }}-${{
|
||||
hashFiles('sql_drills/.pre-commit-config.yaml') }}
|
||||
restore-keys: mypy-sql-drills-${{ env.pythonLocation }}-
|
||||
- name: Run the hooks
|
||||
run: >-
|
||||
pre-commit run --config sql_drills/.pre-commit-config.yaml
|
||||
--all-files --show-diff-on-failure
|
||||
34
.github/workflows/sql-drills-tests.yml
vendored
34
.github/workflows/sql-drills-tests.yml
vendored
@ -1,34 +0,0 @@
|
||||
name: sql-drills tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as sql_drills: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['sql_drills/**', '.github/workflows/sql-drills-tests.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['sql_drills/**', '.github/workflows/sql-drills-tests.yml']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: sql_drills
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: sql_drills/requirements.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt && pip install --no-deps -e .
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: python -m pytest -v
|
||||
36
.github/workflows/sync-settings-ui-tests.yml
vendored
36
.github/workflows/sync-settings-ui-tests.yml
vendored
@ -1,36 +0,0 @@
|
||||
name: sync_settings_ui tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['sync_settings_ui/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['sync_settings_ui/**']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: sync_settings_ui
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Flutter
|
||||
uses: subosito/flutter-action@v2
|
||||
with:
|
||||
channel: stable
|
||||
|
||||
- name: Install dependencies
|
||||
run: flutter pub get
|
||||
|
||||
- name: Analyze
|
||||
run: flutter analyze
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: flutter test --coverage
|
||||
|
||||
- name: Enforce 100% line coverage
|
||||
run: scripts/check_coverage.sh coverage/lcov.info 100
|
||||
37
.github/workflows/ts-config-tests.yml
vendored
37
.github/workflows/ts-config-tests.yml
vendored
@ -1,37 +0,0 @@
|
||||
name: ts_config tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['ts_config/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['ts_config/**']
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ts_config
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 12
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "24.22.1"
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Lint (the preset lints itself)
|
||||
run: pnpm lint
|
||||
|
||||
- name: Fixture + factory tests with the 100% coverage gate
|
||||
run: pnpm coverage
|
||||
57
.github/workflows/ts-core-tests.yml
vendored
57
.github/workflows/ts-core-tests.yml
vendored
@ -1,57 +0,0 @@
|
||||
name: ts_core tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['ts_core/**', 'ts_config/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['ts_core/**', 'ts_config/**']
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ts_core
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 12
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "24.22.1"
|
||||
|
||||
# The preset is a `link:../ts_config` dependency. pnpm links the
|
||||
# directory but installs nothing inside it, and a flat config imports
|
||||
# its plugins from the preset's own node_modules -- so without this
|
||||
# step the lint dies on "Cannot find package '@eslint/js'".
|
||||
- name: Install the shared lint preset's dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
working-directory: ts_config
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Lint
|
||||
run: pnpm lint
|
||||
|
||||
- name: Run tests with the 100% coverage gate
|
||||
run: pnpm coverage
|
||||
|
||||
# The tag ships a prebuilt dist/ because Vite does not transpile
|
||||
# dependencies. A committed dist/ that does not match src/ would ship
|
||||
# stale code to every consumer with no error anywhere, so rebuild and
|
||||
# fail if the result differs from what is checked in.
|
||||
- name: dist/ is in sync with src/
|
||||
run: |
|
||||
pnpm build
|
||||
if ! git diff --exit-code -- dist; then
|
||||
echo "::error::dist/ is stale — run 'pnpm build' in ts_core and commit the result."
|
||||
exit 1
|
||||
fi
|
||||
61
.github/workflows/web-ui-tests.yml
vendored
61
.github/workflows/web-ui-tests.yml
vendored
@ -1,61 +0,0 @@
|
||||
name: web_ui tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['web_ui/**', 'ts_config/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['web_ui/**', 'ts_config/**']
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: web_ui
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 12
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "24.22.1"
|
||||
|
||||
# The preset is a `link:../ts_config` dependency. pnpm links the
|
||||
# directory but installs nothing inside it, and a flat config imports
|
||||
# its plugins from the preset's own node_modules -- so without this
|
||||
# step the lint dies on "Cannot find package '@eslint/js'".
|
||||
- name: Install the shared lint preset's dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
working-directory: ts_config
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Lint
|
||||
run: pnpm lint
|
||||
|
||||
- name: Run tests with the 100% coverage gate
|
||||
run: pnpm coverage
|
||||
|
||||
# The tag ships a prebuilt dist/ because Vite does not transpile
|
||||
# dependencies. A committed dist/ that does not match src/ would ship
|
||||
# stale code to every consumer with no error anywhere, so rebuild and
|
||||
# fail if the result differs from what is checked in.
|
||||
- name: dist/ is in sync with src/
|
||||
run: |
|
||||
pnpm build
|
||||
if ! git diff --exit-code -- dist; then
|
||||
echo "::error::dist/ is stale — run 'pnpm build' in web_ui and commit the result."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Categorical ramp still passes its constraints
|
||||
working-directory: .
|
||||
run: python3 unified-design-system/scripts/ramp_check.py
|
||||
25
.gitignore
vendored
25
.gitignore
vendored
@ -1,25 +0,0 @@
|
||||
# Python build/test artifacts, across every subproject in this monorepo.
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.egg-info/
|
||||
build/
|
||||
dist/
|
||||
.venv/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.pytest_cache/
|
||||
|
||||
# Coverage output. Each subproject writes these next to its own pyproject.toml,
|
||||
# and a run from the repo root drops a pair here too.
|
||||
.coverage
|
||||
.coverage.*
|
||||
coverage.lcov
|
||||
htmlcov/
|
||||
|
||||
# Firebase CLI debug exhaust. Written next to whichever directory the CLI was
|
||||
# invoked from (crdt-sync/ in practice), regenerated on every failed command,
|
||||
# and never worth committing.
|
||||
firebase-debug.log
|
||||
firebase-debug.*.log
|
||||
firestore-debug.log
|
||||
database-debug.log
|
||||
@ -1,84 +0,0 @@
|
||||
# Repo-wide gate for the shared 250-line file cap (refactor_claude_todo.md).
|
||||
#
|
||||
# A file that cannot be read in one piece forces re-reads and partial edits,
|
||||
# which is the largest avoidable cost in an LLM-assisted workflow. A refactor
|
||||
# without a gate silently regrows, so this is the part that makes it stick.
|
||||
#
|
||||
# Checks only the files in the commit, so an unrelated commit never breaks on
|
||||
# a violation it did not introduce, and it FAILS (exit 1) rather than warning.
|
||||
# No baseline and no allowlist -- those are suppressions.
|
||||
#
|
||||
# The subprojects (gatelock/, crdt-sync/, web_ui/, design_system/,
|
||||
# sync_settings_ui/) each keep their own .pre-commit-config.yaml for their own
|
||||
# linters, and carry this same hook so committing with one of those configs is
|
||||
# gated too. This root config is what `pre-commit install` wires up, and it is
|
||||
# the only one that covers the subprojects without a config of their own
|
||||
# (guard-lib/, crdt_sync_dart/, file_length/, ...).
|
||||
#
|
||||
# The path is repo-relative on purpose. pre-commit always runs hooks from the
|
||||
# git root, so this resolves the same locally and on a CI runner -- an absolute
|
||||
# /home/... path exits 127 in Actions, which is how this was first written.
|
||||
repos:
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: file-length-cap
|
||||
name: file length <= 250 lines
|
||||
entry: bash scripts/check_file_length.sh
|
||||
language: system
|
||||
pass_filenames: true
|
||||
|
||||
# Three markdown namespaces (README / CLAUDE* / DOCS*); TODO*.md is banned
|
||||
# (tasks live in the todo app).
|
||||
# Shared gate in ~/src/utils owns the namespaces and the exemption list, so
|
||||
# one repo's idea of a valid name cannot drift from every other repo's.
|
||||
# Since the 2026-09-11 reorganisation ~ holds eleven visible entries and
|
||||
# home-tidy sweeps anything else to inbox/ within the hour. The migration's
|
||||
# rewriter matched *literal* /home/kuhy/<name> strings, so paths assembled
|
||||
# at run time survived it -- the todo wrapper exported to a dead ~/todo for
|
||||
# a day while its MCP read ~/src/todo. This gate matches the shape, so it
|
||||
# stays correct as repos come and go.
|
||||
- id: home-path-layout
|
||||
name: home-relative paths stay inside the ~ layout
|
||||
entry: bash scripts/check_home_paths.sh
|
||||
language: system
|
||||
pass_filenames: true
|
||||
|
||||
- id: repo-contract
|
||||
name: repo contract (CLAUDE.md Commands + test_changed)
|
||||
entry: scripts/check_repo_contract.sh
|
||||
language: system
|
||||
pass_filenames: false
|
||||
files: ^(CLAUDE\.md|AGENTS\.md|scripts/test_changed\.sh)$|(^|/)(AndroidManifest\.xml|build\.gradle(\.kts)?|strings\.xml|pubspec\.yaml|repo-contract\.toml|[^/]+\.desktop|assets/icon/[^/]+\.png)$
|
||||
|
||||
- id: md-naming
|
||||
name: markdown naming convention
|
||||
entry: scripts/check_md_naming.sh
|
||||
language: system
|
||||
files: \.(md|markdown)$
|
||||
|
||||
# Every dependency on its ecosystem's newest STABLE release, exact-pinned
|
||||
# so the question is mechanically answerable at all.
|
||||
#
|
||||
# `files:` is what keeps this cheap: a commit touching only source makes
|
||||
# ZERO network requests, and manifest commits mostly hit a 6h disk cache.
|
||||
#
|
||||
# Runs WITHOUT --strict on purpose. Offline it degrades to the cached
|
||||
# answer and passes with a loud banner; `--no-verify` is banned here, so
|
||||
# a hook that hard-failed on a flaky connection would leave no way to
|
||||
# commit at all. CI, always online, is the strict authority.
|
||||
#
|
||||
# Scoped to manifests this branch changed (vs merge-base with main), so
|
||||
# `--all-files` asks "did THIS change leave a stale pin?" and not "is
|
||||
# every project in the monorepo current?". CI `--all --strict` keeps
|
||||
# the repo-wide answer. See scripts/dep_freshness_changed.sh.
|
||||
- id: dependency-freshness
|
||||
name: dependencies on newest stable
|
||||
entry: bash scripts/dep_freshness_changed.sh
|
||||
language: system
|
||||
pass_filenames: true
|
||||
files: >-
|
||||
(pubspec\.(yaml|lock)|pyproject\.toml|requirements.*\.txt|uv\.lock
|
||||
|package\.json|package-lock\.json|pnpm-lock\.yaml
|
||||
|Cargo\.(toml|lock)|go\.(mod|sum)
|
||||
|\.fvmrc|\.nvmrc|\.python-version
|
||||
|\.github/workflows/.*\.ya?ml)$
|
||||
@ -1,27 +0,0 @@
|
||||
# Hooks other repos consume with `repo: https://github.com/kuhyx/utils`.
|
||||
# All three are post-* stages, so each clone must install them (see
|
||||
# todo_hooks/README.md). They never fail a git operation: the work has
|
||||
# already happened, so server/gh trouble prints one warning and exits 0.
|
||||
- id: todo-closer
|
||||
name: close Todo trailer tasks
|
||||
entry: scripts/todo_hooks.sh closer
|
||||
language: script
|
||||
stages: [post-commit]
|
||||
always_run: true
|
||||
pass_filenames: false
|
||||
|
||||
- id: todo-import-pr-merge
|
||||
name: import PR Todo sections (pull)
|
||||
entry: scripts/todo_hooks.sh import-pr post-merge
|
||||
language: script
|
||||
stages: [post-merge]
|
||||
always_run: true
|
||||
pass_filenames: false
|
||||
|
||||
- id: todo-import-pr-rewrite
|
||||
name: import PR Todo sections (pull --rebase)
|
||||
entry: scripts/todo_hooks.sh import-pr post-rewrite
|
||||
language: script
|
||||
stages: [post-rewrite]
|
||||
always_run: true
|
||||
pass_filenames: false
|
||||
18
CLAUDE.md
18
CLAUDE.md
@ -1,18 +0,0 @@
|
||||
# utils
|
||||
|
||||
Monorepo of shared gates and libraries (dep_freshness, file_length, md_naming,
|
||||
repo_contract, coverage_gaps, crdt-sync, gatelock, freedays, sickdays, earned_time, music_theory,
|
||||
flutter_web_drive (headless Flutter-web E2E driver + MCP), ...).
|
||||
Python packages with a root-level `<pkg>/tests` run with `PYTHONPATH=.`;
|
||||
subprojects with their own `pyproject.toml` run via `scripts/run_subproject_tests.sh <dir>`.
|
||||
`scripts/check_repo_contract.sh` is the REAL shared gate (not a shim): never
|
||||
overwrite it with `repo_contract/templates/check_repo_contract.shim.sh`.
|
||||
|
||||
## Commands
|
||||
|
||||
- run: n/a: collection of gates and libraries, no single entry point
|
||||
- test: `PYTHONPATH=. python -m pytest -q agent_ready/tests coverage_gaps/tests dep_freshness/tests file_length/tests flutter_web_drive/tests home_paths/tests md_naming/tests repo_contract/tests scripts/tests unified-design-system/tests`
|
||||
- test-changed: `scripts/test_changed.sh`
|
||||
- lint: `ruff check agent_ready coverage_gaps flutter_web_drive repo_contract`
|
||||
- coverage: `PYTHONPATH=. python -m pytest -q agent_ready/tests flutter_web_drive/tests repo_contract/tests unified-design-system/tests --cov=agent_ready --cov=flutter_web_drive --cov=repo_contract --cov=conformance --cov=conformance_diff --cov-branch --cov-report=xml --cov-fail-under=100`
|
||||
- coverage-gaps: `coverage-gaps coverage.xml`
|
||||
79
README.md
79
README.md
@ -18,45 +18,10 @@ dependencies:
|
||||
crdt_sync:
|
||||
git:
|
||||
url: https://github.com/kuhyx/utils
|
||||
ref: crdt_sync_dart-v0.10.0
|
||||
ref: crdt_sync_dart-v0.2.0
|
||||
path: crdt_sync_dart
|
||||
```
|
||||
|
||||
## sync_settings_ui/
|
||||
|
||||
Shared Flutter "Sync settings" screen (pub package name: `sync_settings_ui`,
|
||||
not published to pub.dev). Firebase sync (primary) plus an optional local
|
||||
backup slot — no GitHub mirror UI. Pure UI + a `FirebaseSyncController`
|
||||
service layer over injected closures (keystore/Google-plugin access stays
|
||||
in each consuming app, exactly as before this package existed). Consumed by
|
||||
`todo`, `diet_guard_app`, `workout_app`, `wake_alarm/phone_app`, and
|
||||
`home_inventory`'s dedicated "Sync settings" route.
|
||||
|
||||
Consume via:
|
||||
```yaml
|
||||
dependencies:
|
||||
sync_settings_ui:
|
||||
git:
|
||||
url: https://github.com/kuhyx/utils
|
||||
ref: sync_settings_ui-v0.1.0
|
||||
path: sync_settings_ui
|
||||
```
|
||||
|
||||
## google_account_token/
|
||||
|
||||
Flutter plugin (Android only; other platforms return null): one-tap Google ID
|
||||
token via `AccountManager`, replacing `google_sign_in` in the sync apps.
|
||||
|
||||
Consume via:
|
||||
```yaml
|
||||
dependencies:
|
||||
google_account_token:
|
||||
git:
|
||||
url: https://github.com/kuhyx/utils
|
||||
ref: google_account_token-v0.1.0
|
||||
path: google_account_token
|
||||
```
|
||||
|
||||
## crdt-sync/
|
||||
|
||||
Python CRDT sync library (package: `crdt_sync`, not published to PyPI). Same
|
||||
@ -86,45 +51,3 @@ published to PyPI). Consumed via pip by `screen-locker`, `diet-guard`, and
|
||||
```
|
||||
gatelock @ git+https://github.com/kuhyx/utils@gatelock-v0.1.0#subdirectory=gatelock
|
||||
```
|
||||
|
||||
## music_theory/
|
||||
|
||||
Music theory from first principles as typed Python (package: `music_theory`,
|
||||
not published to PyPI): pitch, tuning, scales, chords, progressions, a numpy
|
||||
synth and a `Recipe` → WAV composer with a `degrade` transform. Follows Luke
|
||||
Haas's "Music Theory for Programmers" section by section; `python -m
|
||||
music_theory demo all --out DIR` renders one WAV per section. Consumed by
|
||||
`roadside-assistance` for its three world-state music beds.
|
||||
|
||||
Consume via:
|
||||
```
|
||||
music_theory @ git+https://github.com/kuhyx/utils@music-theory-v0.1.0#subdirectory=music_theory
|
||||
```
|
||||
|
||||
## unified-design-system/
|
||||
|
||||
Not a package — the frozen design tokens (colors, spacing, type, radius,
|
||||
shadow policy) and per-stack (Flutter/web/Tkinter) implementation patterns
|
||||
that keep every one of kuhy's repos visually identical, plus annotated
|
||||
component references (`components.html`, `button.html`). Read
|
||||
[`unified-design-system/README.md`](unified-design-system/README.md) and
|
||||
`DOCS-tokens.md` before touching any theme/CSS/style file in any repo. The
|
||||
`unified-design-system` Claude Code skill is a thin pointer here, not a
|
||||
duplicate.
|
||||
|
||||
## ts_config/
|
||||
|
||||
`@kuhyx/ts-config` — the one lint / TypeScript bar every kuhyx TypeScript
|
||||
repo extends (pnpm only, like `ts_core` and `web_ui`). A strict
|
||||
`tsconfig.base.json`, ESLint flat-config presets (`defineConfig()` stacking
|
||||
base / react / vitest / playwright / layer `boundaries()`), a type-free
|
||||
`eslint/fast` subset for editor hooks, and Knip / jscpd / Stryker bases.
|
||||
Plain JS, no build step; the git tag is the release. Every plugin is a
|
||||
`dependency` of the package so consumers list none. A fixture repo with one
|
||||
planted violation per rule family proves each gate bites.
|
||||
|
||||
```jsonc
|
||||
"devDependencies": {
|
||||
"@kuhyx/ts-config": "github:kuhyx/utils#ts_config-v0.1.0&path:/ts_config"
|
||||
}
|
||||
```
|
||||
|
||||
@ -1,97 +0,0 @@
|
||||
# AgentReady spec notes
|
||||
|
||||
These notes record what the checker implements and where each rule comes from.
|
||||
Fetched 2026-10-22.
|
||||
|
||||
## Sources
|
||||
|
||||
- AgentReady v2.0 (October 2026, MIT):
|
||||
<https://github.com/agentready-org/standard/blob/main/spec/spec.md>,
|
||||
canonical site <https://agentready.org>.
|
||||
- AI Catalog (the format behind `AR-FIND-04`):
|
||||
<https://github.com/Agent-Card/ai-catalog/blob/main/specification/ai-catalog.md>.
|
||||
AgentReady defines no catalog schema of its own ("AgentReady does not
|
||||
introduce its own bespoke manifest"), so the checker uses this one.
|
||||
- llms.txt: the spec's informative "llms.txt shape" example, and <https://llmstxt.org/>.
|
||||
|
||||
## Requirements (quoted from spec.md)
|
||||
|
||||
| ID | Level | Requirement | Checker |
|
||||
|---|---|---|---|
|
||||
| AR-FIND-01 | MUST | "The site **MUST NOT** block agent and crawler access to public content, in `robots.txt` or at the network layer." | robots.txt is 200 `text/plain` and not HTML. It has a `User-agent: *` group, and no group has a blanket `Disallow: /`. In url mode the homepage returns 200 with no challenge page for ClaudeBot, GPTBot, Claude-SearchBot, OAI-SearchBot, Claude-User and ChatGPT-User. |
|
||||
| AR-FIND-02 | SHOULD | "Answers agents are expected to cite **SHOULD** be published on one canonical, current, first-party docs page." | SKIP (manual) |
|
||||
| AR-FIND-03 | SHOULD | "The site **SHOULD** list its pages in a `sitemap.xml` referenced from `robots.txt`." | robots.txt has a `Sitemap:` line. The sitemap is XML (by content type in url mode), and every `<loc>` is on the site's own host. |
|
||||
| AR-FIND-04 | MAY | "...publish an AI Catalog at `/.well-known/ai-catalog.json` listing every agent-facing resource it ships ... linked from the homepage head with `<link rel="ai-catalog">` and an HTTP `Link` header." | The catalog is valid (see the schema below) and lists `/llms.txt`. The homepage has `<link rel="ai-catalog">`. In url mode the homepage also sends a `Link: <...>; rel="ai-catalog"` header. |
|
||||
| AR-READ-01 | MUST | "The answer **MUST** be present in the initial HTML — server-rendered or prerendered, not injected by a single-page app after load." | The homepage and each sitemap page return 200 `text/html` with at least `--min-text` characters of visible body text. A page that is byte-identical to the homepage fails. An empty sitemap is a FAIL. |
|
||||
| AR-READ-02 | MUST | "...accurate HTTP status codes: `404` (or a valid redirect) for missing paths ... never a `200` for a missing page" | url mode: two random `/agentready-probe-<hex>` paths must end, after redirects, on a 404 or 410. dist mode: a `404.html` with text must exist. |
|
||||
| AR-READ-03 | SHOULD | "The homepage **SHOULD** be served as raw HTML with real links to docs, product, and pricing." | The homepage HTML has at least one `<a href>` to another page on the site. |
|
||||
| AR-READ-04 | SHOULD | "Docs pages **SHOULD** answer the tasks users bring to agents." | SKIP (manual) |
|
||||
| AR-READ-05 | SHOULD | "Code snippets **SHOULD** be fenced and language-tagged." | Every `<pre><code>` carries `language-*`, `lang-*` or `data-language`. Every fence in a markdown mirror has an info string. |
|
||||
| AR-READ-06 | SHOULD | "Discovery files (`llms.txt`, `.well-known/*`, `openapi.json`, `AGENTS.md`) **SHOULD** be linked from content agents already visit" | `/llms.txt` and the catalog (plus `/openapi.json` with `--act`) exist and are linked from the homepage, by `<link>` or `<a>`. |
|
||||
| AR-READ-07 | SHOULD | "The site **SHOULD** ship an `llms.txt`: a lean, described index of its key pages." | 200 `text/plain` or `text/markdown`, not HTML. It needs an H1 first, a `>` summary, and an `## H2` with a `- [name](url)` list; other prose is allowed. Every on-site link resolves. |
|
||||
| AR-READ-08 | SHOULD | "Key pages **SHOULD** carry JSON-LD — the homepage above all" | The homepage has a parseable `application/ld+json` block with `@context`. |
|
||||
| AR-READ-09 | MAY | "Pages **MAY** advertise a markdown mirror via `<link rel="alternate" type="text/markdown">` or an equivalent HTTP `Link:` header." | Every checked page advertises a mirror. The mirror is 200 `text/markdown` or `text/plain`, and is not HTML. |
|
||||
| AR-READ-10 | n/a | "`AR-READ-10` is retired." | not emitted |
|
||||
| AR-READ-11 | SHOULD | "Pricing **SHOULD** be published on a first-party page" | SKIP (n/a: no paid plans) |
|
||||
| AR-ACT-01 | SHOULD | auth completable by an agent; "OAuth 2.0 with discoverable metadata under `/.well-known`" | `--act`: either `oauth-protected-resource` has `resource`, or `oauth-authorization-server` has `issuer`. |
|
||||
| AR-ACT-02 | SHOULD | "A public API **SHOULD** be described by a machine-readable OpenAPI contract." | `--act`: `/openapi.json` has `openapi` and `paths`, and every operation has an `operationId`. |
|
||||
| AR-ACT-03 | SHOULD | "...expose an MCP server over Streamable HTTP at `/mcp`." | `--act`, url mode: a POST `initialize` gets a JSON-RPC result, or a 401 with `WWW-Authenticate`. |
|
||||
| AR-ACT-04 | MAY | "An MCP server **MAY** ship a server card" | `--act`: the catalog has an `application/mcp-server-card+json` entry. |
|
||||
| AR-ACT-05/06/08/09 | MAY/SHOULD | MCP Apps, SDKs+CLI, payments, NLWeb `/ask` | `--act`: SKIP (manual) |
|
||||
| AR-ACT-07 | SHOULD | "...register WebMCP tools on `document.modelContext`." | `--act`: `modelContext` appears in the homepage HTML or in its same-site scripts. |
|
||||
| AR-ACT-10 | MUST | "...**MUST** expose at least one interface an agent can act through: an MCP server (`AR-ACT-03`) or a CLI (`AR-ACT-06`)." | `--act`: same verdict as the `/mcp` probe. |
|
||||
|
||||
## kuhy's choices that are stricter than the spec
|
||||
|
||||
- Every in-scope item is required, whatever its strength. That includes the
|
||||
MAY items (FIND-04, READ-09).
|
||||
- The spec says a site "**MAY** block a training crawler". kuhy chose to allow
|
||||
training crawlers too, so a blanket `Disallow` for GPTBot, ClaudeBot or any
|
||||
other agent is a FAIL.
|
||||
- The act stage is off by default. Turn it on with `--act` only for sites with
|
||||
user actions. The spec allows this: "a purely informational site conforms on
|
||||
`AR-FIND-*` and `AR-READ-*` alone".
|
||||
|
||||
## AI Catalog schema (ai-catalog.md, specVersion 1.0)
|
||||
|
||||
- The media type is `application/ai-catalog+json`. When fetched by link
|
||||
discovery, plain `application/json` is also acceptable.
|
||||
- At the top level, the object MUST have:
|
||||
- `specVersion`: a string in "Major.Minor" form, for example `"1.0"`.
|
||||
- `entries`: an array, which may be empty.
|
||||
- The top level MAY have `host`, `extensions` and `signature`.
|
||||
- If `host` is present, it MUST have `displayName`. It MAY have
|
||||
`identifier`, `documentationUrl` and `logoUrl`.
|
||||
- Each entry MUST have:
|
||||
- `identifier`: a URI or URN. The recommended form is
|
||||
`urn:air:{publisher}:{namespace}:{name}`.
|
||||
- `type`: a media type, for example `application/mcp-server-card+json`.
|
||||
- **exactly one** of `url` or `data`.
|
||||
- `displayName` is optional on entries (ai-catalog ADR-0016).
|
||||
- Link discovery uses `Link: <...>; rel="ai-catalog"` or
|
||||
`<link rel="ai-catalog" href="..." type="application/ai-catalog+json">`.
|
||||
|
||||
Minimal valid example:
|
||||
|
||||
```json
|
||||
{"specVersion": "1.0",
|
||||
"host": {"displayName": "kuhy"},
|
||||
"entries": [{"identifier": "urn:air:kuhy.duckdns.org:docs:llms",
|
||||
"type": "text/plain", "url": "https://kuhy.duckdns.org/llms.txt"}]}
|
||||
```
|
||||
|
||||
Agentic Resource Discovery v0.91 (<https://agenticresourcediscovery.org/spec>)
|
||||
renamed the path to `/.well-known/ard.json` and the relation to `rel="ard"`.
|
||||
It keeps `ai-catalog.json` only as an optional predecessor. AgentReady v2.0
|
||||
still names `ai-catalog.json`, so that is the path the checker tests. It does
|
||||
not test `ard.json`.
|
||||
|
||||
## llms.txt shape (spec.md, informative)
|
||||
|
||||
```md
|
||||
# Acme
|
||||
> Acme ships a payments API for developers.
|
||||
|
||||
## Docs
|
||||
- [Quickstart](https://example.com/docs/quickstart): first charge in one request
|
||||
```
|
||||
@ -1,93 +0,0 @@
|
||||
# agent_ready
|
||||
|
||||
This tool checks kuhy's websites against [AgentReady](https://agentready.org)
|
||||
v2.0. It runs every find and read item, and the act items when you pass `--act`.
|
||||
It is Python 3 standard library only, with no dependencies.
|
||||
`DOCS-spec-notes.md` maps each `AR-*` ID to the check that tests it.
|
||||
|
||||
## Usage
|
||||
|
||||
```sh
|
||||
~/src/utils/agent_ready/check.sh dist <build-dir> [--act] [--min-text N] [--max-pages N]
|
||||
~/src/utils/agent_ready/check.sh url <base-url> [--act] [--min-text N] [--max-pages N] [--timeout S] [--budget S]
|
||||
```
|
||||
|
||||
The output is one line per check:
|
||||
|
||||
```text
|
||||
PASS AR-FIND-03 sitemap lists 12 pages
|
||||
FAIL AR-READ-02 soft 404, unknown paths must return 404: /agentready-probe-1a2b -> HTTP 200
|
||||
SKIP AR-READ-11 pricing page: n/a for sites without paid plans
|
||||
```
|
||||
|
||||
| exit | meaning |
|
||||
|---|---|
|
||||
| 0 | every check passed (SKIPs do not fail) |
|
||||
| 1 | at least one FAIL, at any strength: MUST, SHOULD or MAY |
|
||||
| 2 | usage error (bad arguments, missing directory, not an http(s) URL) |
|
||||
| 3 | network undeterminable: the homepage was unreachable, or a fetch timed out or hit the 45 s budget, and nothing failed |
|
||||
|
||||
## Pre-commit and CI (dist mode)
|
||||
|
||||
Point the checker at the build output after the build step. Dist mode checks
|
||||
that each file exists and is well-formed:
|
||||
|
||||
- robots.txt
|
||||
- sitemap.xml
|
||||
- llms.txt
|
||||
- the AI Catalog
|
||||
- 404.html
|
||||
- the text in every sitemap page's HTML
|
||||
- each page's markdown mirror
|
||||
|
||||
```yaml
|
||||
# .github/workflows/*.yml, after the build
|
||||
- run: ~/src/utils/agent_ready/check.sh dist dist/
|
||||
```
|
||||
|
||||
```yaml
|
||||
# .pre-commit-config.yaml (local hook; build first, e.g. in a wrapper script)
|
||||
- id: agent-ready
|
||||
name: AgentReady (build output)
|
||||
entry: bash -c 'pnpm build >/dev/null && ~/src/utils/agent_ready/check.sh dist dist'
|
||||
language: system
|
||||
pass_filenames: false
|
||||
```
|
||||
|
||||
Dist mode cannot check HTTP behaviour: content types, the `Link` header, real
|
||||
404 status, or responses to agent user-agents. Those checks print as SKIP, or
|
||||
the check falls back to "404.html exists".
|
||||
|
||||
## Post-deploy (url mode)
|
||||
|
||||
Run this once the deploy is live:
|
||||
|
||||
```sh
|
||||
~/src/utils/agent_ready/check.sh url https://kuhy.duckdns.org
|
||||
```
|
||||
|
||||
Url mode checks status codes **and** content types. A server that answers
|
||||
every path with `200 text/html` (an SPA fallback) fails on robots.txt, the
|
||||
sitemap, llms.txt, the catalog and the 404 probe. A status-only check would
|
||||
pass it. Url mode also checks that:
|
||||
|
||||
- `/agentready-probe-<random>` ends on a 404, after following redirects.
|
||||
- No sitemap page is byte-identical to the homepage shell.
|
||||
- The homepage answers 200 to ClaudeBot, GPTBot and the other agent
|
||||
user-agents.
|
||||
|
||||
## Tests
|
||||
|
||||
```sh
|
||||
~/.claude/scripts/capped.sh python -m pytest ~/src/utils/agent_ready/tests -q
|
||||
```
|
||||
|
||||
The test fixtures are generated in `tmp_path` by `tests/sitegen.py`, so there
|
||||
are no static mirror `.md` files to trip the markdown-naming gate. The good
|
||||
and bad build trees are served by loopback `http.server` instances in four
|
||||
shapes:
|
||||
|
||||
- compliant
|
||||
- SPA fallback
|
||||
- JSON 404 on every path
|
||||
- 403 to GPTBot
|
||||
@ -1 +0,0 @@
|
||||
"""AgentReady (agentready.org v2.0) compliance checker for kuhy's websites."""
|
||||
@ -1,188 +0,0 @@
|
||||
"""Two backends behind one ``get(path)``: a build directory and a live site.
|
||||
|
||||
DirSite answers like a plain static host (index.html for ``/x/``, a 404 for
|
||||
anything missing), so the find/read checks are written once for both modes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.client
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote, urljoin, urlsplit
|
||||
|
||||
from agent_ready.report import Undeterminable
|
||||
|
||||
DEFAULT_UA = "Claude-User/1.0 (+agentready-check)"
|
||||
MAX_BODY = 5 * 1024 * 1024
|
||||
_EXT_MIME = {
|
||||
".html": "text/html",
|
||||
".md": "text/markdown",
|
||||
".txt": "text/plain",
|
||||
".xml": "application/xml",
|
||||
".json": "application/json",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Response:
|
||||
"""A fetched resource: final status, bare media type, headers, body."""
|
||||
|
||||
status: int
|
||||
ctype: str
|
||||
body: bytes
|
||||
headers: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def text(self) -> str:
|
||||
"""Body decoded as UTF-8 (lenient)."""
|
||||
return self.body.decode("utf-8", "replace")
|
||||
|
||||
|
||||
class Site:
|
||||
"""Common URL handling; subclasses implement ``_fetch``."""
|
||||
|
||||
live = False
|
||||
|
||||
def __init__(self, origin: str) -> None:
|
||||
self.origin = origin.rstrip("/")
|
||||
self._cache: dict[tuple[str, str], Response | Undeterminable] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def to_path(self, url: str, base: str = "/") -> str | None:
|
||||
"""Site path for a URL or relative href; None when off-site."""
|
||||
absolute = urljoin(self.origin + base, url.strip())
|
||||
parts = urlsplit(absolute)
|
||||
if (
|
||||
parts.scheme not in ("http", "https")
|
||||
or parts.netloc != urlsplit(self.origin).netloc
|
||||
):
|
||||
return None
|
||||
return parts.path or "/"
|
||||
|
||||
def get(self, path: str, ua: str = DEFAULT_UA) -> Response:
|
||||
"""Fetch ``path`` (memoized, including network failures)."""
|
||||
key = (path, ua)
|
||||
with self._lock:
|
||||
hit = self._cache.get(key)
|
||||
if hit is None:
|
||||
try:
|
||||
hit = self._fetch(path, ua)
|
||||
except Undeterminable as exc:
|
||||
hit = exc
|
||||
with self._lock:
|
||||
self._cache[key] = hit
|
||||
if isinstance(hit, Undeterminable):
|
||||
raise hit
|
||||
return hit
|
||||
|
||||
def prefetch(self, paths: list[str], uas: tuple[str, ...] = (DEFAULT_UA,)) -> None:
|
||||
"""Warm the cache for every (path, ua) pair; parallel for live sites."""
|
||||
if not self.live:
|
||||
return
|
||||
pairs = [(p, ua) for p in dict.fromkeys(paths) for ua in uas]
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
list(pool.map(lambda pair: self._safe_get(*pair), pairs))
|
||||
|
||||
def _safe_get(self, path: str, ua: str) -> None:
|
||||
try:
|
||||
self.get(path, ua)
|
||||
except Undeterminable:
|
||||
pass
|
||||
|
||||
def _fetch(self, path: str, ua: str) -> Response:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class DirSite(Site):
|
||||
"""A build output directory served the way a static host serves it."""
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
super().__init__("https://dist.invalid")
|
||||
self.root = root.resolve()
|
||||
|
||||
def file_for(self, path: str) -> Path | None:
|
||||
"""The file a static host would serve for ``path``."""
|
||||
rel = unquote(path.split("?", 1)[0]).lstrip("/")
|
||||
base = (self.root / rel).resolve()
|
||||
if not base.is_relative_to(self.root):
|
||||
return None
|
||||
cands = (
|
||||
[base / "index.html"]
|
||||
if path.endswith("/") or not rel
|
||||
else [base, base.with_name(base.name + ".html"), base / "index.html"]
|
||||
)
|
||||
return next((c for c in cands if c.is_file()), None)
|
||||
|
||||
def _fetch(self, path: str, ua: str) -> Response:
|
||||
target = self.file_for(path)
|
||||
if target is None:
|
||||
fallback = self.root / "404.html"
|
||||
body = fallback.read_bytes() if fallback.is_file() else b""
|
||||
return Response(404, "text/html", body)
|
||||
return Response(
|
||||
200,
|
||||
_EXT_MIME.get(target.suffix, "application/octet-stream"),
|
||||
target.read_bytes(),
|
||||
)
|
||||
|
||||
|
||||
class HttpSite(Site):
|
||||
"""A live site. Follows redirects; HTTP errors are responses, not raises."""
|
||||
|
||||
live = True
|
||||
|
||||
def __init__(self, base: str, timeout: float = 8.0, budget: float = 45.0) -> None:
|
||||
parts = urlsplit(base)
|
||||
super().__init__(f"{parts.scheme}://{parts.netloc}")
|
||||
self.timeout = timeout
|
||||
self.deadline = time.monotonic() + budget
|
||||
|
||||
def _remaining(self) -> float:
|
||||
left = self.deadline - time.monotonic()
|
||||
if left < 1:
|
||||
raise Undeterminable("overall time budget used up")
|
||||
return min(self.timeout, left)
|
||||
|
||||
def _fetch(self, path: str, ua: str) -> Response:
|
||||
return self.request(path, ua)
|
||||
|
||||
def request(
|
||||
self,
|
||||
path: str,
|
||||
ua: str = DEFAULT_UA,
|
||||
data: bytes | None = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
) -> Response:
|
||||
"""Uncached GET (or POST when ``data`` is given)."""
|
||||
req = urllib.request.Request(
|
||||
self.origin + path,
|
||||
data=data,
|
||||
headers={"User-Agent": ua, "Accept": "*/*", **(headers or {})},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=self._remaining()) as resp:
|
||||
return _response(resp.status, resp.headers, resp.read(MAX_BODY))
|
||||
except urllib.error.HTTPError as exc:
|
||||
return _response(exc.code, exc.headers, exc.read(MAX_BODY))
|
||||
except (
|
||||
urllib.error.URLError,
|
||||
TimeoutError,
|
||||
OSError,
|
||||
http.client.HTTPException,
|
||||
) as exc:
|
||||
raise Undeterminable(f"{path}: {getattr(exc, 'reason', exc)}") from exc
|
||||
|
||||
|
||||
def _response(status: int, headers: http.client.HTTPMessage, body: bytes) -> Response:
|
||||
"""Normalise headers: lowercase keys, repeated headers joined by ', '."""
|
||||
flat: dict[str, str] = {}
|
||||
for key in set(headers.keys()):
|
||||
flat[key.lower()] = ", ".join(headers.get_all(key) or [])
|
||||
ctype = flat.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
return Response(status, ctype, body, flat)
|
||||
@ -1,137 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""AgentReady compliance checker (agentready.org spec v2.0).
|
||||
|
||||
check.py dist <build-dir> [--act] # pre-commit / CI: build output
|
||||
check.py url <base-url> [--act] # post-deploy: the live site
|
||||
|
||||
Prints ``PASS|FAIL|SKIP AR-XXX-NN <msg>`` per check. Exit 0 all pass, 1 any
|
||||
FAIL (every in-scope item is required, whatever its MUST/SHOULD/MAY
|
||||
strength), 2 usage error, 3 network undeterminable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
if __package__ in (None, ""): # run as a plain script: make the package importable
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from agent_ready import checks_find, checks_read
|
||||
from agent_ready.backends import DirSite, HttpSite, Site
|
||||
from agent_ready.checks_act import run_act
|
||||
from agent_ready.context import HOME, Ctx
|
||||
from agent_ready.report import (
|
||||
EXIT_NETWORK,
|
||||
EXIT_USAGE,
|
||||
Report,
|
||||
Undeterminable,
|
||||
)
|
||||
|
||||
# (AR id used when a network failure aborts the check, check function)
|
||||
CHECKS: list[tuple[str, Callable[[Ctx], None]]] = [
|
||||
("AR-FIND-01", checks_find.check_robots),
|
||||
("AR-FIND-01", checks_find.check_agent_access),
|
||||
("AR-FIND-03", checks_find.check_sitemap),
|
||||
("AR-FIND-04", checks_find.check_catalog),
|
||||
("AR-READ-01", checks_read.check_pages),
|
||||
("AR-READ-02", checks_read.check_status_codes),
|
||||
("AR-READ-03", checks_read.check_homepage),
|
||||
("AR-READ-06", checks_read.check_discovery_links),
|
||||
("AR-READ-07", checks_read.check_llms),
|
||||
("AR-FIND-02", checks_read.skip_manual),
|
||||
("AR-ACT-10", run_act),
|
||||
]
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None) -> argparse.Namespace:
|
||||
"""Parse the command line (argparse exits 2 on usage errors)."""
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="agent_ready", description=__doc__.split("\n\n")[0]
|
||||
)
|
||||
sub = parser.add_subparsers(dest="mode", required=True)
|
||||
for mode, target in (
|
||||
("dist", "build output directory"),
|
||||
("url", "base URL, e.g. https://example.org"),
|
||||
):
|
||||
cmd = sub.add_parser(mode, help=f"check a {target}")
|
||||
cmd.add_argument("target", help=target)
|
||||
cmd.add_argument(
|
||||
"--act",
|
||||
action="store_true",
|
||||
help="also check the act stage (OpenAPI, MCP, auth)",
|
||||
)
|
||||
cmd.add_argument(
|
||||
"--min-text",
|
||||
type=int,
|
||||
default=100,
|
||||
help="min visible chars per page (default 100)",
|
||||
)
|
||||
cmd.add_argument(
|
||||
"--max-pages",
|
||||
type=int,
|
||||
default=40,
|
||||
help="url mode: max pages to fetch (default 40; dist checks all)",
|
||||
)
|
||||
url = sub.choices["url"]
|
||||
url.add_argument(
|
||||
"--timeout",
|
||||
type=float,
|
||||
default=8.0,
|
||||
help="per-request timeout in s (default 8)",
|
||||
)
|
||||
url.add_argument(
|
||||
"--budget",
|
||||
type=float,
|
||||
default=45.0,
|
||||
help="overall network budget in s (default 45)",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def make_site(args: argparse.Namespace) -> Site | None:
|
||||
"""Build the backend, or report a usage error and return None."""
|
||||
if args.mode == "dist":
|
||||
root = Path(args.target)
|
||||
if not root.is_dir():
|
||||
print(f"agent_ready: not a directory: {root}", file=sys.stderr)
|
||||
return None
|
||||
return DirSite(root)
|
||||
parts = urlsplit(args.target)
|
||||
if parts.scheme not in ("http", "https") or not parts.netloc:
|
||||
print(f"agent_ready: not an http(s) URL: {args.target}", file=sys.stderr)
|
||||
return None
|
||||
return HttpSite(args.target, timeout=args.timeout, budget=args.budget)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""Run every check and return the exit code."""
|
||||
args = parse_args(argv)
|
||||
site = make_site(args)
|
||||
if site is None:
|
||||
return EXIT_USAGE
|
||||
report = Report(out=sys.stdout)
|
||||
ctx = Ctx(
|
||||
site=site,
|
||||
report=report,
|
||||
min_text=args.min_text,
|
||||
max_pages=args.max_pages,
|
||||
act=args.act,
|
||||
)
|
||||
try:
|
||||
site.get(HOME)
|
||||
except Undeterminable as exc:
|
||||
report.skip(
|
||||
"AR-FIND-01", f"undeterminable (network): homepage unreachable: {exc}"
|
||||
)
|
||||
return EXIT_NETWORK
|
||||
for ar_id, check in CHECKS:
|
||||
report.guard(ar_id, lambda check=check: check(ctx))
|
||||
return report.exit_code()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@ -1,10 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Thin wrapper: run the AgentReady checker from any cwd.
|
||||
# check.sh dist <build-dir> [--act]
|
||||
# check.sh url <base-url> [--act]
|
||||
# Exit codes: 0 all pass, 1 any FAIL, 2 usage error, 3 network undeterminable.
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
readonly HERE
|
||||
PYTHONPATH="$(dirname "$HERE")${PYTHONPATH:+:$PYTHONPATH}" exec python3 -m agent_ready.check "$@"
|
||||
@ -1,156 +0,0 @@
|
||||
"""Act stage: AR-ACT-01..10. Opt-in (``--act``): only for sites with user actions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from agent_ready import formats
|
||||
from agent_ready.backends import HttpSite
|
||||
from agent_ready.context import CATALOG, Ctx
|
||||
|
||||
_INIT = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "initialize",
|
||||
"params": {
|
||||
"protocolVersion": "2025-06-18",
|
||||
"capabilities": {},
|
||||
"clientInfo": {"name": "agentready-check", "version": "1"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _json(ctx: Ctx, path: str) -> object | None:
|
||||
"""Parsed JSON at ``path`` when it is 200 and parses, else None."""
|
||||
resp = ctx.site.get(path)
|
||||
if resp.status != 200:
|
||||
return None
|
||||
try:
|
||||
return json.loads(resp.body)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def check_auth(ctx: Ctx) -> None:
|
||||
"""AR-ACT-01: OAuth metadata (RFC 9728 ``resource`` or RFC 8414 ``issuer``)."""
|
||||
prm = _json(ctx, "/.well-known/oauth-protected-resource")
|
||||
asm = _json(ctx, "/.well-known/oauth-authorization-server")
|
||||
ok = (isinstance(prm, dict) and "resource" in prm) or (
|
||||
isinstance(asm, dict) and "issuer" in asm
|
||||
)
|
||||
ctx.report.verdict(
|
||||
"AR-ACT-01",
|
||||
[]
|
||||
if ok
|
||||
else [
|
||||
"no OAuth metadata under /.well-known (oauth-protected-resource / oauth-authorization-server)"
|
||||
],
|
||||
"OAuth metadata discoverable under /.well-known",
|
||||
)
|
||||
|
||||
|
||||
def check_openapi(ctx: Ctx) -> None:
|
||||
"""AR-ACT-02: /openapi.json with paths and an operationId on every operation."""
|
||||
doc = _json(ctx, "/openapi.json")
|
||||
if not isinstance(doc, dict) or not isinstance(doc.get("openapi"), str):
|
||||
ctx.report.fail("AR-ACT-02", "/openapi.json missing or not an OpenAPI document")
|
||||
return
|
||||
paths = doc.get("paths") if isinstance(doc.get("paths"), dict) else {}
|
||||
ops = [
|
||||
op
|
||||
for item in paths.values()
|
||||
if isinstance(item, dict)
|
||||
for m, op in item.items()
|
||||
if m in ("get", "post", "put", "patch", "delete")
|
||||
]
|
||||
missing = sum(
|
||||
1 for op in ops if not (isinstance(op, dict) and op.get("operationId"))
|
||||
)
|
||||
problems = ["no operations in paths"] if not ops else []
|
||||
if missing:
|
||||
problems.append(f"{missing} operations lack operationId")
|
||||
ctx.report.verdict(
|
||||
"AR-ACT-02", problems, f"/openapi.json describes {len(ops)} operations"
|
||||
)
|
||||
|
||||
|
||||
def _mcp_ok(ctx: Ctx) -> bool | None:
|
||||
"""True if /mcp answers initialize (or is OAuth-gated); None in dist mode."""
|
||||
if not isinstance(ctx.site, HttpSite):
|
||||
return None
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json, text/event-stream",
|
||||
}
|
||||
resp = ctx.site.request("/mcp", data=json.dumps(_INIT).encode(), headers=headers)
|
||||
if resp.status == 401:
|
||||
return "www-authenticate" in resp.headers
|
||||
return resp.status == 200 and '"jsonrpc"' in resp.text and '"result"' in resp.text
|
||||
|
||||
|
||||
def check_mcp(ctx: Ctx) -> None:
|
||||
"""AR-ACT-03 MCP at /mcp, AR-ACT-04 server card, AR-ACT-10 an interface exists."""
|
||||
ok = _mcp_ok(ctx)
|
||||
if ok is None:
|
||||
ctx.report.skip("AR-ACT-03", "/mcp is a server endpoint: check in url mode")
|
||||
ctx.report.skip("AR-ACT-10", "MCP/CLI interface: check in url mode")
|
||||
else:
|
||||
ctx.report.verdict(
|
||||
"AR-ACT-03",
|
||||
[]
|
||||
if ok
|
||||
else [
|
||||
"/mcp does not answer an MCP initialize (nor 401 + WWW-Authenticate)"
|
||||
],
|
||||
"/mcp answers MCP initialize",
|
||||
)
|
||||
ctx.report.verdict(
|
||||
"AR-ACT-10",
|
||||
[]
|
||||
if ok
|
||||
else ["no MCP server at /mcp; a CLI (AR-ACT-06) cannot be auto-detected"],
|
||||
"agents can act through /mcp",
|
||||
)
|
||||
cards = [
|
||||
t
|
||||
for t in formats.catalog_types(_json(ctx, CATALOG))
|
||||
if t == "application/mcp-server-card+json"
|
||||
]
|
||||
ctx.report.verdict(
|
||||
"AR-ACT-04",
|
||||
[] if cards else ["AI Catalog lists no application/mcp-server-card+json entry"],
|
||||
"AI Catalog lists an MCP server card",
|
||||
)
|
||||
|
||||
|
||||
def check_webmcp(ctx: Ctx) -> None:
|
||||
"""AR-ACT-07: WebMCP tools registered on document.modelContext."""
|
||||
resp, page = ctx.home()
|
||||
sources = [resp.text]
|
||||
for src in page.scripts[:10]:
|
||||
path = ctx.site.to_path(src)
|
||||
if path:
|
||||
sources.append(ctx.site.get(path).text)
|
||||
found = any("modelContext" in s for s in sources)
|
||||
ctx.report.verdict(
|
||||
"AR-ACT-07",
|
||||
[] if found else ["no document.modelContext usage in homepage HTML/scripts"],
|
||||
"homepage registers WebMCP tools",
|
||||
)
|
||||
|
||||
|
||||
def run_act(ctx: Ctx) -> None:
|
||||
"""All act checks, or one SKIP line when the stage is off."""
|
||||
if not ctx.act:
|
||||
ctx.report.skip(
|
||||
"AR-ACT-10", "act stage off (pass --act for sites with user actions)"
|
||||
)
|
||||
return
|
||||
ctx.report.guard("AR-ACT-01", lambda: check_auth(ctx))
|
||||
ctx.report.guard("AR-ACT-02", lambda: check_openapi(ctx))
|
||||
ctx.report.guard("AR-ACT-03", lambda: check_mcp(ctx))
|
||||
ctx.report.guard("AR-ACT-07", lambda: check_webmcp(ctx))
|
||||
ctx.report.skip("AR-ACT-05", "MCP Apps ui:// view: manual review")
|
||||
ctx.report.skip("AR-ACT-06", "SDKs on npm/PyPI and a CLI: manual review")
|
||||
ctx.report.skip("AR-ACT-08", "agent payments: n/a unless the product charges")
|
||||
ctx.report.skip("AR-ACT-09", "NLWeb /ask endpoint: manual review")
|
||||
@ -1,186 +0,0 @@
|
||||
"""Find stage: AR-FIND-01..04 (robots.txt, agent access, sitemap, AI Catalog)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from agent_ready import formats
|
||||
from agent_ready.context import CATALOG, HOME, LLMS, Ctx, link_header_href
|
||||
from agent_ready.report import summarize
|
||||
|
||||
# Training, search-index and user-initiated fetchers named in the spec's
|
||||
# crawl-policy notes. Real UA strings for the two training crawlers.
|
||||
AGENT_UAS = {
|
||||
"ClaudeBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
|
||||
"GPTBot": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.1; +https://openai.com/gptbot)",
|
||||
"Claude-SearchBot": "Claude-SearchBot/1.0",
|
||||
"OAI-SearchBot": "OAI-SearchBot/1.0; +https://openai.com/searchbot",
|
||||
"Claude-User": "Claude-User/1.0",
|
||||
"ChatGPT-User": "ChatGPT-User/1.0; +https://openai.com/bot",
|
||||
}
|
||||
_CHALLENGE = (
|
||||
"just a moment",
|
||||
"cf-chl",
|
||||
"challenge-platform",
|
||||
"attention required",
|
||||
"captcha",
|
||||
)
|
||||
XML_TYPES = ("application/xml", "text/xml", "application/rss+xml")
|
||||
|
||||
|
||||
def check_robots(ctx: Ctx) -> None:
|
||||
"""AR-FIND-01: robots.txt exists, is plain text and blocks no agent."""
|
||||
resp = ctx.site.get("/robots.txt")
|
||||
if resp.status != 200:
|
||||
ctx.report.fail("AR-FIND-01", f"/robots.txt: HTTP {resp.status}")
|
||||
return
|
||||
problems = ctx.type_problem(resp, ("text/plain",), "/robots.txt")
|
||||
if formats.looks_like_html(resp.text):
|
||||
ctx.report.fail(
|
||||
"AR-FIND-01",
|
||||
"; ".join([*problems, "/robots.txt body is an HTML page (SPA fallback?)"]),
|
||||
)
|
||||
return
|
||||
ctx.robots_text = resp.text
|
||||
groups = formats.parse_robots(resp.text)
|
||||
if not formats.has_wildcard_group(groups):
|
||||
problems.append("no 'User-agent: *' group (spec: one rule for all agents)")
|
||||
blocked = formats.blocked_agents(groups)
|
||||
if blocked:
|
||||
problems.append(f"blanket Disallow for: {', '.join(blocked)}")
|
||||
ctx.report.verdict(
|
||||
"AR-FIND-01",
|
||||
problems,
|
||||
"robots.txt is plain text, has a * group and blocks no agent",
|
||||
)
|
||||
|
||||
|
||||
def check_agent_access(ctx: Ctx) -> None:
|
||||
"""AR-FIND-01 (bot protection): homepage is 200 for each agent UA (url mode)."""
|
||||
if not ctx.live:
|
||||
ctx.report.skip("AR-FIND-01", "agent user-agent fetches need url mode")
|
||||
return
|
||||
base, _ = ctx.home()
|
||||
ctx.site.prefetch([HOME], tuple(AGENT_UAS.values()))
|
||||
bad = []
|
||||
for name, ua in AGENT_UAS.items():
|
||||
resp = ctx.site.get(HOME, ua)
|
||||
lowered = resp.text[:20000].lower()
|
||||
if resp.status != 200:
|
||||
bad.append(f"{name} HTTP {resp.status}")
|
||||
elif resp.body != base.body and any(m in lowered for m in _CHALLENGE):
|
||||
bad.append(f"{name} got a challenge page")
|
||||
ctx.report.verdict("AR-FIND-01", bad, f"homepage 200 for {', '.join(AGENT_UAS)}")
|
||||
|
||||
|
||||
def _sitemap_paths(ctx: Ctx) -> list[str]:
|
||||
"""Sitemap locations from robots.txt (also fixes the dist-mode origin)."""
|
||||
urls = formats.robots_sitemaps(ctx.robots_text or "")
|
||||
if urls and not ctx.live:
|
||||
parts = urlsplit(urls[0])
|
||||
ctx.site.origin = f"{parts.scheme}://{parts.netloc}"
|
||||
paths = [p for u in urls if (p := ctx.site.to_path(u))]
|
||||
if ctx.robots_text is None:
|
||||
ctx.report.fail("AR-FIND-03", "no valid robots.txt to reference the sitemap")
|
||||
elif not urls:
|
||||
ctx.report.fail("AR-FIND-03", "robots.txt has no 'Sitemap:' line")
|
||||
elif not paths:
|
||||
ctx.report.fail(
|
||||
"AR-FIND-03", f"robots.txt Sitemap is on another host: {urls[0]}"
|
||||
)
|
||||
else:
|
||||
ctx.report.ok("AR-FIND-03", f"robots.txt references {summarize(urls)}")
|
||||
return paths or ["/sitemap.xml"]
|
||||
|
||||
|
||||
def _read_sitemap(ctx: Ctx, path: str, problems: list[str]) -> tuple[str, list[str]]:
|
||||
"""Fetch and parse one sitemap; record problems; return (kind, locs)."""
|
||||
resp = ctx.site.get(path)
|
||||
if resp.status != 200:
|
||||
problems.append(f"{path}: HTTP {resp.status}")
|
||||
return "", []
|
||||
problems.extend(ctx.type_problem(resp, XML_TYPES, path))
|
||||
try:
|
||||
return formats.parse_sitemap(resp.body)
|
||||
except ValueError as exc:
|
||||
problems.append(f"{path}: {exc}")
|
||||
return "", []
|
||||
|
||||
|
||||
def check_sitemap(ctx: Ctx) -> None:
|
||||
"""AR-FIND-03: sitemap referenced from robots.txt, valid XML, on-site <loc>s."""
|
||||
problems: list[str] = []
|
||||
locs: list[str] = []
|
||||
for path in _sitemap_paths(ctx)[:5]:
|
||||
kind, found = _read_sitemap(ctx, path, problems)
|
||||
if kind == "sitemapindex":
|
||||
for child in [p for u in found[:10] if (p := ctx.site.to_path(u))]:
|
||||
locs += _read_sitemap(ctx, child, problems)[1]
|
||||
else:
|
||||
locs += found
|
||||
if locs and not ctx.live and ctx.site.origin.endswith("dist.invalid"):
|
||||
parts = urlsplit(locs[0])
|
||||
ctx.site.origin = f"{parts.scheme}://{parts.netloc}"
|
||||
pages = [p for loc in locs if (p := ctx.site.to_path(loc))]
|
||||
if len(pages) < len(locs):
|
||||
problems.append(
|
||||
f"{len(locs) - len(pages)} <loc> URLs are not on {ctx.site.origin}"
|
||||
)
|
||||
if not locs and not problems:
|
||||
problems.append("sitemap has no <loc> entries")
|
||||
ctx.sitemap_pages = list(dict.fromkeys(pages))
|
||||
ctx.report.verdict(
|
||||
"AR-FIND-03", problems, f"sitemap lists {len(ctx.sitemap_pages)} pages"
|
||||
)
|
||||
|
||||
|
||||
def check_catalog(ctx: Ctx) -> None:
|
||||
"""AR-FIND-04: valid AI Catalog, lists llms.txt, linked from homepage head."""
|
||||
resp = ctx.site.get(CATALOG)
|
||||
problems: list[str] = []
|
||||
if resp.status != 200:
|
||||
problems.append(f"{CATALOG}: HTTP {resp.status}")
|
||||
else:
|
||||
problems += ctx.type_problem(
|
||||
resp, ("application/ai-catalog+json", "application/json"), CATALOG
|
||||
)
|
||||
try:
|
||||
doc = json.loads(resp.body)
|
||||
except ValueError:
|
||||
doc = None
|
||||
problems.append(f"{CATALOG} is not JSON")
|
||||
if doc is not None:
|
||||
problems += formats.catalog_problems(doc)
|
||||
listed = {ctx.site.to_path(u) for u in formats.catalog_urls(doc)}
|
||||
if ctx.exists(LLMS) and LLMS not in listed:
|
||||
problems.append("catalog has no entry for /llms.txt")
|
||||
ctx.report.verdict("AR-FIND-04", problems, f"{CATALOG} is a valid AI Catalog")
|
||||
home, scan = ctx.home()
|
||||
href = scan.link_href("ai-catalog")
|
||||
ok = href is not None and ctx.site.to_path(href) == CATALOG
|
||||
ctx.report.verdict(
|
||||
"AR-FIND-04",
|
||||
[]
|
||||
if ok
|
||||
else [
|
||||
'homepage head lacks <link rel="ai-catalog" href="/.well-known/ai-catalog.json">'
|
||||
],
|
||||
"homepage head links the catalog",
|
||||
)
|
||||
if not ctx.live:
|
||||
ctx.report.skip(
|
||||
"AR-FIND-04", "HTTP Link header can only be checked in url mode"
|
||||
)
|
||||
return
|
||||
header = link_header_href(home, "ai-catalog")
|
||||
ok = header is not None and ctx.site.to_path(header) == CATALOG
|
||||
ctx.report.verdict(
|
||||
"AR-FIND-04",
|
||||
[]
|
||||
if ok
|
||||
else [
|
||||
'homepage response lacks HTTP header Link: <...ai-catalog.json>; rel="ai-catalog"'
|
||||
],
|
||||
"homepage sends the ai-catalog Link header",
|
||||
)
|
||||
@ -1,212 +0,0 @@
|
||||
"""Read stage: AR-READ-01..09 and 11 (AR-READ-10 is retired in spec v2.0)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
|
||||
from agent_ready import formats
|
||||
from agent_ready.context import CATALOG, HOME, LLMS, TEXT_TYPES, Ctx, link_header_href
|
||||
from agent_ready.htmlscan import scan
|
||||
from agent_ready.report import summarize
|
||||
|
||||
|
||||
def _pages(ctx: Ctx) -> list[str]:
|
||||
"""Homepage plus sitemap pages, capped. An empty sitemap is a READ-01 FAIL."""
|
||||
if not ctx.sitemap_pages:
|
||||
ctx.report.fail(
|
||||
"AR-READ-01", "sitemap yielded no pages; only the homepage was checked"
|
||||
)
|
||||
every = list(dict.fromkeys([HOME, *ctx.sitemap_pages]))
|
||||
pages = ctx.cap(every)
|
||||
if len(pages) < len(every):
|
||||
ctx.report.skip(
|
||||
"AR-READ-01",
|
||||
f"checked {len(pages)} of {len(every)} pages (--max-pages); rest unchecked",
|
||||
)
|
||||
ctx.site.prefetch(pages)
|
||||
return pages
|
||||
|
||||
|
||||
def _mirror_href(ctx: Ctx, path: str) -> str | None:
|
||||
"""Markdown mirror advertised by <link rel=alternate type=text/markdown> or Link header."""
|
||||
resp, page = ctx.page(path)
|
||||
href = page.link_href("alternate", "text/markdown")
|
||||
if href is None and ctx.live:
|
||||
href = link_header_href(resp, "alternate", "text/markdown")
|
||||
return href
|
||||
|
||||
|
||||
def check_pages(ctx: Ctx) -> None:
|
||||
"""AR-READ-01 text in initial HTML, AR-READ-05 tagged code, AR-READ-09 mirrors."""
|
||||
pages = _pages(ctx)
|
||||
home, _ = ctx.home()
|
||||
no_text: list[str] = []
|
||||
untagged: list[str] = []
|
||||
blocks = 0
|
||||
for path in pages:
|
||||
resp, page = ctx.page(path)
|
||||
if resp.status != 200:
|
||||
no_text.append(f"{path} (HTTP {resp.status})")
|
||||
continue
|
||||
if ctx.type_problem(resp, ("text/html",), path):
|
||||
no_text.append(f"{path} ({resp.ctype or 'no content-type'})")
|
||||
elif path != HOME and resp.body == home.body:
|
||||
no_text.append(f"{path} (byte-identical to the homepage shell)")
|
||||
elif len(page.text) < ctx.min_text:
|
||||
no_text.append(f"{path} ({len(page.text)} chars of text < {ctx.min_text})")
|
||||
blocks += page.code_blocks
|
||||
if page.untagged_code:
|
||||
untagged.append(f"{path} ({page.untagged_code} untagged <pre><code>)")
|
||||
ctx.report.verdict(
|
||||
"AR-READ-01",
|
||||
["answer not in initial HTML: " + summarize(no_text)] if no_text else [],
|
||||
f"{len(pages)} pages carry >= {ctx.min_text} chars of text without JS",
|
||||
)
|
||||
md_untagged = _check_mirrors(ctx, pages)
|
||||
untagged += md_untagged
|
||||
ctx.report.verdict(
|
||||
"AR-READ-05",
|
||||
["code without a language tag: " + summarize(untagged)] if untagged else [],
|
||||
f"{blocks} HTML code blocks (plus mirror fences) all language-tagged",
|
||||
)
|
||||
|
||||
|
||||
def _check_mirrors(ctx: Ctx, pages: list[str]) -> list[str]:
|
||||
"""AR-READ-09 for every page; returns mirrors with untagged fences (READ-05)."""
|
||||
bad: list[str] = []
|
||||
untagged: list[str] = []
|
||||
targets: dict[str, str] = {}
|
||||
for path in pages:
|
||||
if ctx.site.get(path).status != 200:
|
||||
continue
|
||||
href = _mirror_href(ctx, path)
|
||||
target = ctx.site.to_path(href, path) if href else None
|
||||
if target is None:
|
||||
bad.append(f"{path} (no text/markdown alternate link)")
|
||||
else:
|
||||
targets[path] = target
|
||||
ctx.site.prefetch(list(targets.values()))
|
||||
for path, target in targets.items():
|
||||
resp = ctx.site.get(target)
|
||||
if resp.status != 200:
|
||||
bad.append(f"{target} (HTTP {resp.status})")
|
||||
elif (
|
||||
ctx.type_problem(resp, TEXT_TYPES, target)
|
||||
or formats.looks_like_html(resp.text)
|
||||
or not resp.text.strip()
|
||||
):
|
||||
bad.append(f"{target} (not markdown: {resp.ctype or 'empty'})")
|
||||
elif n := formats.untagged_fences(resp.text):
|
||||
untagged.append(f"{target} ({n} untagged fences)")
|
||||
ctx.report.verdict(
|
||||
"AR-READ-09",
|
||||
["markdown mirror missing/broken: " + summarize(bad)] if bad else [],
|
||||
f"{len(targets)} pages advertise a working markdown mirror",
|
||||
)
|
||||
return untagged
|
||||
|
||||
|
||||
def check_status_codes(ctx: Ctx) -> None:
|
||||
"""AR-READ-02: unknown paths are a real 404 (url) / a 404.html exists (dist)."""
|
||||
if not ctx.live:
|
||||
resp = ctx.site.get(f"/agentready-probe-{secrets.token_hex(6)}")
|
||||
text = scan(resp.text).text if resp.body else ""
|
||||
ok = resp.status == 404 and len(text) >= 10
|
||||
ctx.report.verdict(
|
||||
"AR-READ-02",
|
||||
[] if ok else ["no 404.html with real text in the build output"],
|
||||
"404.html exists for unknown paths",
|
||||
)
|
||||
return
|
||||
token = secrets.token_hex(6)
|
||||
probes = [f"/agentready-probe-{token}", f"/agentready-probe-{token}/page.html"]
|
||||
ctx.site.prefetch(probes)
|
||||
bad = [
|
||||
f"{p} -> HTTP {s}"
|
||||
for p in probes
|
||||
if (s := ctx.site.get(p).status) not in (404, 410)
|
||||
]
|
||||
ctx.report.verdict(
|
||||
"AR-READ-02",
|
||||
["soft 404, unknown paths must return 404: " + ", ".join(bad)] if bad else [],
|
||||
"unknown paths return 404",
|
||||
)
|
||||
|
||||
|
||||
def check_homepage(ctx: Ctx) -> None:
|
||||
"""AR-READ-03 real links on the homepage, AR-READ-08 JSON-LD on it."""
|
||||
resp, page = ctx.home()
|
||||
links = {
|
||||
p
|
||||
for a in page.anchors
|
||||
if not a.startswith(("#", "javascript:", "mailto:"))
|
||||
and (p := ctx.site.to_path(a))
|
||||
and p != HOME
|
||||
}
|
||||
ok = resp.status == 200 and bool(links)
|
||||
ctx.report.verdict(
|
||||
"AR-READ-03",
|
||||
[] if ok else ["homepage HTML has no <a href> links to other pages"],
|
||||
f"homepage links {len(links)} pages in raw HTML",
|
||||
)
|
||||
problems = []
|
||||
if not page.jsonld_ok:
|
||||
problems.append(
|
||||
'homepage has no <script type="application/ld+json"> with @context'
|
||||
)
|
||||
if page.jsonld_bad:
|
||||
problems.append(
|
||||
f"{page.jsonld_bad} JSON-LD blocks fail to parse or lack @context"
|
||||
)
|
||||
ctx.report.verdict(
|
||||
"AR-READ-08", problems, f"homepage carries {page.jsonld_ok} JSON-LD blocks"
|
||||
)
|
||||
|
||||
|
||||
def check_discovery_links(ctx: Ctx) -> None:
|
||||
"""AR-READ-06: discovery files are linked from the homepage."""
|
||||
_, page = ctx.home()
|
||||
linked = {
|
||||
ctx.site.to_path(h)
|
||||
for h in [*page.anchors, *(href for _, _, href in page.links)]
|
||||
}
|
||||
files = [LLMS, CATALOG] + (["/openapi.json"] if ctx.act else [])
|
||||
for path in files:
|
||||
problems = []
|
||||
if not ctx.exists(path):
|
||||
problems.append(f"{path} does not exist")
|
||||
if path not in linked:
|
||||
problems.append(f"{path} is not linked from the homepage (<link> or <a>)")
|
||||
ctx.report.verdict(
|
||||
"AR-READ-06", problems, f"{path} exists and is linked from the homepage"
|
||||
)
|
||||
|
||||
|
||||
def check_llms(ctx: Ctx) -> None:
|
||||
"""AR-READ-07: llms.txt in the spec shape, every on-site link resolving."""
|
||||
resp = ctx.site.get(LLMS)
|
||||
if resp.status != 200:
|
||||
ctx.report.fail("AR-READ-07", f"{LLMS}: HTTP {resp.status}")
|
||||
return
|
||||
problems = ctx.type_problem(resp, TEXT_TYPES, LLMS) + formats.llms_problems(
|
||||
resp.text
|
||||
)
|
||||
if not formats.looks_like_html(resp.text):
|
||||
paths = [
|
||||
p for u in formats.llms_links(resp.text) if (p := ctx.site.to_path(u, LLMS))
|
||||
]
|
||||
paths = ctx.cap(paths)
|
||||
ctx.site.prefetch(paths)
|
||||
broken = [p for p in paths if ctx.site.get(p).status != 200]
|
||||
if broken:
|
||||
problems.append("broken links: " + summarize(broken))
|
||||
ctx.report.verdict("AR-READ-07", problems, f"{LLMS} has H1, summary and link lists")
|
||||
|
||||
|
||||
def skip_manual(ctx: Ctx) -> None:
|
||||
"""Items with no mechanical check (reported, never silently dropped)."""
|
||||
ctx.report.skip(
|
||||
"AR-FIND-02", "first-party docs page answering user questions: manual review"
|
||||
)
|
||||
ctx.report.skip("AR-READ-04", "docs pages answer users' tasks: manual review")
|
||||
ctx.report.skip("AR-READ-11", "pricing page: n/a for sites without paid plans")
|
||||
@ -1,85 +0,0 @@
|
||||
"""Shared state for one checker run (site, report, options, discoveries)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from agent_ready.backends import Response, Site
|
||||
from agent_ready.formats import looks_like_html
|
||||
from agent_ready.htmlscan import PageScan, scan
|
||||
from agent_ready.report import Report
|
||||
|
||||
HOME = "/"
|
||||
LLMS = "/llms.txt"
|
||||
CATALOG = "/.well-known/ai-catalog.json"
|
||||
TEXT_TYPES = ("text/plain", "text/markdown")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Ctx:
|
||||
"""Everything a check needs; filled in as checks discover the site."""
|
||||
|
||||
site: Site
|
||||
report: Report
|
||||
min_text: int = 100
|
||||
max_pages: int = 40
|
||||
act: bool = False
|
||||
robots_text: str | None = None
|
||||
sitemap_pages: list[str] = field(default_factory=list)
|
||||
_scans: dict[str, PageScan] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def live(self) -> bool:
|
||||
"""True in url mode (headers and content types are real)."""
|
||||
return self.site.live
|
||||
|
||||
def cap(self, items: list[str]) -> list[str]:
|
||||
"""Apply --max-pages in url mode only; dist mode is free, so check everything."""
|
||||
return items[: self.max_pages] if self.live else items
|
||||
|
||||
def page(self, path: str) -> tuple[Response, PageScan]:
|
||||
"""Fetch and scan an HTML page (scan memoized)."""
|
||||
resp = self.site.get(path)
|
||||
if path not in self._scans:
|
||||
self._scans[path] = scan(resp.text) if resp.status == 200 else PageScan()
|
||||
return resp, self._scans[path]
|
||||
|
||||
def home(self) -> tuple[Response, PageScan]:
|
||||
"""The homepage and its scan."""
|
||||
return self.page(HOME)
|
||||
|
||||
def type_problem(
|
||||
self, resp: Response, allowed: tuple[str, ...], what: str
|
||||
) -> list[str]:
|
||||
"""Content-type problem for live responses (dist mode has none)."""
|
||||
if (
|
||||
not self.live
|
||||
or resp.ctype in allowed
|
||||
or any(a.endswith("+xml") and resp.ctype.endswith("+xml") for a in allowed)
|
||||
):
|
||||
return []
|
||||
return [
|
||||
f"{what} served as {resp.ctype or 'no content-type'}, want {' or '.join(allowed)}"
|
||||
]
|
||||
|
||||
def exists(self, path: str) -> bool:
|
||||
"""True when ``path`` is 200 and, for a non-page file, not an HTML fallback."""
|
||||
resp = self.site.get(path)
|
||||
is_page = path.endswith(("/", ".html"))
|
||||
return resp.status == 200 and (is_page or not looks_like_html(resp.text))
|
||||
|
||||
|
||||
def link_header_href(resp: Response, rel: str, mime: str | None = None) -> str | None:
|
||||
"""href of the first RFC 8288 ``Link`` header value with ``rel`` (and type)."""
|
||||
for part in re.split(r",\s*(?=<)", resp.headers.get("link", "")):
|
||||
match = re.match(r"\s*<([^>]*)>(.*)", part)
|
||||
if not match:
|
||||
continue
|
||||
params = match.group(2).lower()
|
||||
rels = re.search(r'rel="?([^";]+)"?', params)
|
||||
if not rels or rel not in rels.group(1).split():
|
||||
continue
|
||||
if mime is None or re.search(rf'type="?{re.escape(mime)}"?', params):
|
||||
return match.group(1)
|
||||
return None
|
||||
@ -1,213 +0,0 @@
|
||||
"""Validators for the file formats AgentReady points at.
|
||||
|
||||
robots.txt (RFC 9309), sitemap.xml (sitemaps.org), llms.txt (agentready
|
||||
spec "llms.txt shape" / llmstxt.org) and the AI Catalog
|
||||
(github.com/Agent-Card/ai-catalog, specification/ai-catalog.md).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
_LINK_ITEM = re.compile(r"^\s*[-*]\s+\[[^\]]+\]\(([^)\s]+)\)")
|
||||
_FENCE = re.compile(r"^\s{0,3}(```+|~~~+)\s*(\S*)")
|
||||
_SPEC_VERSION = re.compile(r"^\d+\.\d+$")
|
||||
|
||||
|
||||
def looks_like_html(text: str) -> bool:
|
||||
"""True when a supposedly plain-text/XML/JSON body is an HTML page."""
|
||||
head = text.lstrip()[:200].lower()
|
||||
return head.startswith(("<!doctype html", "<html")) or "<head" in head
|
||||
|
||||
|
||||
# --------------------------------------------------------------- robots.txt
|
||||
@dataclass
|
||||
class RobotsGroup:
|
||||
"""One ``User-agent`` group and its rules, in file order."""
|
||||
|
||||
agents: list[str] = field(default_factory=list)
|
||||
rules: list[tuple[str, str]] = field(default_factory=list)
|
||||
|
||||
|
||||
def parse_robots(text: str) -> list[RobotsGroup]:
|
||||
"""Parse robots.txt into groups (consecutive User-agent lines share one)."""
|
||||
groups: list[RobotsGroup] = []
|
||||
current: RobotsGroup | None = None
|
||||
for raw in text.splitlines():
|
||||
line = raw.split("#", 1)[0].strip()
|
||||
if ":" not in line:
|
||||
continue
|
||||
key, value = (part.strip() for part in line.split(":", 1))
|
||||
key = key.lower()
|
||||
if key == "user-agent":
|
||||
if current is None or current.rules:
|
||||
current = RobotsGroup()
|
||||
groups.append(current)
|
||||
current.agents.append(value.lower())
|
||||
elif key in ("allow", "disallow") and current is not None:
|
||||
current.rules.append((key, value))
|
||||
return groups
|
||||
|
||||
|
||||
def has_wildcard_group(groups: list[RobotsGroup]) -> bool:
|
||||
"""True when a ``User-agent: *`` group exists (one rule for all agents)."""
|
||||
return any("*" in g.agents for g in groups)
|
||||
|
||||
|
||||
def blocked_agents(groups: list[RobotsGroup]) -> list[str]:
|
||||
"""Agents whose group disallows the whole site (``/`` or ``/*``).
|
||||
|
||||
An empty ``Disallow:`` allows everything; an ``Allow: /`` in the same
|
||||
group wins over the blanket disallow (longest-match tie goes to allow).
|
||||
"""
|
||||
blocked: list[str] = []
|
||||
for group in groups:
|
||||
disallow_all = any(k == "disallow" and v in ("/", "/*") for k, v in group.rules)
|
||||
allow_all = any(k == "allow" and v in ("/", "/*") for k, v in group.rules)
|
||||
if disallow_all and not allow_all:
|
||||
blocked.extend(group.agents)
|
||||
return blocked
|
||||
|
||||
|
||||
def robots_sitemaps(text: str) -> list[str]:
|
||||
"""Return every ``Sitemap:`` URL."""
|
||||
out = []
|
||||
for raw in text.splitlines():
|
||||
key, _, value = raw.partition(":")
|
||||
if key.strip().lower() == "sitemap" and value.strip():
|
||||
out.append(value.strip())
|
||||
return out
|
||||
|
||||
|
||||
# -------------------------------------------------------------- sitemap.xml
|
||||
def parse_sitemap(body: bytes) -> tuple[str, list[str]]:
|
||||
"""Return (root kind ``urlset``/``sitemapindex``, list of <loc> values).
|
||||
|
||||
Raises ValueError for anything that is not a sitemap document.
|
||||
"""
|
||||
try:
|
||||
root = ET.fromstring(body)
|
||||
except ET.ParseError as exc:
|
||||
raise ValueError(f"not XML: {exc}") from exc
|
||||
kind = root.tag.rsplit("}", 1)[-1]
|
||||
if kind not in ("urlset", "sitemapindex"):
|
||||
raise ValueError(f"root element <{kind}> is not <urlset>/<sitemapindex>")
|
||||
locs = [
|
||||
(el.text or "").strip()
|
||||
for el in root.iter()
|
||||
if el.tag.rsplit("}", 1)[-1] == "loc"
|
||||
]
|
||||
return kind, [loc for loc in locs if loc]
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- llms.txt
|
||||
def llms_problems(text: str) -> list[str]:
|
||||
"""Check llms.txt shape: H1 first, blockquote summary, H2 link lists."""
|
||||
if looks_like_html(text):
|
||||
return ["body is an HTML page, not llms.txt markdown"]
|
||||
lines = [ln for ln in text.splitlines() if ln.strip()]
|
||||
problems = []
|
||||
if not lines or not re.match(r"^# \S", lines[0]):
|
||||
problems.append("first line must be an H1 title ('# Name')")
|
||||
if not any(ln.startswith(">") for ln in lines[:4]):
|
||||
problems.append("missing blockquote summary ('> ...') after the H1")
|
||||
in_h2 = False
|
||||
links = 0
|
||||
for ln in lines:
|
||||
if ln.startswith("## "):
|
||||
in_h2 = True
|
||||
elif in_h2 and _LINK_ITEM.match(ln):
|
||||
links += 1
|
||||
if links == 0:
|
||||
problems.append("no '## Section' with a '- [name](url): notes' link list")
|
||||
return problems
|
||||
|
||||
|
||||
def llms_links(text: str) -> list[str]:
|
||||
"""Return every link-list URL in llms.txt, in order."""
|
||||
return [m.group(1) for ln in text.splitlines() if (m := _LINK_ITEM.match(ln))]
|
||||
|
||||
|
||||
# --------------------------------------------------------------- ai-catalog
|
||||
def catalog_problems(doc: object) -> list[str]:
|
||||
"""Validate an AI Catalog (Level 1 + Host Info if present)."""
|
||||
if not isinstance(doc, dict):
|
||||
return ["top level must be a JSON object"]
|
||||
problems = []
|
||||
version = doc.get("specVersion")
|
||||
if not isinstance(version, str):
|
||||
problems.append("missing string specVersion")
|
||||
elif not _SPEC_VERSION.match(version):
|
||||
problems.append(f"specVersion {version!r} is not 'Major.Minor'")
|
||||
host = doc.get("host")
|
||||
if host is not None and not (
|
||||
isinstance(host, dict) and isinstance(host.get("displayName"), str)
|
||||
):
|
||||
problems.append("host.displayName (string) is required when host is present")
|
||||
entries = doc.get("entries")
|
||||
if not isinstance(entries, list):
|
||||
return [*problems, "missing entries array"]
|
||||
for i, entry in enumerate(entries):
|
||||
problems.extend(f"entries[{i}]: {p}" for p in _entry_problems(entry))
|
||||
return problems
|
||||
|
||||
|
||||
def _entry_problems(entry: object) -> list[str]:
|
||||
"""Problems with one Catalog Entry."""
|
||||
if not isinstance(entry, dict):
|
||||
return ["must be an object"]
|
||||
problems = [
|
||||
f"missing string {k}"
|
||||
for k in ("identifier", "type")
|
||||
if not isinstance(entry.get(k), str)
|
||||
]
|
||||
if ("url" in entry) == ("data" in entry):
|
||||
problems.append("must carry exactly one of url/data")
|
||||
return problems
|
||||
|
||||
|
||||
def catalog_urls(doc: object) -> list[str]:
|
||||
"""Every entry ``url`` in a catalog (empty for invalid docs)."""
|
||||
if not isinstance(doc, dict) or not isinstance(doc.get("entries"), list):
|
||||
return []
|
||||
return [
|
||||
e["url"]
|
||||
for e in doc["entries"]
|
||||
if isinstance(e, dict) and isinstance(e.get("url"), str)
|
||||
]
|
||||
|
||||
|
||||
def catalog_types(doc: object) -> list[str]:
|
||||
"""Every entry ``type`` in a catalog."""
|
||||
if not isinstance(doc, dict) or not isinstance(doc.get("entries"), list):
|
||||
return []
|
||||
return [
|
||||
e["type"]
|
||||
for e in doc["entries"]
|
||||
if isinstance(e, dict) and isinstance(e.get("type"), str)
|
||||
]
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- markdown
|
||||
def untagged_fences(text: str) -> int:
|
||||
"""Count opening code fences with no language info string."""
|
||||
count = 0
|
||||
open_fence = ""
|
||||
for line in text.splitlines():
|
||||
match = _FENCE.match(line)
|
||||
if not match:
|
||||
continue
|
||||
marker, info = match.groups()
|
||||
if open_fence:
|
||||
if (
|
||||
marker[0] == open_fence[0]
|
||||
and len(marker) >= len(open_fence)
|
||||
and not info
|
||||
):
|
||||
open_fence = ""
|
||||
continue
|
||||
open_fence = marker
|
||||
count += not info
|
||||
return count
|
||||
@ -1,121 +0,0 @@
|
||||
"""One-pass HTML scan: visible body text, links, JSON-LD and code blocks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from html.parser import HTMLParser
|
||||
|
||||
_HIDDEN = {"script", "style", "noscript", "template", "svg", "head", "title"}
|
||||
_LANG = re.compile(r"\b(language|lang)-\S+")
|
||||
|
||||
|
||||
@dataclass
|
||||
class PageScan:
|
||||
"""What an agent that does not run JavaScript sees in a page."""
|
||||
|
||||
text: str = ""
|
||||
links: list[tuple[set[str], str, str]] = field(default_factory=list)
|
||||
anchors: list[str] = field(default_factory=list)
|
||||
scripts: list[str] = field(default_factory=list)
|
||||
jsonld_ok: int = 0
|
||||
jsonld_bad: int = 0
|
||||
code_blocks: int = 0
|
||||
untagged_code: int = 0
|
||||
|
||||
def link_href(self, rel: str, mime: str | None = None) -> str | None:
|
||||
"""First ``<link>`` href with ``rel`` (and ``type`` if given)."""
|
||||
for rels, typ, href in self.links:
|
||||
if rel in rels and (mime is None or typ == mime):
|
||||
return href
|
||||
return None
|
||||
|
||||
|
||||
def _tagged(attrs: dict[str, str]) -> bool:
|
||||
"""True when an element carries a code-language marker."""
|
||||
return bool(_LANG.search(attrs.get("class", ""))) or bool(
|
||||
attrs.get("data-language") or attrs.get("data-lang")
|
||||
)
|
||||
|
||||
|
||||
class _Scanner(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.out = PageScan()
|
||||
self._hidden: list[str] = []
|
||||
self._text: list[str] = []
|
||||
self._pre: list[list[bool]] = [] # [pre_tagged, code_seen]
|
||||
self._ld: list[str] | None = None
|
||||
|
||||
def handle_starttag(
|
||||
self, tag: str, attr_list: list[tuple[str, str | None]]
|
||||
) -> None:
|
||||
attrs = {k: v or "" for k, v in attr_list}
|
||||
if tag == "body": # </head> is optional in HTML; <body> always ends it
|
||||
self._hidden = [h for h in self._hidden if h not in ("head", "title")]
|
||||
if tag == "link" and attrs.get("href"):
|
||||
rels = set(attrs.get("rel", "").lower().split())
|
||||
self.out.links.append((rels, attrs.get("type", "").lower(), attrs["href"]))
|
||||
elif tag == "a" and attrs.get("href"):
|
||||
self.out.anchors.append(attrs["href"])
|
||||
elif tag == "script":
|
||||
if attrs.get("src"):
|
||||
self.out.scripts.append(attrs["src"])
|
||||
if attrs.get("type", "").lower() == "application/ld+json":
|
||||
self._ld = []
|
||||
elif tag == "pre":
|
||||
self._pre.append([_tagged(attrs), False])
|
||||
elif tag == "code" and self._pre and not self._pre[-1][1]:
|
||||
self._pre[-1][1] = True
|
||||
self.out.code_blocks += 1
|
||||
self.out.untagged_code += not (self._pre[-1][0] or _tagged(attrs))
|
||||
if tag in _HIDDEN:
|
||||
self._hidden.append(tag)
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag == "script" and self._ld is not None:
|
||||
self._finish_ld("".join(self._ld))
|
||||
self._ld = None
|
||||
if tag == "pre" and self._pre:
|
||||
self._pre.pop()
|
||||
if tag in _HIDDEN and tag in self._hidden:
|
||||
while self._hidden and self._hidden.pop() != tag:
|
||||
pass
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._ld is not None:
|
||||
self._ld.append(data)
|
||||
elif not self._hidden:
|
||||
self._text.append(data)
|
||||
|
||||
def _finish_ld(self, raw: str) -> None:
|
||||
try:
|
||||
doc = json.loads(raw)
|
||||
except ValueError:
|
||||
self.out.jsonld_bad += 1
|
||||
return
|
||||
nodes = doc if isinstance(doc, list) else [doc]
|
||||
ok = (
|
||||
all(
|
||||
isinstance(n, dict) and ("@context" in n or "@graph" in n)
|
||||
for n in nodes
|
||||
)
|
||||
and nodes
|
||||
)
|
||||
if ok:
|
||||
self.out.jsonld_ok += 1
|
||||
else:
|
||||
self.out.jsonld_bad += 1
|
||||
|
||||
def result(self) -> PageScan:
|
||||
self.out.text = re.sub(r"\s+", " ", " ".join(self._text)).strip()
|
||||
return self.out
|
||||
|
||||
|
||||
def scan(html: str) -> PageScan:
|
||||
"""Scan an HTML document."""
|
||||
parser = _Scanner()
|
||||
parser.feed(html)
|
||||
parser.close()
|
||||
return parser.result()
|
||||
@ -1,73 +0,0 @@
|
||||
"""Result lines (``PASS|FAIL|SKIP AR-XXX-NN msg``) and the exit-code policy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TextIO
|
||||
|
||||
EXIT_OK, EXIT_FAIL, EXIT_USAGE, EXIT_NETWORK = 0, 1, 2, 3
|
||||
|
||||
|
||||
class Undeterminable(Exception):
|
||||
"""A network-level failure (refused, timeout, deadline): no verdict."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Report:
|
||||
"""Collects check lines and decides the exit code.
|
||||
|
||||
Every in-scope item is required for kuhy's sites, so a FAIL of any
|
||||
strength (MUST/SHOULD/MAY) fails the run. SKIP never fails it.
|
||||
"""
|
||||
|
||||
out: TextIO = field(default_factory=lambda: sys.stdout)
|
||||
fails: int = 0
|
||||
undetermined: int = 0
|
||||
|
||||
def emit(self, status: str, ar_id: str, msg: str) -> None:
|
||||
"""Print one result line."""
|
||||
if status == "FAIL":
|
||||
self.fails += 1
|
||||
print(f"{status} {ar_id} {msg}", file=self.out, flush=True)
|
||||
|
||||
def ok(self, ar_id: str, msg: str) -> None:
|
||||
"""Emit PASS."""
|
||||
self.emit("PASS", ar_id, msg)
|
||||
|
||||
def fail(self, ar_id: str, msg: str) -> None:
|
||||
"""Emit FAIL."""
|
||||
self.emit("FAIL", ar_id, msg)
|
||||
|
||||
def skip(self, ar_id: str, msg: str) -> None:
|
||||
"""Emit SKIP."""
|
||||
self.emit("SKIP", ar_id, msg)
|
||||
|
||||
def verdict(self, ar_id: str, problems: list[str], pass_msg: str) -> bool:
|
||||
"""FAIL with the problems joined, or PASS with ``pass_msg``."""
|
||||
if problems:
|
||||
self.fail(ar_id, "; ".join(problems))
|
||||
return False
|
||||
self.ok(ar_id, pass_msg)
|
||||
return True
|
||||
|
||||
def guard(self, ar_id: str, fn: Callable[[], object]) -> None:
|
||||
"""Run a check; a network failure becomes an undeterminable SKIP."""
|
||||
try:
|
||||
fn()
|
||||
except Undeterminable as exc:
|
||||
self.undetermined += 1
|
||||
self.skip(ar_id, f"undeterminable (network): {exc}")
|
||||
|
||||
def exit_code(self) -> int:
|
||||
"""0 all pass, 1 any fail, 3 nothing failed but something undeterminable."""
|
||||
if self.fails:
|
||||
return EXIT_FAIL
|
||||
return EXIT_NETWORK if self.undetermined else EXIT_OK
|
||||
|
||||
|
||||
def summarize(items: list[str], limit: int = 4) -> str:
|
||||
"""Join up to ``limit`` items and note how many more there were."""
|
||||
head = ", ".join(items[:limit])
|
||||
return head + (f" (+{len(items) - limit} more)" if len(items) > limit else "")
|
||||
@ -1,136 +0,0 @@
|
||||
"""Fixtures: generated site trees and loopback HTTP servers that serve them."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import threading
|
||||
from collections.abc import Callable, Iterator
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from agent_ready.tests import sitegen
|
||||
|
||||
_MIME = {
|
||||
".html": "text/html; charset=utf-8",
|
||||
".md": "text/markdown; charset=utf-8",
|
||||
".txt": "text/plain; charset=utf-8",
|
||||
".xml": "application/xml",
|
||||
".json": "application/json",
|
||||
}
|
||||
|
||||
|
||||
def _resolve(root: Path, path: str) -> Path | None:
|
||||
"""Map a URL path to a file the way a static host does."""
|
||||
rel = path.split("?", 1)[0].lstrip("/")
|
||||
cand = root / rel
|
||||
if path.endswith("/"):
|
||||
cand = cand / "index.html"
|
||||
return cand if cand.is_file() else None
|
||||
|
||||
|
||||
def make_handler(root: Path, mode: str, block_ua: str) -> type[BaseHTTPRequestHandler]:
|
||||
"""Build a handler class.
|
||||
|
||||
``mode`` is ``good`` (real 404s), ``spa`` (200 index.html for every path,
|
||||
the kuhy.duckdns.org trap) or ``json404`` (every non-root path is a JSON
|
||||
404, the popes.kuhy.duckdns.org shape).
|
||||
"""
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *args: object) -> None: # silence test output
|
||||
return
|
||||
|
||||
def _send(
|
||||
self,
|
||||
status: int,
|
||||
ctype: str,
|
||||
body: bytes,
|
||||
extra: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", ctype)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
for key, value in (extra or {}).items():
|
||||
self.send_header(key, value)
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_GET(self) -> None:
|
||||
if block_ua and block_ua in self.headers.get("User-Agent", ""):
|
||||
self._send(403, "text/html", b"<html>Just a moment...</html>")
|
||||
return
|
||||
origin = f"http://{self.headers['Host']}"
|
||||
if mode == "spa" or (mode == "json404" and self.path == "/"):
|
||||
body = (root / "index.html").read_bytes()
|
||||
self._send(200, "text/html; charset=utf-8", body)
|
||||
return
|
||||
target = None if mode == "json404" else _resolve(root, self.path)
|
||||
if target is None:
|
||||
if mode == "json404":
|
||||
self._send(404, "application/json", b'{"detail":"Not Found"}')
|
||||
else:
|
||||
self._send(404, "text/html", (root / "404.html").read_bytes())
|
||||
return
|
||||
if target.name == "ai-catalog.json":
|
||||
ctype = "application/ai-catalog+json"
|
||||
else:
|
||||
ctype = _MIME.get(target.suffix, "application/octet-stream")
|
||||
body = target.read_bytes().replace(sitegen.ORIGIN.encode(), origin.encode())
|
||||
extra = {}
|
||||
if self.path == "/":
|
||||
extra["Link"] = '</.well-known/ai-catalog.json>; rel="ai-catalog"'
|
||||
self._send(200, ctype, body, extra)
|
||||
|
||||
def do_POST(self) -> None:
|
||||
"""Minimal MCP: answer initialize at /mcp in good mode only."""
|
||||
self.rfile.read(int(self.headers.get("Content-Length", 0)))
|
||||
if mode == "good" and self.path == "/mcp":
|
||||
self._send(
|
||||
200, "application/json", b'{"jsonrpc":"2.0","id":1,"result":{}}'
|
||||
)
|
||||
else:
|
||||
self._send(200, "text/html", (root / "index.html").read_bytes())
|
||||
|
||||
return Handler
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def good_dir(tmp_path: Path) -> Path:
|
||||
"""A compliant build output."""
|
||||
return sitegen.build_good(tmp_path / "good")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bad_dir(tmp_path: Path) -> Path:
|
||||
"""A build output failing every find/read item."""
|
||||
return sitegen.build_bad(tmp_path / "bad")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def serve() -> Iterator[Callable[..., str]]:
|
||||
"""Start loopback servers on demand; return their base URL."""
|
||||
servers: list[ThreadingHTTPServer] = []
|
||||
|
||||
def start(root: Path, mode: str = "good", block_ua: str = "") -> str:
|
||||
server = ThreadingHTTPServer(
|
||||
("127.0.0.1", 0), make_handler(root, mode, block_ua)
|
||||
)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
servers.append(server)
|
||||
return f"http://127.0.0.1:{server.server_address[1]}"
|
||||
|
||||
yield start
|
||||
for server in servers:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dead_url() -> str:
|
||||
"""A loopback URL with nothing listening (connection refused)."""
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
return f"http://127.0.0.1:{port}"
|
||||
@ -1,80 +0,0 @@
|
||||
"""In-memory sites and CLI helpers for tests that need exact responses."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
|
||||
import pytest
|
||||
|
||||
from agent_ready.backends import DEFAULT_UA, HttpSite, Response, Site
|
||||
from agent_ready.check import main
|
||||
from agent_ready.context import Ctx
|
||||
from agent_ready.report import Report
|
||||
|
||||
HTML = "text/html"
|
||||
_MISSING = Response(404, HTML, b"")
|
||||
|
||||
|
||||
def resp(body: str, ctype: str = HTML, status: int = 200, **headers: str) -> Response:
|
||||
"""Build a Response; ``headers`` keys are already lowercase names."""
|
||||
return Response(status, ctype, body.encode(), dict(headers))
|
||||
|
||||
|
||||
class FakeSite(Site):
|
||||
"""A dist-like (``live=False``) site answering from a dict."""
|
||||
|
||||
def __init__(self, pages: dict[str, Response]) -> None:
|
||||
super().__init__("https://fake.test")
|
||||
self.pages = pages
|
||||
|
||||
def _fetch(self, path: str, ua: str) -> Response:
|
||||
return self.pages.get(path, _MISSING)
|
||||
|
||||
|
||||
class FakeHttp(HttpSite):
|
||||
"""A live site with no sockets: per-path (and per-UA) canned responses."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
pages: dict[str, Response],
|
||||
ua_pages: dict[tuple[str, str], Response] | None = None,
|
||||
mcp: Response | None = None,
|
||||
) -> None:
|
||||
super().__init__("https://fake.test")
|
||||
self.pages = pages
|
||||
self.ua_pages = ua_pages or {}
|
||||
self.mcp = mcp or _MISSING
|
||||
|
||||
def _fetch(self, path: str, ua: str) -> Response:
|
||||
return self.ua_pages.get((path, ua)) or self.pages.get(path, _MISSING)
|
||||
|
||||
def request(
|
||||
self,
|
||||
path: str,
|
||||
ua: str = DEFAULT_UA,
|
||||
data: bytes | None = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
) -> Response:
|
||||
"""The canned MCP answer, whatever the request."""
|
||||
return self.mcp
|
||||
|
||||
|
||||
def make_ctx(site: Site, *, act: bool = False) -> tuple[Ctx, io.StringIO]:
|
||||
"""A Ctx writing to a StringIO; returns it with that buffer."""
|
||||
out = io.StringIO()
|
||||
return Ctx(site=site, report=Report(out=out), act=act), out
|
||||
|
||||
|
||||
def run_main(capsys: pytest.CaptureFixture[str], *argv: str) -> tuple[int, list[str]]:
|
||||
"""Run the CLI in-process; return exit code and output lines."""
|
||||
code = main(list(argv))
|
||||
return code, capsys.readouterr().out.splitlines()
|
||||
|
||||
|
||||
def lines_for(lines: list[str], ar_id: str, status: str = "") -> list[str]:
|
||||
"""Output lines for ``ar_id`` (optionally only one status)."""
|
||||
return [
|
||||
ln
|
||||
for ln in lines
|
||||
if ln.split()[1] == ar_id and (not status or ln.startswith(status))
|
||||
]
|
||||
@ -1,153 +0,0 @@
|
||||
"""Build good and bad site trees in a temp dir (no static fixture files).
|
||||
|
||||
Generated at test time on purpose: mirror files such as ``index.md`` would
|
||||
otherwise trip the repo's markdown-naming gate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
ORIGIN = "https://example.test"
|
||||
|
||||
_TEXT = (
|
||||
"Dice roller for tabletop games. Roll any combination of polyhedral dice, "
|
||||
"keep the highest, and read the full breakdown of every throw. This text "
|
||||
"is in the initial HTML so an agent that never runs JavaScript can read it."
|
||||
)
|
||||
|
||||
_LD = json.dumps(
|
||||
{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "WebSite",
|
||||
"name": "Dice",
|
||||
"url": ORIGIN,
|
||||
}
|
||||
)
|
||||
|
||||
CATALOG = {
|
||||
"specVersion": "1.0",
|
||||
"host": {"displayName": "kuhy", "identifier": "did:web:example.test"},
|
||||
"entries": [
|
||||
{
|
||||
"identifier": "urn:air:example.test:docs:llms",
|
||||
"displayName": "llms.txt",
|
||||
"type": "text/plain",
|
||||
"url": f"{ORIGIN}/llms.txt",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
LLMS = f"""# Dice
|
||||
> A dice roller for tabletop games.
|
||||
|
||||
Plain prose is allowed between sections.
|
||||
|
||||
## Pages
|
||||
- [Home]({ORIGIN}/): what the roller does
|
||||
- [About]({ORIGIN}/about/): who made it
|
||||
"""
|
||||
|
||||
ROBOTS = f"""# robots.txt
|
||||
User-agent: *
|
||||
Disallow: /search/
|
||||
|
||||
Sitemap: {ORIGIN}/sitemap.xml
|
||||
"""
|
||||
|
||||
SITEMAP = f"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||
<url><loc>{ORIGIN}/</loc></url>
|
||||
<url><loc>{ORIGIN}/about/</loc></url>
|
||||
</urlset>
|
||||
"""
|
||||
|
||||
|
||||
def page(title: str, mirror: str, *, home: bool = False) -> str:
|
||||
"""Return a server-rendered HTML page with every read-stage feature."""
|
||||
head = [
|
||||
f"<title>{title}</title>",
|
||||
f'<link rel="alternate" type="text/markdown" href="{mirror}">',
|
||||
]
|
||||
if home:
|
||||
head += [
|
||||
'<link rel="ai-catalog" href="/.well-known/ai-catalog.json" type="application/ai-catalog+json">',
|
||||
'<link rel="alternate" type="text/plain" href="/llms.txt">',
|
||||
f'<script type="application/ld+json">{_LD}</script>',
|
||||
]
|
||||
return (
|
||||
'<!doctype html><html lang="en"><head>' + "".join(head) + "</head><body>"
|
||||
f"<h1>{title}</h1><p>{_TEXT}</p>"
|
||||
'<pre><code class="language-python">print("d20")</code></pre>'
|
||||
'<nav><a href="/">Home</a> <a href="/about/">About</a> <a href="/llms.txt">llms.txt</a></nav>'
|
||||
"</body></html>"
|
||||
)
|
||||
|
||||
|
||||
def mirror(title: str) -> str:
|
||||
"""Return a markdown mirror with a language-tagged fence."""
|
||||
return f"# {title}\n\n{_TEXT}\n\n```python\nprint('d20')\n```\n"
|
||||
|
||||
|
||||
def write(root: Path, rel: str, text: str) -> None:
|
||||
"""Write ``text`` to ``root/rel``, creating parent dirs."""
|
||||
path = root / rel
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def build_good(root: Path) -> Path:
|
||||
"""Create a fully compliant build output under ``root``."""
|
||||
write(root, "index.html", page("Dice", "/index.md", home=True))
|
||||
write(root, "index.md", mirror("Dice"))
|
||||
write(root, "about/index.html", page("About", "/about/index.md"))
|
||||
write(root, "about/index.md", mirror("About"))
|
||||
write(
|
||||
root,
|
||||
"404.html",
|
||||
"<!doctype html><html><body><h1>Not found</h1><p>No page at this address.</p></body></html>",
|
||||
)
|
||||
write(root, "robots.txt", ROBOTS)
|
||||
write(root, "sitemap.xml", SITEMAP)
|
||||
write(root, "llms.txt", LLMS)
|
||||
write(root, ".well-known/ai-catalog.json", json.dumps(CATALOG))
|
||||
return root
|
||||
|
||||
|
||||
SHELL = '<!doctype html><html><head><title>App</title></head><body><div id="root"></div><script src="/app.js"></script></body></html>'
|
||||
|
||||
|
||||
def build_bad(root: Path) -> Path:
|
||||
"""Create a build output that violates every find/read item."""
|
||||
write(root, "index.html", SHELL)
|
||||
write(root, "about/index.html", SHELL)
|
||||
write(
|
||||
root,
|
||||
"robots.txt",
|
||||
"User-agent: *\nDisallow:\n\nUser-agent: GPTBot\nDisallow: /\n",
|
||||
)
|
||||
write(root, "sitemap.xml", "<html>not xml")
|
||||
write(root, "llms.txt", "# Dice\n\n## Pages\n- [About](/about/)\n")
|
||||
write(
|
||||
root,
|
||||
".well-known/ai-catalog.json",
|
||||
json.dumps({"entries": [{"identifier": "x"}]}),
|
||||
)
|
||||
return root
|
||||
|
||||
|
||||
def add_pages(root: Path, count: int, *, broken_last: bool = False) -> None:
|
||||
"""Add ``count`` more sitemap pages; the last one an empty shell if asked."""
|
||||
urls = []
|
||||
for i in range(count):
|
||||
title = f"Post {i}"
|
||||
html = (
|
||||
SHELL if broken_last and i == count - 1 else page(title, f"/p{i}/index.md")
|
||||
)
|
||||
write(root, f"p{i}/index.html", html)
|
||||
write(root, f"p{i}/index.md", mirror(title))
|
||||
urls.append(f" <url><loc>{ORIGIN}/p{i}/</loc></url>\n")
|
||||
write(
|
||||
root, "sitemap.xml", SITEMAP.replace("</urlset>", "".join(urls) + "</urlset>")
|
||||
)
|
||||
@ -1,75 +0,0 @@
|
||||
"""Backend edge cases: URL mapping, memoized failures, traversal, deadline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from agent_ready.backends import DirSite, HttpSite, Response, Site
|
||||
from agent_ready.report import Undeterminable
|
||||
from agent_ready.tests.fakes import FakeHttp, resp
|
||||
|
||||
|
||||
class FlakySite(Site):
|
||||
"""Raises Undeterminable on every fetch and counts the attempts."""
|
||||
|
||||
live = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__("https://flaky.test")
|
||||
self.attempts = 0
|
||||
|
||||
def _fetch(self, path: str, ua: str) -> Response:
|
||||
self.attempts += 1
|
||||
raise Undeterminable(f"{path}: boom")
|
||||
|
||||
|
||||
def test_to_path_rejects_offsite_and_non_http(tmp_path: Path) -> None:
|
||||
site = DirSite(tmp_path)
|
||||
assert site.to_path("https://other.test/x") is None
|
||||
assert site.to_path("mailto:a@b.test") is None
|
||||
assert site.to_path("/rel/") == "/rel/"
|
||||
assert site.to_path("https://dist.invalid") == "/"
|
||||
|
||||
|
||||
def test_failures_are_memoized() -> None:
|
||||
site = FlakySite()
|
||||
for _ in range(2):
|
||||
with pytest.raises(Undeterminable, match="boom"):
|
||||
site.get("/x")
|
||||
assert site.attempts == 1
|
||||
|
||||
|
||||
def test_prefetch_swallows_failures_then_get_raises() -> None:
|
||||
site = FlakySite()
|
||||
site.prefetch(["/a", "/b", "/a"])
|
||||
assert site.attempts == 2
|
||||
with pytest.raises(Undeterminable):
|
||||
site.get("/a")
|
||||
|
||||
|
||||
def test_base_fetch_is_abstract() -> None:
|
||||
with pytest.raises(NotImplementedError):
|
||||
Site("https://x.test")._fetch("/", "ua")
|
||||
|
||||
|
||||
def test_dist_never_serves_outside_root(tmp_path: Path) -> None:
|
||||
root = tmp_path / "dist"
|
||||
root.mkdir()
|
||||
(tmp_path / "secret.txt").write_text("nope")
|
||||
site = DirSite(root)
|
||||
assert site.file_for("/../secret.txt") is None
|
||||
assert site.get("/../secret.txt").status == 404
|
||||
|
||||
|
||||
def test_exhausted_budget_is_undeterminable() -> None:
|
||||
site = HttpSite("https://budget.test/anything", budget=0)
|
||||
with pytest.raises(Undeterminable, match="time budget"):
|
||||
site.get("/")
|
||||
|
||||
|
||||
def test_fake_http_serves_canned_pages() -> None:
|
||||
site = FakeHttp({"/": resp("hi")})
|
||||
assert site.get("/").text == "hi"
|
||||
assert site.get("/nope").status == 404
|
||||
@ -1,153 +0,0 @@
|
||||
"""Dist-mode scenarios: one mutation of the good tree per failing branch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from agent_ready.tests import sitegen
|
||||
from agent_ready.tests.fakes import lines_for, run_main
|
||||
|
||||
OPENAPI_OP = {"get": {"operationId": "ping"}}
|
||||
|
||||
|
||||
def test_robots_without_wildcard_group(
|
||||
capsys: pytest.CaptureFixture[str], good_dir: Path
|
||||
) -> None:
|
||||
robots = f"User-agent: Foo\nDisallow: /x\n\nSitemap: {sitegen.ORIGIN}/sitemap.xml\n"
|
||||
sitegen.write(good_dir, "robots.txt", robots)
|
||||
_, lines = run_main(capsys, "dist", str(good_dir))
|
||||
assert "no 'User-agent: *' group" in lines_for(lines, "AR-FIND-01", "FAIL")[0]
|
||||
|
||||
|
||||
def test_sitemap_found_without_robots_reference_sets_origin(
|
||||
capsys: pytest.CaptureFixture[str], good_dir: Path
|
||||
) -> None:
|
||||
sitegen.write(good_dir, "robots.txt", "User-agent: *\nDisallow:\n")
|
||||
code, lines = run_main(capsys, "dist", str(good_dir))
|
||||
assert code == 1
|
||||
assert "no 'Sitemap:' line" in lines_for(lines, "AR-FIND-03", "FAIL")[0]
|
||||
assert lines_for(lines, "AR-FIND-03", "FAIL")[1:] == []
|
||||
assert lines_for(lines, "AR-READ-01", "PASS")
|
||||
|
||||
|
||||
def test_sitemap_index_follows_children(
|
||||
capsys: pytest.CaptureFixture[str], good_dir: Path
|
||||
) -> None:
|
||||
child = f"{sitegen.ORIGIN}/sitemap-pages.xml"
|
||||
index = (
|
||||
'<?xml version="1.0"?><sitemapindex '
|
||||
'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
|
||||
f"<sitemap><loc>{child}</loc></sitemap></sitemapindex>"
|
||||
)
|
||||
sitegen.write(good_dir, "sitemap-pages.xml", sitegen.SITEMAP)
|
||||
sitegen.write(good_dir, "sitemap.xml", index)
|
||||
code, lines = run_main(capsys, "dist", str(good_dir))
|
||||
assert code == 0, lines
|
||||
assert "lists 2 pages" in lines_for(lines, "AR-FIND-03", "PASS")[-1]
|
||||
|
||||
|
||||
def test_sitemap_with_offsite_locs_and_none_at_all(
|
||||
capsys: pytest.CaptureFixture[str], good_dir: Path
|
||||
) -> None:
|
||||
urlset = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">{}</urlset>'
|
||||
off = urlset.format("<url><loc>https://other.test/x/</loc></url>")
|
||||
sitegen.write(good_dir, "sitemap.xml", off)
|
||||
_, lines = run_main(capsys, "dist", str(good_dir))
|
||||
assert "1 <loc> URLs are not on" in lines_for(lines, "AR-FIND-03", "FAIL")[0]
|
||||
sitegen.write(good_dir, "sitemap.xml", urlset.format(""))
|
||||
_, lines = run_main(capsys, "dist", str(good_dir))
|
||||
assert "no <loc> entries" in lines_for(lines, "AR-FIND-03", "FAIL")[0]
|
||||
assert "sitemap yielded no pages" in lines_for(lines, "AR-READ-01", "FAIL")[0]
|
||||
|
||||
|
||||
def test_sitemap_pages_missing_or_copies_of_the_shell(
|
||||
capsys: pytest.CaptureFixture[str], good_dir: Path
|
||||
) -> None:
|
||||
home = (good_dir / "index.html").read_text()
|
||||
sitegen.write(good_dir, "dup/index.html", home)
|
||||
extra = "".join(
|
||||
f"<url><loc>{sitegen.ORIGIN}/{p}/</loc></url>" for p in ("dup", "gone")
|
||||
)
|
||||
sitemap = (good_dir / "sitemap.xml").read_text()
|
||||
sitegen.write(
|
||||
good_dir, "sitemap.xml", sitemap.replace("</urlset>", extra + "</urlset>")
|
||||
)
|
||||
code, lines = run_main(capsys, "dist", str(good_dir))
|
||||
assert code == 1
|
||||
problem = lines_for(lines, "AR-READ-01", "FAIL")[0]
|
||||
assert "/gone/ (HTTP 404)" in problem
|
||||
assert "/dup/ (byte-identical to the homepage shell)" in problem
|
||||
|
||||
|
||||
def test_markdown_mirror_with_untagged_fence(
|
||||
capsys: pytest.CaptureFixture[str], good_dir: Path
|
||||
) -> None:
|
||||
sitegen.write(good_dir, "about/index.md", "# About\n\n```\nplain\n```\n")
|
||||
_, lines = run_main(capsys, "dist", str(good_dir))
|
||||
assert "about/index.md (1 untagged fences)" in lines_for(lines, "AR-READ-05")[0]
|
||||
|
||||
|
||||
def test_llms_with_broken_link(
|
||||
capsys: pytest.CaptureFixture[str], good_dir: Path
|
||||
) -> None:
|
||||
llms = (good_dir / "llms.txt").read_text() + "- [Gone](/gone/): missing\n"
|
||||
sitegen.write(good_dir, "llms.txt", llms)
|
||||
_, lines = run_main(capsys, "dist", str(good_dir))
|
||||
assert "broken links: /gone/" in lines_for(lines, "AR-READ-07", "FAIL")[0]
|
||||
|
||||
|
||||
def test_homepage_with_unparseable_json_ld(
|
||||
capsys: pytest.CaptureFixture[str], good_dir: Path
|
||||
) -> None:
|
||||
home = (good_dir / "index.html").read_text()
|
||||
sitegen.write(good_dir, "index.html", home.replace('"@context"', "@context"))
|
||||
_, lines = run_main(capsys, "dist", str(good_dir))
|
||||
assert "1 JSON-LD blocks fail" in lines_for(lines, "AR-READ-08", "FAIL")[-1]
|
||||
|
||||
|
||||
def test_webmcp_found_in_linked_script(
|
||||
capsys: pytest.CaptureFixture[str], good_dir: Path
|
||||
) -> None:
|
||||
home = (good_dir / "index.html").read_text()
|
||||
scripts = '<script src="https://cdn.other.test/x.js"></script><script src="/app.js"></script>'
|
||||
sitegen.write(good_dir, "index.html", home.replace("</body>", scripts + "</body>"))
|
||||
sitegen.write(good_dir, "app.js", "document.modelContext.registerTool({});")
|
||||
_, lines = run_main(capsys, "dist", str(good_dir), "--act")
|
||||
assert lines_for(lines, "AR-ACT-07", "PASS")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("doc", "status", "needle"),
|
||||
[
|
||||
({"openapi": "3.1.0", "paths": {"/p": OPENAPI_OP}}, "PASS", "1 operations"),
|
||||
(
|
||||
{"openapi": "3.1.0", "paths": {"/p": {"get": {}}}},
|
||||
"FAIL",
|
||||
"lack operationId",
|
||||
),
|
||||
({"openapi": "3.1.0", "paths": {}}, "FAIL", "no operations in paths"),
|
||||
({"openapi": "3.1.0", "paths": 5}, "FAIL", "no operations in paths"),
|
||||
(
|
||||
{
|
||||
"openapi": "3.1.0",
|
||||
"paths": {"/a": "x", "/b": {"parameters": [], "get": 1}},
|
||||
},
|
||||
"FAIL",
|
||||
"1 operations lack",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_openapi_document_shapes(
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
good_dir: Path,
|
||||
doc: dict[str, object],
|
||||
status: str,
|
||||
needle: str,
|
||||
) -> None:
|
||||
sitegen.write(good_dir, "openapi.json", json.dumps(doc))
|
||||
_, lines = run_main(capsys, "dist", str(good_dir), "--act")
|
||||
found = lines_for(lines, "AR-ACT-02", status)
|
||||
assert found and needle in found[0], lines_for(lines, "AR-ACT-02")
|
||||
@ -1,83 +0,0 @@
|
||||
"""Live-mode checks driven by canned responses (no sockets, no network)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agent_ready import checks_act, checks_find, checks_read
|
||||
from agent_ready.backends import Response
|
||||
from agent_ready.tests import sitegen
|
||||
from agent_ready.tests.fakes import FakeHttp, make_ctx, resp
|
||||
|
||||
LONG = "Readable text for agents that never run JavaScript. " * 4
|
||||
MD_LINK = '</index.md>; rel="alternate"; type="text/markdown"'
|
||||
|
||||
|
||||
def _home(*, ctype: str = "text/html", **headers: str) -> Response:
|
||||
return resp(f"<html><body><p>{LONG}</p></body></html>", ctype, **headers)
|
||||
|
||||
|
||||
def test_challenge_page_for_one_agent_is_reported() -> None:
|
||||
challenge = resp("<html>Just a moment...</html>")
|
||||
ua = checks_find.AGENT_UAS["GPTBot"]
|
||||
site = FakeHttp({"/": _home()}, ua_pages={("/", ua): challenge})
|
||||
ctx, out = make_ctx(site)
|
||||
checks_find.check_agent_access(ctx)
|
||||
assert out.getvalue().startswith("FAIL AR-FIND-01 GPTBot got a challenge page")
|
||||
|
||||
|
||||
def test_sitemap_on_another_host_is_a_failure() -> None:
|
||||
robots = "User-agent: *\nDisallow:\nSitemap: https://other.test/sitemap.xml\n"
|
||||
site = FakeHttp({"/robots.txt": resp(robots, "text/plain")})
|
||||
ctx, out = make_ctx(site)
|
||||
checks_find.check_robots(ctx)
|
||||
ctx.sitemap_pages = []
|
||||
assert checks_find._sitemap_paths(ctx) == ["/sitemap.xml"]
|
||||
assert (
|
||||
"Sitemap is on another host: https://other.test/sitemap.xml" in out.getvalue()
|
||||
)
|
||||
|
||||
|
||||
def test_markdown_mirror_advertised_only_by_link_header() -> None:
|
||||
site = FakeHttp(
|
||||
{
|
||||
"/": _home(link=MD_LINK),
|
||||
"/index.md": resp(sitegen.mirror("Home"), "text/markdown"),
|
||||
}
|
||||
)
|
||||
ctx, out = make_ctx(site)
|
||||
checks_read.check_pages(ctx)
|
||||
assert (
|
||||
"PASS AR-READ-09 1 pages advertise a working markdown mirror" in out.getvalue()
|
||||
)
|
||||
|
||||
|
||||
def test_page_served_with_wrong_content_type() -> None:
|
||||
ctx, out = make_ctx(FakeHttp({"/": _home(ctype="text/plain")}))
|
||||
checks_read.check_pages(ctx)
|
||||
assert (
|
||||
"FAIL AR-READ-01 answer not in initial HTML: / (text/plain)" in out.getvalue()
|
||||
)
|
||||
|
||||
|
||||
def _mcp(status: int, **headers: str) -> Response:
|
||||
return Response(status, "application/json", b"{}", dict(headers))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("answer", "status"),
|
||||
[
|
||||
(_mcp(401, **{"www-authenticate": "Bearer"}), "PASS"),
|
||||
(_mcp(401), "FAIL"),
|
||||
],
|
||||
)
|
||||
def test_mcp_behind_oauth_needs_www_authenticate(answer: Response, status: str) -> None:
|
||||
ctx, out = make_ctx(FakeHttp({}, mcp=answer), act=True)
|
||||
checks_act.check_mcp(ctx)
|
||||
assert f"{status} AR-ACT-03" in out.getvalue()
|
||||
|
||||
|
||||
def test_act_off_reports_a_single_skip() -> None:
|
||||
ctx, out = make_ctx(FakeHttp({}))
|
||||
checks_act.run_act(ctx)
|
||||
assert out.getvalue().startswith("SKIP AR-ACT-10 act stage off")
|
||||
@ -1,202 +0,0 @@
|
||||
"""End-to-end tests of ``check.py`` in dist and url mode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import runpy
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from agent_ready.check import main
|
||||
from agent_ready.tests import sitegen
|
||||
|
||||
LINE = re.compile(r"^(PASS|FAIL|SKIP) AR-(FIND|READ|ACT)-\d\d \S")
|
||||
PKG = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def run(capsys: pytest.CaptureFixture[str], *argv: str) -> tuple[int, list[str]]:
|
||||
"""Run the CLI in-process; return exit code and output lines."""
|
||||
code = main(list(argv))
|
||||
lines = capsys.readouterr().out.splitlines()
|
||||
bad = [ln for ln in lines if not LINE.match(ln)]
|
||||
assert not bad, bad
|
||||
return code, lines
|
||||
|
||||
|
||||
def statuses(lines: list[str], ar_id: str) -> set[str]:
|
||||
"""Return the set of statuses reported for ``ar_id``."""
|
||||
return {ln.split()[0] for ln in lines if ln.split()[1] == ar_id}
|
||||
|
||||
|
||||
def test_dist_good_passes(capsys: pytest.CaptureFixture[str], good_dir: Path) -> None:
|
||||
code, lines = run(capsys, "dist", str(good_dir))
|
||||
assert code == 0, "\n".join(lines)
|
||||
assert not [ln for ln in lines if ln.startswith("FAIL")]
|
||||
for ar_id in (
|
||||
"AR-FIND-01",
|
||||
"AR-FIND-03",
|
||||
"AR-FIND-04",
|
||||
"AR-READ-01",
|
||||
"AR-READ-02",
|
||||
"AR-READ-07",
|
||||
"AR-READ-09",
|
||||
):
|
||||
found = statuses(lines, ar_id)
|
||||
assert "PASS" in found and "FAIL" not in found, ar_id
|
||||
assert statuses(lines, "AR-READ-11") == {"SKIP"}
|
||||
assert statuses(lines, "AR-ACT-10") == {"SKIP"}
|
||||
|
||||
|
||||
def test_dist_bad_fails_every_item(
|
||||
capsys: pytest.CaptureFixture[str], bad_dir: Path
|
||||
) -> None:
|
||||
code, lines = run(capsys, "dist", str(bad_dir))
|
||||
assert code == 1
|
||||
for ar_id in (
|
||||
"AR-FIND-01",
|
||||
"AR-FIND-03",
|
||||
"AR-FIND-04",
|
||||
"AR-READ-01",
|
||||
"AR-READ-02",
|
||||
"AR-READ-03",
|
||||
"AR-READ-06",
|
||||
"AR-READ-07",
|
||||
"AR-READ-08",
|
||||
"AR-READ-09",
|
||||
):
|
||||
assert "FAIL" in statuses(lines, ar_id), ar_id
|
||||
assert any("gptbot" in ln for ln in lines if "AR-FIND-01" in ln)
|
||||
|
||||
|
||||
def test_dist_untagged_code_fails(
|
||||
capsys: pytest.CaptureFixture[str], good_dir: Path
|
||||
) -> None:
|
||||
about = good_dir / "about/index.html"
|
||||
about.write_text(about.read_text().replace(' class="language-python"', ""))
|
||||
code, lines = run(capsys, "dist", str(good_dir))
|
||||
assert code == 1
|
||||
assert "FAIL" in statuses(lines, "AR-READ-05")
|
||||
|
||||
|
||||
def test_dist_act_opt_in(capsys: pytest.CaptureFixture[str], good_dir: Path) -> None:
|
||||
code, lines = run(capsys, "dist", str(good_dir), "--act")
|
||||
assert code == 1
|
||||
assert "FAIL" in statuses(lines, "AR-ACT-02")
|
||||
|
||||
|
||||
def test_usage_errors(capsys: pytest.CaptureFixture[str], tmp_path: Path) -> None:
|
||||
assert main(["dist", str(tmp_path / "missing")]) == 2
|
||||
assert main(["url", "not-a-url"]) == 2
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
main([])
|
||||
assert exc.value.code == 2
|
||||
|
||||
|
||||
def test_url_good_passes(
|
||||
capsys: pytest.CaptureFixture[str], good_dir: Path, serve: Callable[..., str]
|
||||
) -> None:
|
||||
code, lines = run(capsys, "url", serve(good_dir))
|
||||
assert code == 0, "\n".join(lines)
|
||||
assert statuses(lines, "AR-READ-02") == {"PASS"}
|
||||
|
||||
|
||||
def test_url_spa_fallback_fails(
|
||||
capsys: pytest.CaptureFixture[str], good_dir: Path, serve: Callable[..., str]
|
||||
) -> None:
|
||||
"""Every path 200 text/html: a status-only check would wrongly pass."""
|
||||
code, lines = run(capsys, "url", serve(good_dir, "spa"))
|
||||
assert code == 1
|
||||
for ar_id in (
|
||||
"AR-FIND-01",
|
||||
"AR-FIND-03",
|
||||
"AR-FIND-04",
|
||||
"AR-READ-01",
|
||||
"AR-READ-02",
|
||||
"AR-READ-07",
|
||||
"AR-READ-09",
|
||||
):
|
||||
assert "FAIL" in statuses(lines, ar_id), ar_id
|
||||
assert any("text/html" in ln for ln in lines if "AR-FIND-01" in ln)
|
||||
assert any(
|
||||
"does not exist" in ln for ln in lines if "AR-READ-06" in ln
|
||||
) # HTML fallback != file
|
||||
|
||||
|
||||
def test_url_json_404(
|
||||
capsys: pytest.CaptureFixture[str], good_dir: Path, serve: Callable[..., str]
|
||||
) -> None:
|
||||
code, lines = run(capsys, "url", serve(good_dir, "json404"))
|
||||
assert code == 1
|
||||
assert statuses(lines, "AR-READ-02") == {"PASS"}
|
||||
assert "FAIL" in statuses(lines, "AR-FIND-01")
|
||||
|
||||
|
||||
def test_url_blocked_agent(
|
||||
capsys: pytest.CaptureFixture[str], good_dir: Path, serve: Callable[..., str]
|
||||
) -> None:
|
||||
code, lines = run(capsys, "url", serve(good_dir, block_ua="GPTBot"))
|
||||
assert code == 1
|
||||
assert any(ln.startswith("FAIL AR-FIND-01") and "GPTBot" in ln for ln in lines)
|
||||
|
||||
|
||||
def test_url_unreachable_is_3(
|
||||
capsys: pytest.CaptureFixture[str], dead_url: str
|
||||
) -> None:
|
||||
assert main(["url", dead_url]) == 3
|
||||
|
||||
|
||||
def test_entry_points_from_other_cwd(good_dir: Path, tmp_path: Path) -> None:
|
||||
for cmd in (
|
||||
["bash", str(PKG / "check.sh")],
|
||||
[sys.executable, str(PKG / "check.py")],
|
||||
):
|
||||
proc = subprocess.run(
|
||||
[*cmd, "dist", str(good_dir)],
|
||||
cwd=tmp_path,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert proc.returncode == 0, proc.stdout + proc.stderr
|
||||
|
||||
|
||||
def test_dist_checks_every_page_url_caps_visibly(
|
||||
capsys: pytest.CaptureFixture[str], good_dir: Path, serve: Callable[..., str]
|
||||
) -> None:
|
||||
"""Dist mode ignores --max-pages; url mode names the cap instead of hiding it."""
|
||||
sitegen.add_pages(good_dir, 6, broken_last=True)
|
||||
code, lines = run(capsys, "dist", str(good_dir), "--max-pages", "3")
|
||||
assert code == 1
|
||||
assert any(ln.startswith("FAIL AR-READ-01") and "/p5/" in ln for ln in lines)
|
||||
code, lines = run(capsys, "url", serve(good_dir), "--max-pages", "3")
|
||||
assert any(ln.startswith("SKIP AR-READ-01 checked 3 of 8 pages") for ln in lines), (
|
||||
lines
|
||||
)
|
||||
|
||||
|
||||
def test_url_act_mcp(
|
||||
capsys: pytest.CaptureFixture[str], good_dir: Path, serve: Callable[..., str]
|
||||
) -> None:
|
||||
_, lines = run(capsys, "url", serve(good_dir), "--act")
|
||||
assert statuses(lines, "AR-ACT-03") == {"PASS"}
|
||||
assert statuses(lines, "AR-ACT-10") == {"PASS"}
|
||||
assert "FAIL" in statuses(lines, "AR-ACT-07")
|
||||
code, lines = run(capsys, "url", serve(good_dir, "spa"), "--act")
|
||||
assert code == 1
|
||||
assert statuses(lines, "AR-ACT-03") == {"FAIL"}
|
||||
assert statuses(lines, "AR-ACT-10") == {"FAIL"}
|
||||
|
||||
|
||||
def test_script_entry_point_in_process(good_dir: Path) -> None:
|
||||
"""Run check.py as ``__main__`` here so the guard lines count for coverage."""
|
||||
argv = ["check.py", "dist", str(good_dir)]
|
||||
with pytest.MonkeyPatch.context() as patch:
|
||||
patch.setattr(sys, "argv", argv)
|
||||
patch.setattr(sys, "path", list(sys.path)) # check.py prepends to it
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
runpy.run_path(str(PKG / "check.py"), run_name="__main__")
|
||||
assert exc.value.code == 0
|
||||
@ -1,186 +0,0 @@
|
||||
"""Unit tests for the format validators and the HTML scanner."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agent_ready import formats, htmlscan
|
||||
from agent_ready.tests import sitegen
|
||||
|
||||
# Verbatim from spec/spec.md "llms.txt shape (informative)".
|
||||
SPEC_LLMS = """# Acme
|
||||
> Acme ships a payments API for developers.
|
||||
|
||||
## Docs
|
||||
- [Quickstart](https://example.com/docs/quickstart): first charge in one request
|
||||
- [Authentication](https://example.com/docs/auth): API keys, scopes, rotation
|
||||
|
||||
## API
|
||||
- [OpenAPI](https://example.com/openapi.json): full request and response contract
|
||||
"""
|
||||
|
||||
# Shape of https://vercel.com/.well-known/ai-catalog.json (2026-10-22), trimmed.
|
||||
VERCEL_CATALOG = {
|
||||
"specVersion": "1.0",
|
||||
"host": {"displayName": "Vercel", "identifier": "https://vercel.com"},
|
||||
"entries": [
|
||||
{
|
||||
"identifier": "urn:air:vercel.com:api:rest",
|
||||
"displayName": "Vercel REST API",
|
||||
"type": "application/json",
|
||||
"url": "https://vercel.com/openapi.json",
|
||||
"tags": ["api"],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_llms_spec_example_is_valid() -> None:
|
||||
assert formats.llms_problems(SPEC_LLMS) == []
|
||||
|
||||
|
||||
def test_llms_allows_prose_and_plain_bullets() -> None:
|
||||
text = SPEC_LLMS.replace(
|
||||
"## API", "Version: v2.0\n\n## Notes\n- plain bullet\n\n## API"
|
||||
)
|
||||
assert formats.llms_problems(text) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mutate", "needle"),
|
||||
[
|
||||
(lambda t: t.replace("# Acme\n", ""), "H1"),
|
||||
(
|
||||
lambda t: t.replace("> Acme ships a payments API for developers.\n", ""),
|
||||
"blockquote",
|
||||
),
|
||||
(lambda t: "# Acme\n> summary\n\nno sections\n", "link list"),
|
||||
(lambda t: "<!doctype html><html></html>", "HTML"),
|
||||
],
|
||||
)
|
||||
def test_llms_rejects(mutate: object, needle: str) -> None:
|
||||
problems = formats.llms_problems(mutate(SPEC_LLMS)) # type: ignore[operator]
|
||||
assert any(needle in p for p in problems), problems
|
||||
|
||||
|
||||
def test_llms_links() -> None:
|
||||
assert formats.llms_links(SPEC_LLMS)[0] == "https://example.com/docs/quickstart"
|
||||
|
||||
|
||||
def test_catalog_valid_examples() -> None:
|
||||
assert formats.catalog_problems(VERCEL_CATALOG) == []
|
||||
assert formats.catalog_problems(sitegen.CATALOG) == []
|
||||
minimal = {
|
||||
"specVersion": "1.0",
|
||||
"entries": [{"identifier": "a", "type": "t", "data": {}}],
|
||||
}
|
||||
assert formats.catalog_problems(minimal) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("doc", "needle"),
|
||||
[
|
||||
([], "object"),
|
||||
({"entries": []}, "specVersion"),
|
||||
({"specVersion": "1", "entries": []}, "Major.Minor"),
|
||||
({"specVersion": "1.0"}, "entries"),
|
||||
({"specVersion": "1.0", "entries": [{"type": "t", "url": "u"}]}, "identifier"),
|
||||
({"specVersion": "1.0", "entries": [{"identifier": "i", "url": "u"}]}, "type"),
|
||||
(
|
||||
{
|
||||
"specVersion": "1.0",
|
||||
"entries": [{"identifier": "i", "type": "t", "url": "u", "data": 1}],
|
||||
},
|
||||
"exactly one",
|
||||
),
|
||||
({"specVersion": "1.0", "entries": [], "host": {}}, "host.displayName"),
|
||||
],
|
||||
)
|
||||
def test_catalog_rejects(doc: object, needle: str) -> None:
|
||||
assert any(needle in p for p in formats.catalog_problems(doc)), (
|
||||
formats.catalog_problems(doc)
|
||||
)
|
||||
|
||||
|
||||
def test_robots_blanket_blocks() -> None:
|
||||
text = "User-agent: *\nDisallow:\n\nUser-Agent: GPTBot\nUser-agent: CCBot\nDisallow: /*\n"
|
||||
groups = formats.parse_robots(text)
|
||||
assert formats.has_wildcard_group(groups)
|
||||
assert formats.blocked_agents(groups) == ["gptbot", "ccbot"]
|
||||
|
||||
|
||||
def test_robots_partial_disallow_and_allow_override() -> None:
|
||||
text = "User-agent: *\nDisallow: /search/\n\nUser-agent: ClaudeBot\nDisallow: /\nAllow: /\n"
|
||||
assert formats.blocked_agents(formats.parse_robots(text)) == []
|
||||
assert formats.robots_sitemaps("Sitemap: https://x.test/s.xml\n") == [
|
||||
"https://x.test/s.xml"
|
||||
]
|
||||
|
||||
|
||||
def test_sitemap_parse() -> None:
|
||||
kind, locs = formats.parse_sitemap(sitegen.SITEMAP.encode())
|
||||
assert kind == "urlset"
|
||||
assert locs == ["https://example.test/", "https://example.test/about/"]
|
||||
with pytest.raises(ValueError):
|
||||
formats.parse_sitemap(b"<html>not xml")
|
||||
|
||||
|
||||
def test_untagged_fences() -> None:
|
||||
assert formats.untagged_fences("```python\nx\n```\n") == 0
|
||||
assert formats.untagged_fences("```\nx\n```\n~~~\ny\n~~~\n") == 2
|
||||
|
||||
|
||||
def test_htmlscan_good_home() -> None:
|
||||
scan = htmlscan.scan(sitegen.page("Dice", "/index.md", home=True))
|
||||
assert len(scan.text) > 150
|
||||
assert scan.code_blocks == 1 and scan.untagged_code == 0
|
||||
assert scan.link_href("alternate", "text/markdown") == "/index.md"
|
||||
assert scan.link_href("ai-catalog") == "/.well-known/ai-catalog.json"
|
||||
assert "/about/" in scan.anchors
|
||||
assert scan.jsonld_ok == 1
|
||||
|
||||
|
||||
def test_htmlscan_shell_has_no_text() -> None:
|
||||
scan = htmlscan.scan(sitegen.SHELL)
|
||||
assert scan.text.strip() == "" # <title> is head, not body text
|
||||
assert scan.anchors == []
|
||||
bare = htmlscan.scan(
|
||||
'<pre><code>x</code></pre><script type="application/ld+json">{bad</script>'
|
||||
)
|
||||
assert bare.untagged_code == 1 and bare.jsonld_bad == 1
|
||||
|
||||
|
||||
def test_sitemap_wrong_root_is_rejected() -> None:
|
||||
with pytest.raises(ValueError, match="not <urlset>"):
|
||||
formats.parse_sitemap(b"<feed><loc>https://x.test/</loc></feed>")
|
||||
|
||||
|
||||
def test_catalog_entry_must_be_an_object() -> None:
|
||||
doc = {"specVersion": "1.0", "entries": ["nope"]}
|
||||
assert formats.catalog_problems(doc) == ["entries[0]: must be an object"]
|
||||
|
||||
|
||||
def test_catalog_accessors_tolerate_invalid_docs() -> None:
|
||||
for doc in ([], {"entries": "x"}):
|
||||
assert formats.catalog_urls(doc) == []
|
||||
assert formats.catalog_types(doc) == []
|
||||
assert formats.catalog_urls(sitegen.CATALOG) == [f"{sitegen.ORIGIN}/llms.txt"]
|
||||
|
||||
|
||||
def test_fence_inside_fence_is_not_a_new_block() -> None:
|
||||
nested = "````md\n```\ninner\n```\n````\n"
|
||||
assert formats.untagged_fences(nested) == 0
|
||||
assert formats.untagged_fences("```python\n~~~\n```\n```\nz\n```\n") == 1
|
||||
|
||||
|
||||
def test_htmlscan_jsonld_without_context_is_bad() -> None:
|
||||
scan = htmlscan.scan(
|
||||
'<script type="application/ld+json">{"name": "x"}</script>'
|
||||
'<script type="application/ld+json">[]</script>'
|
||||
)
|
||||
assert scan.jsonld_ok == 0 and scan.jsonld_bad == 2
|
||||
|
||||
|
||||
def test_htmlscan_unclosed_inner_hidden_tag_closes_with_outer() -> None:
|
||||
scan = htmlscan.scan("<body><template><noscript>x</template><p>visible</p></body>")
|
||||
assert scan.text == "visible"
|
||||
@ -1,54 +0,0 @@
|
||||
"""Report policy, Link-header parsing and context helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
|
||||
from agent_ready.backends import Response
|
||||
from agent_ready.context import link_header_href
|
||||
from agent_ready.report import Report, Undeterminable, summarize
|
||||
from agent_ready.tests.fakes import FakeSite, make_ctx, resp
|
||||
|
||||
|
||||
def _boom() -> None:
|
||||
raise Undeterminable("refused")
|
||||
|
||||
|
||||
def test_guard_turns_network_failure_into_skip_and_exit_3() -> None:
|
||||
out = io.StringIO()
|
||||
report = Report(out=out)
|
||||
report.guard("AR-FIND-01", _boom)
|
||||
assert out.getvalue() == "SKIP AR-FIND-01 undeterminable (network): refused\n"
|
||||
assert report.exit_code() == 3
|
||||
report.fail("AR-FIND-01", "x")
|
||||
assert report.exit_code() == 1
|
||||
|
||||
|
||||
def test_summarize_notes_the_overflow() -> None:
|
||||
assert summarize(["a", "b"]) == "a, b"
|
||||
assert summarize(list("abcdef")) == "a, b, c, d (+2 more)"
|
||||
|
||||
|
||||
def _linked(link: str) -> Response:
|
||||
return Response(200, "text/html", b"", {"link": link})
|
||||
|
||||
|
||||
def test_link_header_skips_unusable_values() -> None:
|
||||
header = (
|
||||
"garbage, </no-rel>; type=x, </other>; rel=preload, "
|
||||
'</a.md>; rel="alternate"; type="text/plain", '
|
||||
'</b.md>; rel="alternate"; type="text/markdown"'
|
||||
)
|
||||
assert link_header_href(_linked(header), "alternate", "text/markdown") == "/b.md"
|
||||
assert link_header_href(_linked(header), "alternate") == "/a.md"
|
||||
assert link_header_href(_linked(header), "ai-catalog") is None
|
||||
assert link_header_href(_linked(""), "alternate") is None
|
||||
|
||||
|
||||
def test_exists_rejects_html_fallback_for_files() -> None:
|
||||
ctx, _ = make_ctx(
|
||||
FakeSite({"/f.txt": resp("<html><head></head>"), "/p/": resp("<html>")})
|
||||
)
|
||||
assert not ctx.exists("/f.txt")
|
||||
assert ctx.exists("/p/")
|
||||
assert not ctx.exists("/missing.txt")
|
||||
@ -1,7 +0,0 @@
|
||||
"""Deterministic coverage-gap lister: a coverage report in, a task list out.
|
||||
|
||||
Replaces "spawn an LLM agent to explore where coverage is missing". Reads lcov,
|
||||
Cobertura XML, JaCoCo XML or coverage.py JSON and lists, per file, exactly
|
||||
which lines and branches are uncovered. Exit codes are the verdict (0 covered,
|
||||
1 gaps, 2 unreadable) so CI and hooks adjudicate without a model.
|
||||
"""
|
||||
@ -1,5 +0,0 @@
|
||||
"""`python3 -m coverage_gaps` -- the entrypoint the `coverage-gaps` wrapper calls."""
|
||||
|
||||
from coverage_gaps.cli import main
|
||||
|
||||
raise SystemExit(main())
|
||||
@ -1,16 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# ============================================================================
|
||||
# coverage-gaps -- per-file task list of uncovered lines/branches.
|
||||
# Thin wrapper over `python3 -m coverage_gaps` (see coverage_gaps/cli.py).
|
||||
# Usage: coverage-gaps <report> [--json] [--max-files N] [--min-missed N]
|
||||
# [--path-prefix P] Exit: 0 covered | 1 gaps | 2 bad input
|
||||
# ============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
BIN_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd -P)"
|
||||
readonly BIN_DIR
|
||||
|
||||
PYTHONPATH="$BIN_DIR/../.." python3 -m coverage_gaps "$@"
|
||||
exit $?
|
||||
@ -1,63 +0,0 @@
|
||||
"""Command line: `coverage-gaps REPORT [--json] [--max-files N] ...`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
|
||||
from coverage_gaps.models import ReportError
|
||||
from coverage_gaps.parsers import load_report
|
||||
from coverage_gaps.render import select, to_json, to_markdown
|
||||
|
||||
EXIT_COVERED = 0
|
||||
EXIT_GAPS = 1
|
||||
EXIT_UNREADABLE = 2
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
"""Argument parser (format is auto-detected from file content)."""
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="coverage-gaps",
|
||||
description="List uncovered lines/branches from lcov, Cobertura, JaCoCo "
|
||||
"or coverage.py JSON reports. Exit 0 covered, 1 gaps, 2 unreadable.",
|
||||
)
|
||||
parser.add_argument("report", type=Path, help="coverage report file")
|
||||
parser.add_argument("--json", action="store_true", help="emit JSON, not markdown")
|
||||
parser.add_argument(
|
||||
"--max-files",
|
||||
type=int,
|
||||
default=0,
|
||||
metavar="N",
|
||||
help="list only the N worst files (0 = all)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--min-missed",
|
||||
type=int,
|
||||
default=1,
|
||||
metavar="N",
|
||||
help="skip files with fewer than N missed lines+branches",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--path-prefix",
|
||||
default="",
|
||||
metavar="PREFIX",
|
||||
help="only files whose path starts with PREFIX",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
"""Run the tool and return the process exit code."""
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
fmt, gaps = load_report(args.report)
|
||||
except ReportError as exc:
|
||||
print(f"coverage-gaps: {exc}", file=sys.stderr)
|
||||
return EXIT_UNREADABLE
|
||||
files = select(gaps, path_prefix=args.path_prefix, min_missed=args.min_missed)
|
||||
shown = args.max_files if args.max_files > 0 else len(files)
|
||||
render = to_json if args.json else to_markdown
|
||||
sys.stdout.write(render(fmt, files, shown) + ("\n" if args.json else ""))
|
||||
return EXIT_GAPS if files else EXIT_COVERED
|
||||
@ -1,69 +0,0 @@
|
||||
"""Format-neutral data model shared by every parser and renderer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
class ReportError(Exception):
|
||||
"""The input is unreadable, unrecognised or malformed (exit code 2)."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileAccumulator:
|
||||
"""Mutable per-file collector that merges repeated records for one path.
|
||||
|
||||
Reports may list a file more than once (lcov re-runs, Cobertura inner
|
||||
classes). A line or branch counts as covered if ANY record covered it.
|
||||
"""
|
||||
|
||||
lines: dict[int, bool] = field(default_factory=dict)
|
||||
branches: dict[tuple[int, int, int], bool] = field(default_factory=dict)
|
||||
|
||||
def add_line(self, line: int, covered: bool) -> None:
|
||||
"""Record a line, OR-merging with earlier records of the same line."""
|
||||
self.lines[line] = self.lines.get(line, False) or covered
|
||||
|
||||
def add_branch(self, line: int, block: int, index: int, covered: bool) -> None:
|
||||
"""Record one branch outcome, OR-merging on (line, block, index)."""
|
||||
key = (line, block, index)
|
||||
self.branches[key] = self.branches.get(key, False) or covered
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FileGap:
|
||||
"""Coverage state of one source file."""
|
||||
|
||||
path: str
|
||||
total_lines: int
|
||||
missed_lines: tuple[int, ...]
|
||||
total_branches: int
|
||||
missed_branch_lines: tuple[int, ...]
|
||||
missed_branches: int
|
||||
|
||||
@property
|
||||
def missed(self) -> int:
|
||||
"""Missed lines plus missed branches: the sort key."""
|
||||
return len(self.missed_lines) + self.missed_branches
|
||||
|
||||
@property
|
||||
def percent(self) -> float:
|
||||
"""Percent covered over lines and branches combined (100 if empty)."""
|
||||
total = self.total_lines + self.total_branches
|
||||
if total == 0:
|
||||
return 100.0
|
||||
return 100.0 * (total - self.missed) / total
|
||||
|
||||
|
||||
def build_gap(path: str, acc: FileAccumulator) -> FileGap:
|
||||
"""Freeze an accumulator into a FileGap."""
|
||||
missed_lines = tuple(sorted(n for n, ok in acc.lines.items() if not ok))
|
||||
missed_br = [key for key, ok in acc.branches.items() if not ok]
|
||||
return FileGap(
|
||||
path=path,
|
||||
total_lines=len(acc.lines),
|
||||
missed_lines=missed_lines,
|
||||
total_branches=len(acc.branches),
|
||||
missed_branch_lines=tuple(sorted({key[0] for key in missed_br})),
|
||||
missed_branches=len(missed_br),
|
||||
)
|
||||
@ -1,132 +0,0 @@
|
||||
"""Parsers for lcov, Cobertura XML, JaCoCo XML and coverage.py JSON."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
from coverage_gaps.models import FileAccumulator, FileGap, ReportError, build_gap
|
||||
|
||||
_CONDITION = re.compile(r"\((\d+)/(\d+)\)")
|
||||
|
||||
|
||||
def _finish(files: dict[str, FileAccumulator]) -> list[FileGap]:
|
||||
return [build_gap(path, acc) for path, acc in files.items()]
|
||||
|
||||
|
||||
def parse_lcov(text: str) -> list[FileGap]:
|
||||
"""Parse lcov.info: SF / DA / BRDA records, `end_of_record` optional."""
|
||||
files: dict[str, FileAccumulator] = {}
|
||||
acc: FileAccumulator | None = None
|
||||
for raw in text.splitlines():
|
||||
line = raw.strip()
|
||||
if line.startswith("SF:"):
|
||||
acc = files.setdefault(line[3:], FileAccumulator())
|
||||
elif acc is not None and line.startswith("DA:"):
|
||||
num, hits, *_ = line[3:].split(",")
|
||||
acc.add_line(int(num), int(hits) > 0)
|
||||
elif acc is not None and line.startswith("BRDA:"):
|
||||
num, block, branch, taken = line[5:].split(",")
|
||||
covered = taken not in ("-", "0")
|
||||
acc.add_branch(int(num), int(block), _branch_id(branch), covered)
|
||||
elif line == "end_of_record":
|
||||
acc = None
|
||||
return _finish(files)
|
||||
|
||||
|
||||
def _branch_id(branch: str) -> int:
|
||||
return int(branch) if branch.isdigit() else 0
|
||||
|
||||
|
||||
def parse_cobertura(text: str) -> list[FileGap]:
|
||||
"""Parse Cobertura `coverage.xml` (coverage.py, gcovr, ...)."""
|
||||
root = ET.fromstring(text)
|
||||
files: dict[str, FileAccumulator] = {}
|
||||
for cls in root.iter("class"):
|
||||
acc = files.setdefault(cls.get("filename", ""), FileAccumulator())
|
||||
for node in cls.iter("line"):
|
||||
num = int(node.get("number", "0"))
|
||||
acc.add_line(num, int(node.get("hits", "0")) > 0)
|
||||
match = _CONDITION.search(node.get("condition-coverage", ""))
|
||||
if match:
|
||||
hit, total = int(match[1]), int(match[2])
|
||||
for idx in range(total):
|
||||
acc.add_branch(num, 0, idx, idx < hit)
|
||||
return _finish(files)
|
||||
|
||||
|
||||
def parse_jacoco(text: str) -> list[FileGap]:
|
||||
"""Parse JaCoCo XML: per `sourcefile` line records (mi/ci/mb/cb)."""
|
||||
root = ET.fromstring(text)
|
||||
files: dict[str, FileAccumulator] = {}
|
||||
for package in root.iter("package"):
|
||||
base = package.get("name", "")
|
||||
for source in package.iter("sourcefile"):
|
||||
name = source.get("name", "")
|
||||
acc = files.setdefault(
|
||||
f"{base}/{name}" if base else name, FileAccumulator()
|
||||
)
|
||||
for node in source.iter("line"):
|
||||
num = int(node.get("nr", "0"))
|
||||
acc.add_line(num, int(node.get("ci", "0")) > 0)
|
||||
missed_b, covered_b = int(node.get("mb", "0")), int(node.get("cb", "0"))
|
||||
for idx in range(missed_b + covered_b):
|
||||
acc.add_branch(num, 0, idx, idx >= missed_b)
|
||||
return _finish(files)
|
||||
|
||||
|
||||
def parse_coverage_json(text: str) -> list[FileGap]:
|
||||
"""Parse `coverage json` output (with or without branch data)."""
|
||||
files: dict[str, FileAccumulator] = {}
|
||||
for path, data in json.loads(text).get("files", {}).items():
|
||||
acc = files.setdefault(path, FileAccumulator())
|
||||
for num in data.get("executed_lines", []):
|
||||
acc.add_line(num, True)
|
||||
for num in data.get("missing_lines", []):
|
||||
acc.add_line(num, False)
|
||||
missing = data.get("missing_branches", [])
|
||||
for idx, (src, _dst) in enumerate(missing):
|
||||
acc.add_branch(abs(src), 1, idx, False)
|
||||
covered = data.get("summary", {}).get("covered_branches", 0)
|
||||
for idx in range(covered):
|
||||
acc.add_branch(0, 2, idx, True)
|
||||
return _finish(files)
|
||||
|
||||
|
||||
def detect_format(text: str) -> str:
|
||||
"""Return 'lcov' | 'cobertura' | 'jacoco' | 'json' from file content."""
|
||||
head = text.lstrip()[:4000]
|
||||
if head.startswith("{"):
|
||||
return "json"
|
||||
if head.startswith("<"):
|
||||
if "<report" in head:
|
||||
return "jacoco"
|
||||
if "<coverage" in head:
|
||||
return "cobertura"
|
||||
elif re.search(r"^(TN|SF):", head, re.MULTILINE):
|
||||
return "lcov"
|
||||
raise ReportError("unrecognised coverage report format")
|
||||
|
||||
|
||||
_PARSERS: dict[str, Callable[[str], list[FileGap]]] = {
|
||||
"lcov": parse_lcov,
|
||||
"cobertura": parse_cobertura,
|
||||
"jacoco": parse_jacoco,
|
||||
"json": parse_coverage_json,
|
||||
}
|
||||
|
||||
|
||||
def load_report(path: Path) -> tuple[str, list[FileGap]]:
|
||||
"""Read and parse a report; raise ReportError on any failure."""
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError as exc:
|
||||
raise ReportError(f"cannot read {path}: {exc.strerror}") from exc
|
||||
fmt = detect_format(text)
|
||||
try:
|
||||
return fmt, _PARSERS[fmt](text)
|
||||
except (ET.ParseError, ValueError, TypeError, AttributeError) as exc:
|
||||
raise ReportError(f"malformed {fmt} report {path}: {exc}") from exc
|
||||
@ -1,102 +0,0 @@
|
||||
"""Filtering and rendering: markdown (agent task list) and JSON."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable, Sequence
|
||||
|
||||
from coverage_gaps.models import FileGap
|
||||
|
||||
|
||||
def compact_ranges(lines: Iterable[int]) -> str:
|
||||
"""Collapse line numbers into '12-18, 40'."""
|
||||
nums = sorted(set(lines))
|
||||
parts: list[str] = []
|
||||
idx = 0
|
||||
while idx < len(nums):
|
||||
end = idx
|
||||
while end + 1 < len(nums) and nums[end + 1] == nums[end] + 1:
|
||||
end += 1
|
||||
parts.append(_span(nums[idx], nums[end]))
|
||||
idx = end + 1
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
def _span(start: int, end: int) -> str:
|
||||
return str(start) if start == end else f"{start}-{end}"
|
||||
|
||||
|
||||
def select(
|
||||
gaps: Sequence[FileGap], *, path_prefix: str = "", min_missed: int = 1
|
||||
) -> list[FileGap]:
|
||||
"""Keep files with gaps under the prefix, worst first (ties by path)."""
|
||||
kept = [
|
||||
gap
|
||||
for gap in gaps
|
||||
if gap.path.startswith(path_prefix) and gap.missed >= max(min_missed, 1)
|
||||
]
|
||||
return sorted(kept, key=lambda gap: (-gap.missed, gap.path))
|
||||
|
||||
|
||||
def _totals(files: Sequence[FileGap]) -> dict[str, int]:
|
||||
return {
|
||||
"files": len(files),
|
||||
"missed_lines": sum(len(gap.missed_lines) for gap in files),
|
||||
"missed_branches": sum(gap.missed_branches for gap in files),
|
||||
}
|
||||
|
||||
|
||||
def to_json(fmt: str, files: Sequence[FileGap], shown: int) -> str:
|
||||
"""Machine-readable output; `files` is already sorted and filtered."""
|
||||
entries = [
|
||||
{
|
||||
"path": gap.path,
|
||||
"percent_covered": round(gap.percent, 2),
|
||||
"missed": gap.missed,
|
||||
"missed_lines": compact_ranges(gap.missed_lines),
|
||||
"missed_line_numbers": list(gap.missed_lines),
|
||||
"missed_branch_count": gap.missed_branches,
|
||||
"missed_branch_lines": list(gap.missed_branch_lines),
|
||||
}
|
||||
for gap in files[:shown]
|
||||
]
|
||||
doc = {
|
||||
"format": fmt,
|
||||
"total": _totals(files),
|
||||
"shown": len(entries),
|
||||
"files": entries,
|
||||
}
|
||||
return json.dumps(doc, indent=2)
|
||||
|
||||
|
||||
def to_markdown(fmt: str, files: Sequence[FileGap], shown: int) -> str:
|
||||
"""Markdown task list meant to be pasted to an agent verbatim."""
|
||||
total = _totals(files)
|
||||
out = [
|
||||
"# Coverage gaps",
|
||||
"",
|
||||
(
|
||||
f"Format: {fmt}. Total: {total['files']} files with gaps, "
|
||||
f"{total['missed_lines']} missed lines, "
|
||||
f"{total['missed_branches']} missed branches."
|
||||
),
|
||||
]
|
||||
if shown < len(files):
|
||||
out.append(f"Showing the {shown} worst files.")
|
||||
if not files:
|
||||
return "\n".join(out) + "\n\nNo coverage gaps.\n"
|
||||
out += ["", "Write tests that cover every item below, then re-run the report.", ""]
|
||||
for rank, gap in enumerate(files[:shown], start=1):
|
||||
out.append(f"## {rank}. `{gap.path}` - {gap.percent:.1f}% covered")
|
||||
if gap.missed_lines:
|
||||
out.append(
|
||||
f"- missed lines ({len(gap.missed_lines)}): "
|
||||
f"{compact_ranges(gap.missed_lines)}"
|
||||
)
|
||||
if gap.missed_branches:
|
||||
out.append(
|
||||
f"- missed branches ({gap.missed_branches}) at lines: "
|
||||
f"{compact_ranges(gap.missed_branch_lines)}"
|
||||
)
|
||||
out.append("")
|
||||
return "\n".join(out).rstrip() + "\n"
|
||||
@ -1,89 +0,0 @@
|
||||
"""Small fixture reports, one per supported format, all describing a.py/b.py.
|
||||
|
||||
a.py: lines 1-5 with 2,3 and 5 missed + one missed branch on line 4.
|
||||
b.py: fully covered (must never be listed).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
LCOV = """TN:
|
||||
SF:a.py
|
||||
DA:1,1
|
||||
DA:2,0
|
||||
DA:3,0
|
||||
DA:4,1
|
||||
DA:5,0
|
||||
BRDA:4,0,0,1
|
||||
BRDA:4,0,1,-
|
||||
end_of_record
|
||||
SF:b.py
|
||||
DA:1,3
|
||||
end_of_record
|
||||
SF:a.py
|
||||
DA:1,1
|
||||
DA:2,0
|
||||
end_of_record
|
||||
"""
|
||||
|
||||
COBERTURA = """<?xml version="1.0" ?>
|
||||
<coverage version="7"><packages><package name="p"><classes>
|
||||
<class filename="a.py"><lines>
|
||||
<line number="1" hits="1"/><line number="2" hits="0"/><line number="3" hits="0"/>
|
||||
<line number="4" hits="1" branch="true" condition-coverage="50% (1/2)"/>
|
||||
<line number="5" hits="0"/></lines></class>
|
||||
<class filename="b.py"><lines><line number="1" hits="2"/></lines></class>
|
||||
<class filename="a.py"><lines><line number="1" hits="1"/></lines></class>
|
||||
</classes></package></packages></coverage>
|
||||
"""
|
||||
|
||||
JACOCO = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE report PUBLIC "-//JACOCO//DTD Report 1.1//EN" "report.dtd">
|
||||
<report name="x"><package name="com/x">
|
||||
<sourcefile name="A.kt">
|
||||
<line nr="1" mi="0" ci="3"/><line nr="2" mi="2" ci="0"/><line nr="3" mi="1" ci="0"/>
|
||||
<line nr="4" mi="0" ci="4" mb="1" cb="1"/><line nr="5" mi="1" ci="0"/></sourcefile>
|
||||
<sourcefile name="B.kt"><line nr="1" mi="0" ci="1"/></sourcefile>
|
||||
</package><package name=""><sourcefile name="Top.kt">
|
||||
<line nr="1" mi="0" ci="1"/></sourcefile></package></report>
|
||||
"""
|
||||
|
||||
COVERAGE_JSON = json.dumps(
|
||||
{
|
||||
"files": {
|
||||
"a.py": {
|
||||
"executed_lines": [1, 4],
|
||||
"missing_lines": [2, 3, 5],
|
||||
"missing_branches": [[4, 5]],
|
||||
"summary": {"covered_branches": 1},
|
||||
},
|
||||
"b.py": {
|
||||
"executed_lines": [1],
|
||||
"missing_lines": [],
|
||||
"missing_branches": [],
|
||||
},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def reports(tmp_path: Path) -> dict[str, Path]:
|
||||
"""Write each fixture report to disk, keyed by format name."""
|
||||
out = {}
|
||||
for name, body in {
|
||||
"lcov": LCOV,
|
||||
"cobertura": COBERTURA,
|
||||
"jacoco": JACOCO,
|
||||
"json": COVERAGE_JSON,
|
||||
}.items():
|
||||
out[name] = tmp_path / f"{name}.rpt"
|
||||
out[name].write_text(body, encoding="utf-8")
|
||||
return out
|
||||
@ -1,73 +0,0 @@
|
||||
"""Parser and format-detection tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from coverage_gaps.models import FileGap, ReportError
|
||||
from coverage_gaps.parsers import detect_format, load_report
|
||||
|
||||
|
||||
def _by_path(gaps: list[FileGap]) -> dict[str, FileGap]:
|
||||
return {gap.path: gap for gap in gaps}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fmt", ["lcov", "cobertura", "json"])
|
||||
def test_a_py_gaps_match_across_formats(reports: dict[str, Path], fmt: str) -> None:
|
||||
detected, gaps = load_report(reports[fmt])
|
||||
assert detected == fmt
|
||||
a = _by_path(gaps)["a.py"]
|
||||
assert a.missed_lines == (2, 3, 5)
|
||||
assert a.missed_branch_lines == (4,)
|
||||
assert a.missed_branches == 1
|
||||
assert a.total_lines == 5
|
||||
assert a.total_branches == 2
|
||||
assert _by_path(gaps)["b.py"].missed == 0
|
||||
|
||||
|
||||
def test_jacoco(reports: dict[str, Path]) -> None:
|
||||
fmt, gaps = load_report(reports["jacoco"])
|
||||
assert fmt == "jacoco"
|
||||
by = _by_path(gaps)
|
||||
assert by["com/x/A.kt"].missed_lines == (2, 3, 5)
|
||||
assert by["com/x/A.kt"].missed_branch_lines == (4,)
|
||||
assert by["com/x/A.kt"].total_branches == 2
|
||||
assert "Top.kt" in by
|
||||
|
||||
|
||||
def test_percent_and_empty_file() -> None:
|
||||
gap = FileGap("x", 4, (1,), 4, (2,), 1)
|
||||
assert gap.percent == 75.0
|
||||
assert FileGap("y", 0, (), 0, (), 0).percent == 100.0
|
||||
|
||||
|
||||
def test_lcov_ignores_records_outside_a_file_and_dash_branch_ids() -> None:
|
||||
from coverage_gaps.parsers import parse_lcov
|
||||
|
||||
gaps = parse_lcov(
|
||||
"DA:1,0\nBRDA:1,0,0,0\nSF:z\nBRDA:2,0,-,0\nend_of_record\nDA:9,0\n"
|
||||
)
|
||||
assert len(gaps) == 1
|
||||
assert gaps[0].missed_lines == ()
|
||||
assert gaps[0].missed_branch_lines == (2,)
|
||||
|
||||
|
||||
def test_detect_unknown_and_xml_without_known_root() -> None:
|
||||
for text in ("hello", "<other/>", ""):
|
||||
with pytest.raises(ReportError):
|
||||
detect_format(text)
|
||||
|
||||
|
||||
def test_load_errors(tmp_path: Path) -> None:
|
||||
with pytest.raises(ReportError, match="cannot read"):
|
||||
load_report(tmp_path / "missing")
|
||||
bad_xml = tmp_path / "bad.xml"
|
||||
bad_xml.write_text("<coverage><oops", encoding="utf-8")
|
||||
with pytest.raises(ReportError, match="malformed cobertura"):
|
||||
load_report(bad_xml)
|
||||
bad_json = tmp_path / "bad.json"
|
||||
bad_json.write_text("{nope", encoding="utf-8")
|
||||
with pytest.raises(ReportError, match="malformed json"):
|
||||
load_report(bad_json)
|
||||
@ -1,97 +0,0 @@
|
||||
"""Rendering, filtering and CLI exit-code tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from coverage_gaps.cli import main
|
||||
from coverage_gaps.models import FileGap
|
||||
from coverage_gaps.render import compact_ranges, select, to_markdown
|
||||
|
||||
|
||||
def test_compact_ranges() -> None:
|
||||
assert compact_ranges([]) == ""
|
||||
assert compact_ranges([40, 12, 13, 14, 15, 16, 17, 18, 18]) == "12-18, 40"
|
||||
assert compact_ranges([1, 3, 5, 6]) == "1, 3, 5-6"
|
||||
|
||||
|
||||
def test_select_sorts_and_filters() -> None:
|
||||
small = FileGap("src/b.py", 5, (1,), 0, (), 0)
|
||||
big = FileGap("lib/a.py", 5, (1, 2), 2, (3,), 1)
|
||||
clean = FileGap("src/c.py", 5, (), 0, (), 0)
|
||||
assert [g.path for g in select([small, clean, big])] == ["lib/a.py", "src/b.py"]
|
||||
assert [g.path for g in select([small, big], path_prefix="src")] == ["src/b.py"]
|
||||
assert [g.path for g in select([small, big], min_missed=2)] == ["lib/a.py"]
|
||||
|
||||
|
||||
def test_markdown_branch_only_gap() -> None:
|
||||
out = to_markdown("lcov", [FileGap("z.py", 2, (), 2, (7,), 1)], 1)
|
||||
assert "- missed lines" not in out
|
||||
assert "missed branches (1) at lines: 7" in out
|
||||
|
||||
|
||||
def test_module_entrypoint(
|
||||
reports: dict[str, Path], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
import runpy
|
||||
|
||||
monkeypatch.setattr("sys.argv", ["coverage_gaps", str(reports["lcov"])])
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
runpy.run_module("coverage_gaps", run_name="__main__")
|
||||
assert exc.value.code == 1
|
||||
|
||||
|
||||
def test_markdown_no_gaps() -> None:
|
||||
assert "No coverage gaps." in to_markdown("lcov", [], 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fmt", ["lcov", "cobertura", "jacoco", "json"])
|
||||
def test_cli_markdown_exit_1(
|
||||
reports: dict[str, Path], fmt: str, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
assert main([str(reports[fmt])]) == 1
|
||||
out = capsys.readouterr().out
|
||||
assert "2-3, 5" in out
|
||||
assert "missed branches (1) at lines: 4" in out
|
||||
assert "b.py" not in out.replace("A.kt", "")
|
||||
|
||||
|
||||
def test_cli_json_and_max_files(
|
||||
reports: dict[str, Path], capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
assert main([str(reports["jacoco"]), "--json", "--max-files", "1"]) == 1
|
||||
doc = json.loads(capsys.readouterr().out)
|
||||
assert doc["shown"] == 1
|
||||
assert doc["total"] == {"files": 1, "missed_lines": 3, "missed_branches": 1}
|
||||
assert doc["files"][0]["missed_lines"] == "2-3, 5"
|
||||
|
||||
|
||||
def test_cli_markdown_truncation_note(
|
||||
reports: dict[str, Path], capsys: pytest.CaptureFixture[str], tmp_path: Path
|
||||
) -> None:
|
||||
two = tmp_path / "two.info"
|
||||
two.write_text("SF:x\nDA:1,0\nend_of_record\nSF:y\nDA:1,0\nDA:2,0\nend_of_record\n")
|
||||
assert main([str(two), "--max-files", "1"]) == 1
|
||||
out = capsys.readouterr().out
|
||||
assert "Showing the 1 worst files." in out
|
||||
assert "`y`" in out
|
||||
assert "`x`" not in out
|
||||
|
||||
|
||||
def test_cli_exit_0_when_filtered_clean(
|
||||
reports: dict[str, Path], capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
assert main([str(reports["lcov"]), "--path-prefix", "b."]) == 0
|
||||
assert main([str(reports["lcov"]), "--json", "--min-missed", "99"]) == 0
|
||||
assert "No coverage gaps." in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_cli_exit_2(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
||||
assert main([str(tmp_path / "nope")]) == 2
|
||||
junk = tmp_path / "junk"
|
||||
junk.write_text("junk")
|
||||
assert main([str(junk)]) == 2
|
||||
assert "coverage-gaps:" in capsys.readouterr().err
|
||||
19
crdt-sync/.github/workflows/pre-commit.yml
vendored
Normal file
19
crdt-sync/.github/workflows/pre-commit.yml
vendored
Normal file
@ -0,0 +1,19 @@
|
||||
name: pre-commit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt
|
||||
- uses: pre-commit/action@v3.0.1
|
||||
28
crdt-sync/.github/workflows/python-tests.yml
vendored
Normal file
28
crdt-sync/.github/workflows/python-tests.yml
vendored
Normal file
@ -0,0 +1,28 @@
|
||||
name: Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: python -m pytest -v
|
||||
12
crdt-sync/.gitignore
vendored
Normal file
12
crdt-sync/.gitignore
vendored
Normal file
@ -0,0 +1,12 @@
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.venv/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
coverage.lcov
|
||||
htmlcov/
|
||||
dist/
|
||||
build/
|
||||
*.egg-info/
|
||||
@ -3,14 +3,6 @@
|
||||
# Run: pre-commit run --all-files
|
||||
# Update: pre-commit autoupdate
|
||||
|
||||
# NOTE: this subproject lives inside the ~/src/utils monorepo, whose git root is
|
||||
# one level up. pre-commit always runs hooks from the git root, so every path
|
||||
# below is written relative to THAT, not to this directory, and the top-level
|
||||
# `files` key keeps these hooks off the sibling subprojects. CI invokes this
|
||||
# config exactly the same way:
|
||||
# pre-commit run --config crdt-sync/.pre-commit-config.yaml --all-files
|
||||
files: ^crdt-sync/
|
||||
|
||||
default_language_version:
|
||||
python: python3
|
||||
default_stages: [pre-commit]
|
||||
@ -40,19 +32,6 @@ repos:
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: file-length-cap
|
||||
name: file length <= 250 lines
|
||||
entry: bash scripts/check_file_length.sh
|
||||
language: system
|
||||
pass_filenames: true
|
||||
- id: subproject-scope
|
||||
name: subproject scope regex still matches
|
||||
entry: scripts/check_subproject_scope.sh crdt-sync
|
||||
language: script
|
||||
# always_run so the top-level `files` filter cannot skip the one hook
|
||||
# whose job is to notice that filter has stopped matching anything.
|
||||
always_run: true
|
||||
pass_filenames: false
|
||||
- id: no-noqa
|
||||
name: Block noqa comments
|
||||
entry: '(?i)#\s*(noqa|type:\s*ignore)'
|
||||
@ -65,7 +44,7 @@ repos:
|
||||
types: [python]
|
||||
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.17.2
|
||||
rev: v0.15.2
|
||||
hooks:
|
||||
- id: ruff
|
||||
args: [--fix, --unsafe-fixes, --exit-non-zero-on-fix, --show-fixes]
|
||||
@ -74,7 +53,7 @@ repos:
|
||||
types_or: [python, pyi]
|
||||
|
||||
- repo: https://github.com/pre-commit/mirrors-mypy
|
||||
rev: v2.4.0
|
||||
rev: v1.13.0
|
||||
hooks:
|
||||
- id: mypy
|
||||
additional_dependencies:
|
||||
@ -103,36 +82,36 @@ repos:
|
||||
- --disable-error-code=list-item
|
||||
|
||||
- repo: https://github.com/pylint-dev/pylint
|
||||
rev: v4.0.8
|
||||
rev: v3.3.2
|
||||
hooks:
|
||||
- id: pylint
|
||||
args:
|
||||
- --rcfile=crdt-sync/pyproject.toml
|
||||
- --fail-under=10
|
||||
- --rcfile=pyproject.toml
|
||||
- --fail-under=8.0
|
||||
- --jobs=4
|
||||
additional_dependencies:
|
||||
- pytest
|
||||
- requests
|
||||
|
||||
- repo: https://github.com/PyCQA/bandit
|
||||
rev: 1.9.4
|
||||
rev: 1.7.10
|
||||
hooks:
|
||||
- id: bandit
|
||||
args:
|
||||
- -c
|
||||
- crdt-sync/pyproject.toml
|
||||
- pyproject.toml
|
||||
- --severity-level=high
|
||||
- --confidence-level=medium
|
||||
- --skip=B113
|
||||
additional_dependencies: ["bandit[toml]"]
|
||||
exclude: ^(crdt-sync/crdt_sync/tests/|.*test.*\.py$)
|
||||
exclude: ^(tests/|.*test.*\.py$)
|
||||
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: pytest-coverage
|
||||
name: pytest with coverage enforcement
|
||||
entry: scripts/run_subproject_tests.sh crdt-sync
|
||||
language: script
|
||||
entry: python -m pytest
|
||||
language: system
|
||||
types: [python]
|
||||
pass_filenames: false
|
||||
require_serial: true
|
||||
|
||||
@ -1,98 +1,28 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Shared CRDT merge scheme + pluggable remote sync transports."""
|
||||
"""Shared CRDT merge scheme + GitHub-Contents-API sync transport."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from crdt_sync._config import (
|
||||
CONFIG_DIR,
|
||||
CONFIG_FILE,
|
||||
PASSWORD_FILE,
|
||||
ConfigError,
|
||||
FirebaseConfig,
|
||||
credential_store_for,
|
||||
firebase_client_for,
|
||||
mirror_client_for,
|
||||
)
|
||||
from crdt_sync._device import DeviceIdentity, load_device_identity
|
||||
from crdt_sync._firebase import (
|
||||
DatabaseNotFoundError,
|
||||
FirebaseSyncClient,
|
||||
FirebaseSyncError,
|
||||
)
|
||||
from crdt_sync._firebase_auth import (
|
||||
CredentialStore,
|
||||
FileCredentialStore,
|
||||
FirebaseAuthError,
|
||||
FirebaseCredentials,
|
||||
FirebaseTokenProvider,
|
||||
MemoryCredentialStore,
|
||||
)
|
||||
from crdt_sync._github import GitHubSyncClient, GitHubSyncError, RepoNotFoundError
|
||||
from crdt_sync._hlc import Hlc
|
||||
from crdt_sync._log import Log, merge_logs
|
||||
from crdt_sync._mirror import MirrorSyncClient
|
||||
from crdt_sync._record import Field, Record, merge_field, merge_record
|
||||
from crdt_sync._remote import RemoteNotFoundError, RemoteStore, RemoteSyncError
|
||||
from crdt_sync._store import dump_log, load_log, read_log, write_log
|
||||
from crdt_sync._sync import (
|
||||
FileSyncStateStore,
|
||||
LogCodec,
|
||||
MemorySyncStateStore,
|
||||
RevisionTracking,
|
||||
SyncState,
|
||||
SyncStateStore,
|
||||
SyncTarget,
|
||||
default_revs_path,
|
||||
revision_of,
|
||||
sync_log,
|
||||
)
|
||||
from crdt_sync._sync import sync_log
|
||||
|
||||
__all__ = [
|
||||
"CONFIG_DIR",
|
||||
"CONFIG_FILE",
|
||||
"PASSWORD_FILE",
|
||||
"ConfigError",
|
||||
"CredentialStore",
|
||||
"DatabaseNotFoundError",
|
||||
"DeviceIdentity",
|
||||
"Field",
|
||||
"FileCredentialStore",
|
||||
"FileSyncStateStore",
|
||||
"FirebaseAuthError",
|
||||
"FirebaseConfig",
|
||||
"FirebaseCredentials",
|
||||
"FirebaseSyncClient",
|
||||
"FirebaseSyncError",
|
||||
"FirebaseTokenProvider",
|
||||
"GitHubSyncClient",
|
||||
"GitHubSyncError",
|
||||
"Hlc",
|
||||
"Log",
|
||||
"LogCodec",
|
||||
"MemoryCredentialStore",
|
||||
"MemorySyncStateStore",
|
||||
"MirrorSyncClient",
|
||||
"Record",
|
||||
"RemoteNotFoundError",
|
||||
"RemoteStore",
|
||||
"RemoteSyncError",
|
||||
"RepoNotFoundError",
|
||||
"RevisionTracking",
|
||||
"SyncState",
|
||||
"SyncStateStore",
|
||||
"SyncTarget",
|
||||
"credential_store_for",
|
||||
"default_revs_path",
|
||||
"dump_log",
|
||||
"firebase_client_for",
|
||||
"load_device_identity",
|
||||
"load_log",
|
||||
"merge_field",
|
||||
"merge_logs",
|
||||
"merge_record",
|
||||
"mirror_client_for",
|
||||
"read_log",
|
||||
"revision_of",
|
||||
"sync_log",
|
||||
"write_log",
|
||||
]
|
||||
|
||||
@ -1,71 +0,0 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Turning a Firebase auth response into a message, and reading its verdict.
|
||||
|
||||
Split from :mod:`crdt_sync._firebase_auth`, which keeps the token provider.
|
||||
identitytoolkit reports failures as a nested JSON error whose useful part is
|
||||
buried; these turn that into something a log line can carry, and decide
|
||||
whether the failure is permanent (the refresh token is dead and the user must
|
||||
sign in again) or worth retrying.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from crdt_sync._remote import RemoteSyncError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import requests as _requests
|
||||
|
||||
|
||||
class FirebaseAuthError(RemoteSyncError):
|
||||
"""Raised for an authentication failure the caller must not ignore.
|
||||
|
||||
A :class:`crdt_sync.RemoteSyncError` so callers that only care about
|
||||
"sync is broken" can catch one type, while a settings screen can single
|
||||
this out to say "your password is wrong" rather than "the network is
|
||||
down".
|
||||
"""
|
||||
|
||||
|
||||
def _reason(response: _requests.Response) -> str:
|
||||
"""Return Google's machine-readable error reason, in parentheses.
|
||||
|
||||
Pulls ``INVALID_PASSWORD`` / ``TOKEN_EXPIRED`` / ``USER_DISABLED`` out of
|
||||
the body so the raised message says what actually went wrong rather than
|
||||
just reporting a status code.
|
||||
"""
|
||||
try:
|
||||
data = response.json()
|
||||
except ValueError:
|
||||
# A non-JSON body (a proxy error page, say): the status code is all
|
||||
# the detail there is.
|
||||
return ""
|
||||
if not isinstance(data, dict):
|
||||
return ""
|
||||
error = data.get("error")
|
||||
if isinstance(error, dict):
|
||||
return f" ({error.get('message')})"
|
||||
if isinstance(error, str):
|
||||
return f" ({error})"
|
||||
return ""
|
||||
|
||||
|
||||
#: Reasons that mean the refresh token is permanently dead. Only these clear
|
||||
#: the session: a network error or a 5xx must not, because a device that signs
|
||||
#: itself out whenever the wifi drops needs a manual sign-in to recover, which
|
||||
#: is a worse failure than the stale banner this guards against.
|
||||
_TERMINAL_AUTH_REASONS = (
|
||||
"TOKEN_EXPIRED",
|
||||
"USER_DISABLED",
|
||||
"USER_NOT_FOUND",
|
||||
"INVALID_REFRESH_TOKEN",
|
||||
"INVALID_GRANT_TYPE",
|
||||
"MISSING_REFRESH_TOKEN",
|
||||
)
|
||||
|
||||
|
||||
def _is_revoked(error: FirebaseAuthError) -> bool:
|
||||
"""Return whether ``error`` means the refresh token is permanently dead."""
|
||||
message = str(error)
|
||||
return any(reason in message for reason in _TERMINAL_AUTH_REASONS)
|
||||
@ -1,199 +0,0 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""The one place every app learns how to reach Firebase.
|
||||
|
||||
Before this module each caller re-read ``~/.config/crdt-sync/firebase.json``
|
||||
by hand, so a change to the schema meant editing every consumer, and each
|
||||
consumer got to invent its own half of the credential-cache convention. The
|
||||
apps share one Firebase project and one account, so they should share one
|
||||
loader.
|
||||
|
||||
Two things live here:
|
||||
|
||||
* :class:`FirebaseConfig` -- the parsed, validated config file.
|
||||
* :func:`firebase_client_for` -- the whole "config file to usable client"
|
||||
path, including the per-app credential cache that keeps a fresh process
|
||||
from re-authenticating. ``wake_alarm``'s PC side runs every minute and
|
||||
``diet_guard``'s every 15, so caching the refresh token is what makes the
|
||||
sign-in cost negligible rather than constant.
|
||||
|
||||
Deliberately *not* here: any choice about which backend an app uses. That
|
||||
stays a constructor change at the call site, per :mod:`crdt_sync._remote`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from crdt_sync._firebase import _DEFAULT_TIMEOUT_SECONDS, FirebaseSyncClient
|
||||
from crdt_sync._firebase_auth import FileCredentialStore, FirebaseTokenProvider
|
||||
from crdt_sync._mirror import MirrorSyncClient
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from crdt_sync._remote import RemoteStore
|
||||
|
||||
CONFIG_DIR = Path.home() / ".config" / "crdt-sync"
|
||||
CONFIG_FILE = CONFIG_DIR / "firebase.json"
|
||||
PASSWORD_FILE = CONFIG_DIR / "password"
|
||||
|
||||
# The scaffold ships "_comment_*" keys explaining where each value comes
|
||||
# from. They are documentation for a human filling the file in, not config.
|
||||
_COMMENT_PREFIX = "_comment"
|
||||
|
||||
|
||||
class ConfigError(Exception):
|
||||
"""The shared Firebase config is missing, malformed or unfilled."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FirebaseConfig:
|
||||
"""Everything needed to reach the shared Firebase project.
|
||||
|
||||
``api_key`` is not a secret -- it ships inside the Android APKs, and the
|
||||
security rules are what actually protect the data. The password is.
|
||||
"""
|
||||
|
||||
api_key: str
|
||||
database_url: str
|
||||
project_id: str
|
||||
uid: str
|
||||
email: str
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path | None = None) -> FirebaseConfig:
|
||||
"""Return the config at ``path``, defaulting to :data:`CONFIG_FILE`.
|
||||
|
||||
Raises:
|
||||
ConfigError: If the file is absent, unparsable, missing a key, or
|
||||
still holding a scaffold placeholder -- each reported with the
|
||||
specific field at fault, because the alternative is a
|
||||
confusing auth failure much later.
|
||||
"""
|
||||
source = path if path is not None else CONFIG_FILE
|
||||
try:
|
||||
raw = json.loads(source.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError as exc:
|
||||
msg = f"{source} does not exist; see utils/firebase/README.md"
|
||||
raise ConfigError(msg) from exc
|
||||
except OSError as exc:
|
||||
msg = f"{source} could not be read: {exc}"
|
||||
raise ConfigError(msg) from exc
|
||||
except ValueError as exc:
|
||||
msg = f"{source} is not valid JSON: {exc}"
|
||||
raise ConfigError(msg) from exc
|
||||
|
||||
if not isinstance(raw, dict):
|
||||
msg = f"{source} must contain a JSON object"
|
||||
raise ConfigError(msg)
|
||||
|
||||
data = {k: v for k, v in raw.items() if not k.startswith(_COMMENT_PREFIX)}
|
||||
fields = {
|
||||
"api_key": "apiKey",
|
||||
"database_url": "databaseUrl",
|
||||
"project_id": "projectId",
|
||||
"uid": "uid",
|
||||
"email": "email",
|
||||
}
|
||||
missing = [key for key in fields.values() if not str(data.get(key, "")).strip()]
|
||||
if missing:
|
||||
msg = f"{source} is missing or has empty: {', '.join(missing)}"
|
||||
raise ConfigError(msg)
|
||||
|
||||
unfilled = [key for key in fields.values() if "PASTE_" in str(data[key])]
|
||||
if unfilled:
|
||||
msg = f"{source} still holds the placeholder for: {', '.join(unfilled)}"
|
||||
raise ConfigError(msg)
|
||||
|
||||
return cls(**{attr: str(data[key]) for attr, key in fields.items()})
|
||||
|
||||
def read_password(self, path: Path | None = None) -> str:
|
||||
"""Return the sync account's password.
|
||||
|
||||
Stripped, because a text editor that appends a trailing newline would
|
||||
otherwise turn into an authentication failure with no visible cause.
|
||||
"""
|
||||
source = path if path is not None else PASSWORD_FILE
|
||||
try:
|
||||
password = source.read_text(encoding="utf-8").strip()
|
||||
except OSError as exc:
|
||||
msg = f"{source} could not be read: {exc}"
|
||||
raise ConfigError(msg) from exc
|
||||
if not password or "PASTE_" in password:
|
||||
msg = f"{source} is empty or still holds the scaffold placeholder"
|
||||
raise ConfigError(msg)
|
||||
return password
|
||||
|
||||
|
||||
def credential_store_for(app_name: str) -> FileCredentialStore:
|
||||
"""Return the credential cache for ``app_name``.
|
||||
|
||||
Per app rather than shared: two apps signing in as the same account still
|
||||
refresh tokens on their own schedules, and a single file written by
|
||||
several processes at once is a corruption waiting to happen.
|
||||
"""
|
||||
return FileCredentialStore(
|
||||
Path.home() / ".config" / app_name / "firebase_auth.json"
|
||||
)
|
||||
|
||||
|
||||
def firebase_client_for(
|
||||
app_name: str,
|
||||
*,
|
||||
config: FirebaseConfig | None = None,
|
||||
timeout_seconds: float = _DEFAULT_TIMEOUT_SECONDS,
|
||||
) -> FirebaseSyncClient:
|
||||
"""Return a signed-in client for ``app_name``.
|
||||
|
||||
Reuses the cached refresh token when there is one and signs in with the
|
||||
password only when there is not, so the common path costs no
|
||||
authentication round trip at all.
|
||||
|
||||
Args:
|
||||
app_name: Names the credential cache, e.g. ``"diet_guard"``.
|
||||
config: Overrides the on-disk config; for tests.
|
||||
timeout_seconds: Per-request network timeout. Without this the
|
||||
client fell back to its own default and a caller's tighter
|
||||
budget reached the GitHub backend only.
|
||||
|
||||
Raises:
|
||||
ConfigError: If the shared config is unusable.
|
||||
FirebaseAuthError: If the credentials are rejected.
|
||||
"""
|
||||
resolved = config if config is not None else FirebaseConfig.load()
|
||||
auth = FirebaseTokenProvider(resolved.api_key, credential_store_for(app_name))
|
||||
if not auth.has_session():
|
||||
auth.sign_in(resolved.email, resolved.read_password())
|
||||
return FirebaseSyncClient(
|
||||
resolved.database_url, auth, timeout_seconds=timeout_seconds
|
||||
)
|
||||
|
||||
|
||||
def mirror_client_for(
|
||||
app_name: str,
|
||||
github_client: RemoteStore,
|
||||
*,
|
||||
config: FirebaseConfig | None = None,
|
||||
) -> MirrorSyncClient:
|
||||
"""Return a Firebase-primary client that still mirrors to ``github_client``.
|
||||
|
||||
This is what an app uses *during* the cutover. Firebase is authoritative,
|
||||
so its failures fail the tick; GitHub is kept in step but never allowed
|
||||
to. Reads union both, which is what lets devices cut over one at a time
|
||||
without a migrated PC losing sight of an un-migrated phone.
|
||||
|
||||
Rolling back is deleting the call: pass the GitHub client straight to
|
||||
:func:`crdt_sync.sync_log` again. Retiring the mirror is swapping this for
|
||||
:func:`firebase_client_for`. Both are one-line changes at the call site,
|
||||
which is the whole point of the :class:`~crdt_sync.RemoteStore` seam.
|
||||
|
||||
Args:
|
||||
app_name: Names the credential cache, e.g. ``"diet_guard"``.
|
||||
github_client: The existing GitHub client, kept as the fallback.
|
||||
config: Overrides the on-disk config; for tests.
|
||||
"""
|
||||
return MirrorSyncClient(
|
||||
primary=firebase_client_for(app_name, config=config),
|
||||
mirror=github_client,
|
||||
)
|
||||
@ -1,158 +0,0 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Firebase credentials and the stores that persist them.
|
||||
|
||||
Split from :mod:`crdt_sync._firebase_auth`, which keeps the token provider.
|
||||
A credential set is an ID token, a refresh token and an expiry; the stores
|
||||
decide where it survives a restart -- memory for tests, a mode-0600 file for
|
||||
real installs.
|
||||
|
||||
Re-exported from :mod:`crdt_sync._firebase_auth`, so existing imports work.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import datetime as dt
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
from typing import Protocol
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
# Refresh this long before the token actually expires. An ID token lives ~1h;
|
||||
# refreshing early means a tick that starts just under the wire still finishes
|
||||
# with a valid token rather than 401ing halfway through a multi-file push.
|
||||
_REFRESH_SKEW = dt.timedelta(minutes=5)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FirebaseCredentials:
|
||||
"""One device's session: a short-lived ID token plus its refresh token.
|
||||
|
||||
Attributes:
|
||||
----------
|
||||
id_token:
|
||||
The bearer credential for Realtime Database requests. Expires fast.
|
||||
refresh_token:
|
||||
The long-lived credential. **This is the secret worth protecting.**
|
||||
expires_at:
|
||||
When ``id_token`` stops being accepted, in UTC.
|
||||
"""
|
||||
|
||||
id_token: str
|
||||
refresh_token: str
|
||||
expires_at: dt.datetime
|
||||
|
||||
def is_expired_at(self, now: dt.datetime) -> bool:
|
||||
"""Return whether the token is expired, or close enough to it."""
|
||||
return now + _REFRESH_SKEW >= self.expires_at
|
||||
|
||||
def to_json(self) -> dict[str, str]:
|
||||
"""Return a JSON-serializable form."""
|
||||
return {
|
||||
"id_token": self.id_token,
|
||||
"refresh_token": self.refresh_token,
|
||||
"expires_at": self.expires_at.isoformat(),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, data: dict[str, str]) -> FirebaseCredentials:
|
||||
"""Rebuild credentials from :meth:`to_json` output."""
|
||||
return cls(
|
||||
id_token=data["id_token"],
|
||||
refresh_token=data["refresh_token"],
|
||||
expires_at=dt.datetime.fromisoformat(data["expires_at"]),
|
||||
)
|
||||
|
||||
|
||||
class CredentialStore(Protocol):
|
||||
"""Where a device keeps its credentials between runs."""
|
||||
|
||||
def load(self) -> FirebaseCredentials | None:
|
||||
"""Return the stored credentials, or ``None`` if not signed in."""
|
||||
|
||||
def save(self, credentials: FirebaseCredentials) -> None:
|
||||
"""Persist ``credentials``."""
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Forget any stored credentials."""
|
||||
|
||||
|
||||
class MemoryCredentialStore:
|
||||
"""A :class:`CredentialStore` for tests and one-shot scripts."""
|
||||
|
||||
def __init__(self, credentials: FirebaseCredentials | None = None) -> None:
|
||||
"""Start holding ``credentials`` (or nothing)."""
|
||||
self._credentials = credentials
|
||||
|
||||
def load(self) -> FirebaseCredentials | None:
|
||||
"""Return the held credentials."""
|
||||
return self._credentials
|
||||
|
||||
def save(self, credentials: FirebaseCredentials) -> None:
|
||||
"""Replace the held credentials."""
|
||||
self._credentials = credentials
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Drop the held credentials."""
|
||||
self._credentials = None
|
||||
|
||||
|
||||
class FileCredentialStore:
|
||||
"""A :class:`CredentialStore` backed by a ``0600`` JSON file.
|
||||
|
||||
Persistence is not an optimisation here: ``wake_alarm``'s PC side is a
|
||||
fresh process every minute and ``diet_guard``'s every 15 minutes, so an
|
||||
in-memory store would re-authenticate thousands of times a day.
|
||||
"""
|
||||
|
||||
def __init__(self, path: Path) -> None:
|
||||
"""Store credentials at ``path``, creating parent dirs as needed."""
|
||||
self._path = path
|
||||
|
||||
def load(self) -> FirebaseCredentials | None:
|
||||
"""Return the stored credentials, or ``None`` if the file is absent.
|
||||
|
||||
A corrupt or truncated file (an interrupted write) reads as "not
|
||||
signed in" rather than raising: the caller's next step is to sign in
|
||||
again, which repairs it.
|
||||
"""
|
||||
try:
|
||||
data = json.loads(self._path.read_text(encoding="utf-8"))
|
||||
except OSError, ValueError:
|
||||
return None
|
||||
try:
|
||||
return FirebaseCredentials.from_json(data)
|
||||
except KeyError, TypeError, ValueError:
|
||||
return None
|
||||
|
||||
def save(self, credentials: FirebaseCredentials) -> None:
|
||||
"""Write ``credentials`` atomically, readable only by this user."""
|
||||
self._path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# A unique temp name per write, not a fixed ``.tmp``: two processes
|
||||
# (a long-running stream and a periodic sync) share one cache, and a
|
||||
# fixed name let one rename the other's half-written file into place.
|
||||
# mkstemp creates it 0600 from the outset -- writing then chmod'ing
|
||||
# would leave the refresh token world-readable for the gap between.
|
||||
fd, name = tempfile.mkstemp(
|
||||
dir=self._path.parent, prefix=f".{self._path.name}.", suffix=".tmp"
|
||||
)
|
||||
temp = Path(name)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(credentials.to_json()))
|
||||
temp.replace(self._path)
|
||||
except BaseException:
|
||||
temp.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Delete the credentials file if it exists."""
|
||||
self._path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _utcnow() -> dt.datetime:
|
||||
return dt.datetime.now(dt.UTC)
|
||||
@ -1,88 +0,0 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""This device's stable sync identity.
|
||||
|
||||
Every device that pushes into a namespace needs an id that is *its own*: it
|
||||
is the directory segment its log is pushed under
|
||||
(``<path_prefix>/<device_id>/<filename>``), the revision key, and the node
|
||||
component baked into every HLC stamp it writes (``<iso>-<counter>-<id>``).
|
||||
|
||||
Two devices sharing an id overwrite each other's pushed file on every tick,
|
||||
so the id must be per-*install*, not per-role: a fixed ``"pc"``/``"phone"``
|
||||
constant silently collides the moment a second machine takes the same role,
|
||||
and a reinstall inherits the previous install's CRDT identity.
|
||||
|
||||
A migration from a role constant to a persisted uuid cannot rewrite history.
|
||||
Stamps already written keep the old id, and the old path keeps the log that
|
||||
was pushed under it -- so a device that has just switched ids must still
|
||||
recognise its *former* id as itself. That is what [legacy_id] carries, and
|
||||
why [DeviceIdentity.own_ids] exists: skip-own-writes has to test membership
|
||||
of that set rather than equality with one id, or the device re-downloads and
|
||||
re-merges its own pre-migration history as though a peer had written it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
import uuid
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DeviceIdentity:
|
||||
"""This device's current sync id, plus the id it used to push under.
|
||||
|
||||
Attributes:
|
||||
device_id: The id to push, stamp and key revisions under from now on.
|
||||
legacy_id: The id this device pushed under before the migration to
|
||||
persisted uuids, or ``None`` once its old path has been reclaimed.
|
||||
"""
|
||||
|
||||
device_id: str
|
||||
legacy_id: str | None = None
|
||||
|
||||
@property
|
||||
def own_ids(self) -> frozenset[str]:
|
||||
"""Every id that means "this device", for skip-own-writes checks."""
|
||||
if self.legacy_id is None:
|
||||
return frozenset({self.device_id})
|
||||
return frozenset({self.device_id, self.legacy_id})
|
||||
|
||||
def is_own(self, other_device_id: str) -> bool:
|
||||
"""Return whether [other_device_id] is one of this device's own ids."""
|
||||
return other_device_id in self.own_ids
|
||||
|
||||
|
||||
def load_device_identity(path: Path, *, legacy_id: str | None = None) -> DeviceIdentity:
|
||||
"""Return the id persisted at [path], creating it on first call.
|
||||
|
||||
The file holds exactly one uuid4 and is written once; every later call
|
||||
returns the same value, which is what makes the id stable across restarts
|
||||
(and is why it must live in durable state, not a temp dir).
|
||||
|
||||
Args:
|
||||
path: File holding this device's uuid. Parent dirs are created.
|
||||
legacy_id: The role constant this device pushed under before
|
||||
migrating, recorded on the returned identity so skip-own-writes
|
||||
still recognises the old path as this device's own. Pass ``None``
|
||||
once that path has been reclaimed.
|
||||
|
||||
Returns:
|
||||
This device's [DeviceIdentity].
|
||||
"""
|
||||
try:
|
||||
existing = path.read_text(encoding="utf-8").strip()
|
||||
except FileNotFoundError:
|
||||
existing = ""
|
||||
if existing:
|
||||
return DeviceIdentity(device_id=existing, legacy_id=legacy_id)
|
||||
|
||||
# Deliberately not guarded: if the state directory cannot be created the
|
||||
# id cannot be persisted, and returning an unpersisted one would mint a
|
||||
# fresh uuid on every restart, stranding a device directory each time.
|
||||
minted = str(uuid.uuid4())
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(f"{minted}\n", encoding="utf-8")
|
||||
return DeviceIdentity(device_id=minted, legacy_id=legacy_id)
|
||||
@ -1,26 +0,0 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""The errors the Firebase sync backend raises.
|
||||
|
||||
Split from :mod:`crdt_sync._firebase` for the 250-line cap, mirroring
|
||||
:mod:`crdt_sync._gherrors` on the GitHub side.
|
||||
|
||||
Re-exported from :mod:`crdt_sync._firebase`, so existing imports keep working.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from crdt_sync._remote import RemoteNotFoundError, RemoteSyncError
|
||||
|
||||
|
||||
class FirebaseSyncError(RemoteSyncError):
|
||||
"""Raised for an RTDB failure the caller must not silently ignore."""
|
||||
|
||||
|
||||
class DatabaseNotFoundError(FirebaseSyncError, RemoteNotFoundError):
|
||||
"""Raised when the database is unreachable or the uid is not allowed.
|
||||
|
||||
The Firebase counterpart of :class:`crdt_sync.RepoNotFoundError`: "the
|
||||
database URL is wrong, or the security rules reject this account", as
|
||||
opposed to "nothing has been pushed to that path yet", which is benign
|
||||
and surfaces as ``None``.
|
||||
"""
|
||||
@ -1,101 +0,0 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Escaping arbitrary strings into Firebase Realtime Database keys.
|
||||
|
||||
Split from :mod:`crdt_sync._firebase`, which keeps the client. RTDB forbids
|
||||
``.``, ``$``, ``#``, ``[``, ``]`` and ``/`` in a key, but the app stores
|
||||
record ids and file paths that contain them, so every key is escaped on the
|
||||
way in and unescaped on the way out. The pair has to round-trip exactly or a
|
||||
device silently stops seeing its own writes.
|
||||
|
||||
Re-exported from :mod:`crdt_sync._firebase`, so existing imports keep working.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# Characters Realtime Database forbids in a key, plus ``~`` itself because it
|
||||
# is this module's escape character. ``/`` is absent deliberately: it is the
|
||||
# path separator, handled by splitting into segments before escaping.
|
||||
_ESCAPES = {
|
||||
"~": "~7E",
|
||||
".": "~2E",
|
||||
"$": "~24",
|
||||
"#": "~23",
|
||||
"[": "~5B",
|
||||
"]": "~5D",
|
||||
}
|
||||
_UNESCAPES = {escape: char for char, escape in _ESCAPES.items()}
|
||||
_ESCAPE_LENGTH = 3
|
||||
|
||||
|
||||
def encode_key(segment: str) -> str:
|
||||
"""Escape one path segment into a legal Realtime Database key.
|
||||
|
||||
RTDB rejects ``. $ # [ ] /`` in keys, and the REST API's trailing
|
||||
``.json`` is a *format suffix* rather than part of the path -- so a
|
||||
filename like ``log.json`` cannot be stored verbatim. The mapping is a
|
||||
reversible ``~XX`` escape (``log.json`` -> ``log~2Ejson``) rather than a
|
||||
lossy "strip the extension", because callers see these names again:
|
||||
todo-app lists *filenames*, not device directories, and must keep getting
|
||||
``<uuid>.json`` back.
|
||||
|
||||
``~`` is the escape character because it is legal in RTDB keys and
|
||||
unreserved in URLs, so the escaped form needs no percent-encoding --
|
||||
which the server would otherwise decode back into the illegal character.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
segment : str
|
||||
One path segment, with no ``/`` in it.
|
||||
|
||||
Returns:
|
||||
-------
|
||||
str
|
||||
The escaped key.
|
||||
|
||||
"""
|
||||
return "".join(_ESCAPES.get(char, char) for char in segment)
|
||||
|
||||
|
||||
def decode_key(key: str) -> str:
|
||||
"""Reverse :func:`encode_key`.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
key : str
|
||||
An escaped Realtime Database key.
|
||||
|
||||
Returns:
|
||||
-------
|
||||
str
|
||||
The original segment.
|
||||
|
||||
"""
|
||||
out: list[str] = []
|
||||
index = 0
|
||||
while index < len(key):
|
||||
escape = key[index : index + _ESCAPE_LENGTH]
|
||||
char = _UNESCAPES.get(escape)
|
||||
if char is None:
|
||||
out.append(key[index])
|
||||
index += 1
|
||||
else:
|
||||
out.append(char)
|
||||
index += _ESCAPE_LENGTH
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def encode_path(path: str) -> str:
|
||||
"""Escape every segment of a ``/``-separated logical path.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path : str
|
||||
A logical path such as ``"todo-sync/notes/abc.json"``.
|
||||
|
||||
Returns:
|
||||
-------
|
||||
str
|
||||
The path with each segment escaped, empty segments dropped.
|
||||
|
||||
"""
|
||||
return "/".join(encode_key(part) for part in path.split("/") if part)
|
||||
@ -1,248 +0,0 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Firebase Realtime Database as dumb keyed storage, over its REST API.
|
||||
|
||||
The RTDB REST endpoints are plain HTTPS, so this works identically under
|
||||
systemd, in a venv, and alongside the Dart client on Android -- and the two
|
||||
write byte-identical payloads, so a device on either implementation reads the
|
||||
other's.
|
||||
|
||||
Why RTDB rather than Firestore: on the Spark (free) plan RTDB bills only
|
||||
storage and bandwidth, with **no per-operation quota**, so a misbehaving sync
|
||||
loop can never exhaust a daily budget and silently stop working mid-day.
|
||||
|
||||
Mirrors ``crdt_sync_dart``'s ``lib/src/firebase_client.dart``; keep the two in
|
||||
step.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import requests as _requests
|
||||
|
||||
from crdt_sync._fberrors import DatabaseNotFoundError, FirebaseSyncError
|
||||
from crdt_sync._fbkeys import decode_key, encode_key, encode_path
|
||||
|
||||
# Named explicitly so the autofixer cannot prune an import that exists for
|
||||
# its re-export: the tests import encode_key/decode_key from this module.
|
||||
__all__ = [
|
||||
"DatabaseNotFoundError",
|
||||
"FirebaseSyncClient",
|
||||
"FirebaseSyncError",
|
||||
"decode_key",
|
||||
"encode_key",
|
||||
"encode_path",
|
||||
]
|
||||
from crdt_sync._http import new_session
|
||||
from crdt_sync._remote import RemoteSyncError
|
||||
|
||||
# Bound to the name ``requests`` so the call sites below -- and the tests
|
||||
# that patch ``<module>.requests.<verb>`` -- are unchanged by pooling.
|
||||
requests = new_session()
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from crdt_sync._firebase_auth import FirebaseTokenProvider
|
||||
|
||||
_DEFAULT_TIMEOUT_SECONDS = 15
|
||||
_HTTP_UNAUTHORIZED = 401
|
||||
_HTTP_FORBIDDEN = 403
|
||||
|
||||
|
||||
class FirebaseSyncClient:
|
||||
"""Realtime Database seen through the same contract as GitHub.
|
||||
|
||||
Reads and writes UTF-8 text blobs stored as JSON string leaves, so the
|
||||
payload is byte-identical to what the GitHub backend stored and the two
|
||||
can be mirrored against each other during migration.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
database_url: str,
|
||||
auth: FirebaseTokenProvider,
|
||||
*,
|
||||
timeout_seconds: float = _DEFAULT_TIMEOUT_SECONDS,
|
||||
) -> None:
|
||||
"""Create a client for ``database_url`` authenticated by ``auth``.
|
||||
|
||||
Args:
|
||||
database_url: The database's HTTPS origin.
|
||||
auth: Mints the ID token every request carries.
|
||||
timeout_seconds: Per-request network timeout.
|
||||
"""
|
||||
self._database_url = database_url.rstrip("/")
|
||||
self.auth = auth
|
||||
self._timeout_seconds = timeout_seconds
|
||||
|
||||
def _url(self, path: str) -> str:
|
||||
return f"{self._database_url}/{encode_path(path)}.json"
|
||||
|
||||
def _params(self, **extra: str) -> dict[str, str]:
|
||||
return {"auth": self.auth.id_token(), **extra}
|
||||
|
||||
def _raise(self, what: str, response: _requests.Response) -> None:
|
||||
"""Raise the right error type for a non-2xx response.
|
||||
|
||||
401/403 mean the rules rejected this uid or the token is bad.
|
||||
Everything else -- including a Spark quota exhaustion, which answers
|
||||
with an error rather than billing you -- is a plain
|
||||
:class:`FirebaseSyncError`. Nothing is ever swallowed: a
|
||||
quota-exhausted database that silently returned "no data" would look
|
||||
exactly like "nothing to sync".
|
||||
"""
|
||||
detail = response.text.strip()
|
||||
if response.status_code in (_HTTP_UNAUTHORIZED, _HTTP_FORBIDDEN):
|
||||
msg = (
|
||||
f"{what} rejected: HTTP {response.status_code} {detail} -- "
|
||||
f"the database URL or the security rules do not allow this "
|
||||
f"account"
|
||||
)
|
||||
raise DatabaseNotFoundError(msg)
|
||||
msg = f"{what} failed: HTTP {response.status_code} {detail}"
|
||||
raise FirebaseSyncError(msg)
|
||||
|
||||
def _get(self, path: str, **params: str) -> _requests.Response:
|
||||
try:
|
||||
return requests.get(
|
||||
self._url(path),
|
||||
params=self._params(**params),
|
||||
timeout=self._timeout_seconds,
|
||||
)
|
||||
except _requests.RequestException as exc:
|
||||
msg = f"network error reading {path}"
|
||||
raise FirebaseSyncError(msg) from exc
|
||||
|
||||
def list_directory(self, path: str) -> list[str]:
|
||||
"""Return the entry names directly under ``path``.
|
||||
|
||||
Uses ``shallow=true``, which returns ``{key: true}`` without any
|
||||
values -- so a listing costs bytes rather than the whole subtree.
|
||||
That is the difference between a sync tick costing hundreds of bytes
|
||||
and hundreds of kilobytes.
|
||||
|
||||
Raises:
|
||||
DatabaseNotFoundError: If the rules reject this account.
|
||||
FirebaseSyncError: For any other non-2xx response or network
|
||||
error.
|
||||
"""
|
||||
response = self._get(path, shallow="true")
|
||||
if not response.ok:
|
||||
self._raise(f"listing {path}", response)
|
||||
data = response.json()
|
||||
if not isinstance(data, dict):
|
||||
return []
|
||||
return [decode_key(key) for key in data]
|
||||
|
||||
def get_file_text(self, path: str) -> str | None:
|
||||
"""Return the text stored at ``path``, or ``None`` if absent.
|
||||
|
||||
Raises:
|
||||
DatabaseNotFoundError: If the rules reject this account.
|
||||
FirebaseSyncError: If the value is not a text blob, or for any
|
||||
other non-2xx response or network error.
|
||||
"""
|
||||
response = self._get(path)
|
||||
if not response.ok:
|
||||
self._raise(f"reading {path}", response)
|
||||
data = response.json()
|
||||
if data is None:
|
||||
return None
|
||||
if not isinstance(data, str):
|
||||
msg = (
|
||||
f"value at {path} is {type(data).__name__}, not the expected text blob"
|
||||
)
|
||||
raise FirebaseSyncError(msg)
|
||||
return data
|
||||
|
||||
def put_file_text(self, path: str, text: str, *, message: str) -> None:
|
||||
"""Write ``text`` at ``path``.
|
||||
|
||||
Args:
|
||||
path: A logical path such as ``"ns/devices/pc/log.json"``.
|
||||
text: The full new content.
|
||||
message: Ignored -- RTDB has no commit log to attach a reason to.
|
||||
Present so this matches the ``RemoteStore`` contract.
|
||||
|
||||
Raises:
|
||||
FirebaseSyncError: On any non-2xx response or network error.
|
||||
"""
|
||||
del message
|
||||
try:
|
||||
response = requests.put(
|
||||
self._url(path),
|
||||
params=self._params(),
|
||||
json=text,
|
||||
timeout=self._timeout_seconds,
|
||||
)
|
||||
except _requests.RequestException as exc:
|
||||
msg = f"network error writing {path}"
|
||||
raise FirebaseSyncError(msg) from exc
|
||||
if not response.ok:
|
||||
self._raise(f"writing {path}", response)
|
||||
|
||||
def get_string_map(self, path: str) -> dict[str, str]:
|
||||
"""Return the map at ``path`` as ``key -> text``, empty if absent.
|
||||
|
||||
Tolerates a non-map or malformed value rather than raising: this
|
||||
backs the revision cache, which is an *optimisation*. A corrupt revs
|
||||
node must degrade into "fetch everything", never into a failed sync.
|
||||
|
||||
Raises:
|
||||
FirebaseSyncError: Only for a transport-level failure.
|
||||
"""
|
||||
response = self._get(path)
|
||||
if not response.ok:
|
||||
self._raise(f"reading {path}", response)
|
||||
data = response.json()
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
return {
|
||||
decode_key(key): value
|
||||
for key, value in data.items()
|
||||
if isinstance(value, str)
|
||||
}
|
||||
|
||||
def delete_file(self, path: str, *, message: str = "crdt_sync: delete") -> None:
|
||||
"""Delete ``path``. A no-op if it does not exist.
|
||||
|
||||
Args:
|
||||
path: A logical path.
|
||||
message: Ignored, as for :meth:`put_file_text`.
|
||||
|
||||
Raises:
|
||||
FirebaseSyncError: On any non-2xx response or network error.
|
||||
"""
|
||||
del message
|
||||
try:
|
||||
response = requests.delete(
|
||||
self._url(path),
|
||||
params=self._params(),
|
||||
timeout=self._timeout_seconds,
|
||||
)
|
||||
except _requests.RequestException as exc:
|
||||
msg = f"network error deleting {path}"
|
||||
raise FirebaseSyncError(msg) from exc
|
||||
if not response.ok:
|
||||
self._raise(f"deleting {path}", response)
|
||||
|
||||
def can_access_remote(self) -> bool:
|
||||
"""Return whether the credential can reach the database.
|
||||
|
||||
Never raises -- a bad URL, a rejected token, a missing session or a
|
||||
network failure all report ``False``, so a settings "Test connection"
|
||||
button cannot blow up.
|
||||
"""
|
||||
try:
|
||||
response = requests.get(
|
||||
self._url(""),
|
||||
params=self._params(shallow="true"),
|
||||
timeout=self._timeout_seconds,
|
||||
)
|
||||
except _requests.RequestException:
|
||||
return False
|
||||
except RemoteSyncError:
|
||||
# Includes FirebaseAuthError: "cannot get a token" is exactly
|
||||
# "cannot access the remote".
|
||||
return False
|
||||
return response.ok
|
||||
@ -1,245 +0,0 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Firebase Authentication over its REST API, with no SDK dependency.
|
||||
|
||||
The Firebase Admin SDK authenticates with a service-account key, which
|
||||
bypasses security rules entirely and would have to be distributed to every
|
||||
device. These are single-user personal apps, so devices instead sign in as
|
||||
one ordinary user and hold a refresh token -- the same path the Flutter side
|
||||
takes, so both platforms behave identically.
|
||||
|
||||
Mirrors ``crdt_sync_dart``'s ``lib/src/firebase_auth_rest.dart``; keep the
|
||||
two in step.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
|
||||
import requests as _requests
|
||||
|
||||
from crdt_sync._autherrors import FirebaseAuthError, _is_revoked, _reason
|
||||
from crdt_sync._credentials import (
|
||||
CredentialStore,
|
||||
FileCredentialStore,
|
||||
FirebaseCredentials,
|
||||
MemoryCredentialStore,
|
||||
_utcnow,
|
||||
)
|
||||
from crdt_sync._http import new_session
|
||||
|
||||
__all__ = [
|
||||
"CredentialStore",
|
||||
"FileCredentialStore",
|
||||
"FirebaseAuthError",
|
||||
"FirebaseCredentials",
|
||||
"FirebaseTokenProvider",
|
||||
"MemoryCredentialStore",
|
||||
]
|
||||
|
||||
# Bound to the name ``requests`` so the call sites below -- and the tests
|
||||
# that patch ``<module>.requests.<verb>`` -- are unchanged by pooling.
|
||||
requests = new_session()
|
||||
|
||||
|
||||
_SIGN_IN_BASE = "https://identitytoolkit.googleapis.com/v1"
|
||||
_REFRESH_BASE = "https://securetoken.googleapis.com/v1"
|
||||
_DEFAULT_TIMEOUT_SECONDS = 15
|
||||
|
||||
|
||||
class FirebaseTokenProvider:
|
||||
"""Signs in and keeps a valid ID token available, refreshing as needed.
|
||||
|
||||
``api_key`` is the project's public Web API key -- not a secret, and safe
|
||||
to ship inside an APK. The actual credential is the refresh token held by
|
||||
the store.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str,
|
||||
store: CredentialStore,
|
||||
*,
|
||||
timeout_seconds: float = _DEFAULT_TIMEOUT_SECONDS,
|
||||
clock: object = None,
|
||||
) -> None:
|
||||
"""Create a provider for ``api_key`` persisting through ``store``.
|
||||
|
||||
Args:
|
||||
api_key: The project's public Web API key.
|
||||
store: Where the refresh token lives between runs.
|
||||
timeout_seconds: Per-request network timeout.
|
||||
clock: A zero-arg callable returning "now" (tests inject one).
|
||||
"""
|
||||
self._api_key = api_key
|
||||
self._store = store
|
||||
self._timeout_seconds = timeout_seconds
|
||||
self._clock = clock if callable(clock) else _utcnow
|
||||
self._cached: FirebaseCredentials | None = None
|
||||
|
||||
def sign_in(self, email: str, password: str) -> None:
|
||||
"""Exchange an email/password for a session and persist it.
|
||||
|
||||
Called once per device from a setup command; every later run reuses
|
||||
the stored refresh token.
|
||||
|
||||
Raises:
|
||||
FirebaseAuthError: If the credentials are rejected.
|
||||
"""
|
||||
body = self._post(
|
||||
f"{_SIGN_IN_BASE}/accounts:signInWithPassword?key={self._api_key}",
|
||||
{"email": email, "password": password, "returnSecureToken": True},
|
||||
"sign in",
|
||||
)
|
||||
self._adopt(
|
||||
id_token=body["idToken"],
|
||||
refresh_token=body["refreshToken"],
|
||||
expires_in_seconds=body["expiresIn"],
|
||||
)
|
||||
|
||||
def sign_in_with_google(self, id_token: str, expected_uid: str = "") -> str:
|
||||
"""Exchange a Google ID token for a session and persist it.
|
||||
|
||||
Mirrors ``crdt_sync_dart``'s ``signInWithGoogle``. Used to reseed a
|
||||
machine's session after the sync account moved to Google-only sign-in:
|
||||
the refresh token this stores is what keeps a headless job working
|
||||
between runs, so the browser flow is needed once per machine, not once
|
||||
per tick.
|
||||
|
||||
Deliberately *not* a service-account key. That would authenticate as
|
||||
an admin and bypass the security rules entirely, so the pinned-uid
|
||||
rule that protects this data would stop applying to every PC write.
|
||||
|
||||
Args:
|
||||
id_token: A Google ID token whose audience is this project.
|
||||
expected_uid: When given, asserted against the uid Firebase
|
||||
returns. A mismatch raises without storing anything -- an
|
||||
unlinked Google identity is *signed up* rather than refused,
|
||||
and that session would then be denied every read and write.
|
||||
|
||||
Returns:
|
||||
The email Firebase reports for the account.
|
||||
|
||||
Raises:
|
||||
FirebaseAuthError: If the token is rejected, or resolves to a uid
|
||||
other than ``expected_uid``.
|
||||
"""
|
||||
body = self._post(
|
||||
f"{_SIGN_IN_BASE}/accounts:signInWithIdp?key={self._api_key}",
|
||||
{
|
||||
# The IdP credential travels form-encoded, not as JSON -- an
|
||||
# identitytoolkit quirk. Sending it as a field returns
|
||||
# INVALID_IDP_RESPONSE, which reads like a bad token.
|
||||
"postBody": f"id_token={id_token}&providerId=google.com",
|
||||
"requestUri": "http://localhost",
|
||||
"returnSecureToken": True,
|
||||
},
|
||||
"sign in with Google",
|
||||
)
|
||||
uid = body.get("localId", "")
|
||||
if expected_uid and uid != expected_uid:
|
||||
msg = (
|
||||
f"signed in as the wrong account: Google resolved to uid "
|
||||
f"{uid!r}, but this data belongs to {expected_uid!r}"
|
||||
)
|
||||
raise FirebaseAuthError(msg)
|
||||
self._adopt(
|
||||
id_token=body["idToken"],
|
||||
refresh_token=body["refreshToken"],
|
||||
expires_in_seconds=body["expiresIn"],
|
||||
)
|
||||
return str(body.get("email", ""))
|
||||
|
||||
def id_token(self) -> str:
|
||||
"""Return a currently-valid ID token, refreshing if needed.
|
||||
|
||||
Never returns a stale token and never silently no-ops: a sync that
|
||||
quietly stops syncing is the failure mode this design exists to
|
||||
prevent.
|
||||
|
||||
Raises:
|
||||
FirebaseAuthError: If no session is stored, or the refresh token
|
||||
has been revoked.
|
||||
"""
|
||||
credentials = self._load()
|
||||
if credentials is None:
|
||||
msg = "not signed in: no stored refresh token for this device"
|
||||
raise FirebaseAuthError(msg)
|
||||
if not credentials.is_expired_at(self._clock()):
|
||||
return credentials.id_token
|
||||
return self._refresh(credentials.refresh_token)
|
||||
|
||||
def has_session(self) -> bool:
|
||||
"""Return whether this device has a stored session at all.
|
||||
|
||||
Distinguishes "sync is not configured" -- a normal state for
|
||||
``screen-locker`` -- from "sync is configured and broken", an error.
|
||||
"""
|
||||
return self._load() is not None
|
||||
|
||||
def sign_out(self) -> None:
|
||||
"""Forget the stored session, so the next token request fails."""
|
||||
self._cached = None
|
||||
self._store.clear()
|
||||
|
||||
def _load(self) -> FirebaseCredentials | None:
|
||||
if self._cached is None:
|
||||
self._cached = self._store.load()
|
||||
return self._cached
|
||||
|
||||
def _refresh(self, refresh_token: str) -> str:
|
||||
try:
|
||||
body = self._post(
|
||||
f"{_REFRESH_BASE}/token?key={self._api_key}",
|
||||
{"grant_type": "refresh_token", "refresh_token": refresh_token},
|
||||
"refresh the session",
|
||||
)
|
||||
except FirebaseAuthError as error:
|
||||
# A revoked refresh token never becomes valid again, so keeping it
|
||||
# is what let a dead device report "connected" and then fail every
|
||||
# sync with TOKEN_EXPIRED. Drop it so has_session() is honest.
|
||||
if _is_revoked(error):
|
||||
self.sign_out()
|
||||
raise
|
||||
return self._adopt(
|
||||
id_token=body["id_token"],
|
||||
# A refresh may hand back a rotated token; keep the new one.
|
||||
refresh_token=body["refresh_token"],
|
||||
expires_in_seconds=body["expires_in"],
|
||||
)
|
||||
|
||||
def _adopt(
|
||||
self,
|
||||
*,
|
||||
id_token: str,
|
||||
refresh_token: str,
|
||||
expires_in_seconds: str,
|
||||
) -> str:
|
||||
credentials = FirebaseCredentials(
|
||||
id_token=id_token,
|
||||
refresh_token=refresh_token,
|
||||
expires_at=self._clock() + dt.timedelta(seconds=int(expires_in_seconds)),
|
||||
)
|
||||
self._cached = credentials
|
||||
self._store.save(credentials)
|
||||
return id_token
|
||||
|
||||
def _post(
|
||||
self,
|
||||
url: str,
|
||||
payload: dict[str, object],
|
||||
what: str,
|
||||
) -> dict[str, str]:
|
||||
try:
|
||||
response = requests.post(
|
||||
url,
|
||||
json=payload,
|
||||
timeout=self._timeout_seconds,
|
||||
)
|
||||
except _requests.RequestException as exc:
|
||||
msg = f"network error trying to {what}"
|
||||
raise FirebaseAuthError(msg) from exc
|
||||
if not response.ok:
|
||||
reason = _reason(response)
|
||||
msg = f"failed to {what}: HTTP {response.status_code}{reason}"
|
||||
raise FirebaseAuthError(msg)
|
||||
return response.json()
|
||||
@ -1,31 +0,0 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""The errors the GitHub sync backend raises.
|
||||
|
||||
Split from :mod:`crdt_sync._github` for the 250-line cap. Kept separate so a
|
||||
caller can catch them without importing the client -- and so the client file
|
||||
is the client, not its taxonomy.
|
||||
|
||||
Re-exported from :mod:`crdt_sync._github`, so existing imports keep working.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from crdt_sync._remote import RemoteNotFoundError, RemoteSyncError
|
||||
|
||||
|
||||
class GitHubSyncError(RemoteSyncError):
|
||||
"""Raised for a GitHub API failure the caller must not silently ignore."""
|
||||
|
||||
|
||||
class RepoNotFoundError(GitHubSyncError, RemoteNotFoundError):
|
||||
"""Raised when the configured repo itself is unreachable.
|
||||
|
||||
Distinguished from a path-404 (nothing pushed to that path yet, which is
|
||||
benign -- it just means no other device has synced before) so the caller
|
||||
can tell "the repo name is wrong or the token isn't scoped to it" apart
|
||||
from "no other device has synced yet".
|
||||
|
||||
Derives from both so existing ``except GitHubSyncError`` handlers still
|
||||
cover it *and* backend-neutral callers can catch "the remote itself is
|
||||
missing" without naming GitHub.
|
||||
"""
|
||||
@ -1,4 +1,3 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Minimal GitHub Contents API client used as dumb file storage.
|
||||
|
||||
GitHub is used purely as file storage via the REST Contents API, not a git
|
||||
@ -12,21 +11,27 @@ from __future__ import annotations
|
||||
|
||||
import base64
|
||||
|
||||
import requests as _requests
|
||||
|
||||
from crdt_sync._gherrors import GitHubSyncError, RepoNotFoundError
|
||||
from crdt_sync._http import new_session
|
||||
|
||||
# Bound to the name ``requests`` so the call sites below -- and the tests
|
||||
# that patch ``<module>.requests.<verb>`` -- are unchanged by pooling.
|
||||
requests = new_session()
|
||||
|
||||
import requests
|
||||
|
||||
_API_BASE = "https://api.github.com"
|
||||
_HTTP_NOT_FOUND = 404
|
||||
_DEFAULT_TIMEOUT_SECONDS = 15
|
||||
|
||||
|
||||
class GitHubSyncError(Exception):
|
||||
"""Raised for a GitHub API failure the caller must not silently ignore."""
|
||||
|
||||
|
||||
class RepoNotFoundError(GitHubSyncError):
|
||||
"""Raised when the configured repo itself is unreachable.
|
||||
|
||||
Distinguished from a path-404 (nothing pushed to that path yet, which is
|
||||
benign -- it just means no other device has synced before) so the caller
|
||||
can tell "the repo name is wrong or the token isn't scoped to it" apart
|
||||
from "no other device has synced yet".
|
||||
"""
|
||||
|
||||
|
||||
class GitHubSyncClient:
|
||||
"""Thin wrapper around the subset of the Contents API sync needs."""
|
||||
|
||||
@ -57,14 +62,14 @@ class GitHubSyncClient:
|
||||
def _contents_url(self, path: str) -> str:
|
||||
return f"{_API_BASE}/repos/{self._owner}/{self._repo}/contents/{path}"
|
||||
|
||||
def _get(self, path: str) -> _requests.Response:
|
||||
def _get(self, path: str) -> requests.Response:
|
||||
try:
|
||||
return requests.get(
|
||||
self._contents_url(path),
|
||||
headers=self._headers,
|
||||
timeout=self._timeout_seconds,
|
||||
)
|
||||
except _requests.RequestException as exc:
|
||||
except requests.RequestException as exc:
|
||||
msg = f"network error reading {path}"
|
||||
raise GitHubSyncError(msg) from exc
|
||||
|
||||
@ -75,7 +80,7 @@ class GitHubSyncClient:
|
||||
headers=self._headers,
|
||||
timeout=self._timeout_seconds,
|
||||
)
|
||||
except _requests.RequestException:
|
||||
except requests.RequestException:
|
||||
return False
|
||||
return response.ok
|
||||
|
||||
@ -155,7 +160,7 @@ class GitHubSyncClient:
|
||||
json=payload,
|
||||
timeout=self._timeout_seconds,
|
||||
)
|
||||
except _requests.RequestException as exc:
|
||||
except requests.RequestException as exc:
|
||||
msg = f"network error pushing {path}"
|
||||
raise GitHubSyncError(msg) from exc
|
||||
if not response.ok:
|
||||
@ -194,17 +199,6 @@ class GitHubSyncClient:
|
||||
Hits the bare repo endpoint, so it succeeds even before any file has
|
||||
been pushed. Never raises -- a network failure or missing repo is
|
||||
reported as ``False``.
|
||||
|
||||
The GitHub-era name for :meth:`can_access_remote`; kept because app
|
||||
settings screens call it. Both do the same probe.
|
||||
"""
|
||||
return self._repo_exists()
|
||||
|
||||
def can_access_remote(self) -> bool:
|
||||
"""Return whether the token can reach the remote (a connection test).
|
||||
|
||||
The :class:`crdt_sync.RemoteStore` spelling of
|
||||
:meth:`can_access_repo`.
|
||||
"""
|
||||
return self._repo_exists()
|
||||
|
||||
@ -230,7 +224,7 @@ class GitHubSyncClient:
|
||||
json={"message": message, "sha": sha},
|
||||
timeout=self._timeout_seconds,
|
||||
)
|
||||
except _requests.RequestException as exc:
|
||||
except requests.RequestException as exc:
|
||||
msg = f"network error deleting {path}"
|
||||
raise GitHubSyncError(msg) from exc
|
||||
if not response.ok:
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Hybrid Logical Clock: a totally-ordered, monotonic per-node timestamp.
|
||||
|
||||
An HLC combines wall-clock time with a logical counter so that two ticks
|
||||
@ -10,11 +9,11 @@ ties), without requiring synchronized clocks across devices.
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from datetime import datetime, timezone
|
||||
import time
|
||||
|
||||
_ISO_PREFIX_LEN = 23 # len("YYYY-MM-DDTHH:MM:SS.mmm")
|
||||
_EPOCH = datetime(1970, 1, 1, tzinfo=UTC)
|
||||
_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
@dataclass(frozen=True, order=True)
|
||||
@ -67,7 +66,7 @@ class Hlc:
|
||||
comparison for the wall-clock component.
|
||||
"""
|
||||
seconds, millis = divmod(self.wall_time_ms, 1000)
|
||||
dt = datetime.fromtimestamp(seconds, tz=UTC)
|
||||
dt = datetime.fromtimestamp(seconds, tz=timezone.utc)
|
||||
iso = dt.strftime("%Y-%m-%dT%H:%M:%S")
|
||||
return f"{iso}.{millis:03d}Z-{self.counter:04x}-{self.node_id}"
|
||||
|
||||
@ -86,7 +85,9 @@ class Hlc:
|
||||
if not sep:
|
||||
msg = f"not a valid Hlc string: {text!r}"
|
||||
raise ValueError(msg)
|
||||
dt = datetime.strptime(iso_part, "%Y-%m-%dT%H:%M:%S.%f").replace(tzinfo=UTC)
|
||||
dt = datetime.strptime(iso_part, "%Y-%m-%dT%H:%M:%S.%f").replace(
|
||||
tzinfo=timezone.utc
|
||||
)
|
||||
delta = dt - _EPOCH
|
||||
wall_time_ms = (
|
||||
delta.days * 86_400_000 + delta.seconds * 1000 + delta.microseconds // 1000
|
||||
|
||||
@ -1,58 +0,0 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Pooled HTTP sessions, so every request does not re-handshake.
|
||||
|
||||
Every call site in this package used the module-level ``requests.get`` /
|
||||
``put`` / ``post`` / ``delete`` helpers, each of which opens a throwaway
|
||||
connection: fresh DNS, TCP and TLS on every single request. Measured against
|
||||
the two real backends on 2026-08-21 that costs 82ms per request to
|
||||
``api.github.com`` and 51ms to Realtime Database -- on a 27-request sync tick,
|
||||
most of the wall clock.
|
||||
|
||||
A :class:`requests.Session` keeps the connection alive between requests, and
|
||||
is a drop-in for the module-level helpers: same verb names, same signatures.
|
||||
So each transport module binds one of these to the name ``requests`` and its
|
||||
call sites are unchanged.
|
||||
|
||||
**One session per module, deliberately not one global.** The tests patch the
|
||||
verb on the module attribute (``patch.object(fb.requests, "get", ...)``), and
|
||||
several use sequence-consuming ``side_effect`` lists. Were all three modules
|
||||
to share a single object, a call from one would consume another's queued
|
||||
response and the failure would look like a logic bug. Per-module also happens
|
||||
to be per-host here -- GitHub, Realtime Database, and the token endpoint are
|
||||
three different origins -- so pooling loses nothing by the split.
|
||||
|
||||
Not thread-safe, in that :class:`requests.Session` is not documented to be;
|
||||
this package spawns no threads and its consumers drive sync from one.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter, Retry
|
||||
|
||||
_RETRY_TOTAL = 3
|
||||
_RETRY_BACKOFF_SECONDS = 0.3
|
||||
_RETRY_STATUSES = (429, 500, 502, 503, 504)
|
||||
|
||||
|
||||
def new_session() -> requests.Session:
|
||||
"""Return a connection-pooling session that retries transient GET failures.
|
||||
|
||||
Retries are restricted to ``GET``. A ``PUT`` here carries a Contents-API
|
||||
``sha`` (and a Realtime Database write is a full-node replace); replaying
|
||||
one whose response was merely lost would push a second write built on a
|
||||
now-stale precondition, against a remote another device reads. Reads are
|
||||
idempotent, so only they are replayed.
|
||||
"""
|
||||
session = requests.Session()
|
||||
retry = Retry(
|
||||
total=_RETRY_TOTAL,
|
||||
backoff_factor=_RETRY_BACKOFF_SECONDS,
|
||||
status_forcelist=_RETRY_STATUSES,
|
||||
allowed_methods=frozenset({"GET"}),
|
||||
raise_on_status=False,
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry)
|
||||
session.mount("https://", adapter)
|
||||
session.mount("http://", adapter)
|
||||
return session
|
||||
@ -1,4 +1,3 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Log: a collection of Records keyed by id, with a union-merge."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@ -1,231 +0,0 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""A remote store that writes to two backends and reads from both.
|
||||
|
||||
Exists for the GitHub -> Firebase cutover: each app can move to Firebase
|
||||
while still mirroring to the old repo, so rolling back is a constructor
|
||||
change rather than a data recovery.
|
||||
|
||||
Mirrors ``crdt_sync_dart``'s ``lib/src/mirror_store.dart``; keep the two in
|
||||
step.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
from crdt_sync._remote import RemoteSyncError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from crdt_sync._remote import RemoteStore
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
# PEP 695 type parameters need Python 3.12; this package targets 3.10.
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
class MirrorSyncClient:
|
||||
"""Writes to both backends; reads from both, preferring the primary.
|
||||
|
||||
**Reads consult both on purpose.** Each app spans two devices, and they
|
||||
cut over one at a time. If reads were primary-only, a migrated PC would
|
||||
never see an un-migrated phone's writes while still mirroring its own
|
||||
back -- silent one-directional convergence, with no error raised. Because
|
||||
:func:`crdt_sync.merge_logs` is commutative and idempotent, a union read
|
||||
costs nothing semantically and makes the cutover order irrelevant.
|
||||
|
||||
Write asymmetry is deliberate too: the primary is authoritative, so a
|
||||
primary failure fails the tick (fail-closed), while a mirror failure is
|
||||
reported and otherwise tolerated. Once the mirror is retired, the old
|
||||
backend going away must not break sync.
|
||||
|
||||
**Reads are resilient on BOTH sides; writes are not.** A read is only
|
||||
fail-closed when neither backend can answer. Reads used to call the
|
||||
primary unguarded, so a Firebase outage produced no data at all and the
|
||||
mirror was never consulted -- the union read silently degraded to nothing,
|
||||
contradicting the paragraph above. Writes keep the asymmetry: a write the
|
||||
primary did not accept has not happened, so it must fail the tick.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
primary: RemoteStore,
|
||||
mirror: RemoteStore,
|
||||
*,
|
||||
on_mirror_failure: Callable[[str, Exception], None] | None = None,
|
||||
) -> None:
|
||||
"""Wrap ``primary`` (authoritative) and ``mirror`` (best effort).
|
||||
|
||||
Args:
|
||||
primary: The backend whose failures fail the tick.
|
||||
mirror: The backend kept in step but never allowed to fail a tick.
|
||||
on_mirror_failure: Notified when the mirror misbehaves. Defaults
|
||||
to a warning log -- a silent mirror failure would let the
|
||||
fallback rot unnoticed until the day it was needed.
|
||||
"""
|
||||
self.primary = primary
|
||||
self.mirror = mirror
|
||||
self._on_mirror_failure = on_mirror_failure or _log_mirror_failure
|
||||
|
||||
def list_directory(self, path: str) -> list[str]:
|
||||
"""Return the union of both backends' entries under ``path``.
|
||||
|
||||
A reachable primary that lists nothing is a real answer; a primary
|
||||
that *raises* is not. Reads are the half of this class that exists to
|
||||
be resilient, so a primary failure degrades to the mirror's entries
|
||||
rather than failing the read -- returning nothing because one side is
|
||||
down breaks the union promise exactly when the fallback is most
|
||||
needed. Only a failure on BOTH sides propagates, since then there is
|
||||
genuinely no answer to give.
|
||||
"""
|
||||
primary_names = self._try_primary_read(
|
||||
f"list_directory {path}", lambda: self.primary.list_directory(path)
|
||||
)
|
||||
mirror_names = self._try_mirror(
|
||||
f"list_directory {path}", lambda: self.mirror.list_directory(path)
|
||||
)
|
||||
if primary_names is None and mirror_names is None:
|
||||
self._raise_both_failed(f"list_directory {path}")
|
||||
names = list(primary_names or [])
|
||||
seen = set(names)
|
||||
for name in mirror_names or []:
|
||||
if name not in seen:
|
||||
seen.add(name)
|
||||
names.append(name)
|
||||
return names
|
||||
|
||||
def get_file_text(self, path: str) -> str | None:
|
||||
"""Return ``path``'s text from the primary, else from the mirror.
|
||||
|
||||
Absent from the primary means "this device has not migrated yet",
|
||||
not "no data" -- so it falls through rather than reporting nothing.
|
||||
A primary that *raises* falls through for the same reason (see
|
||||
:meth:`list_directory`); only a failure on both sides propagates.
|
||||
"""
|
||||
try:
|
||||
text = self.primary.get_file_text(path)
|
||||
except RemoteSyncError as error:
|
||||
self._on_primary_read_failure(f"get_file_text {path}", error)
|
||||
mirror_text = self._try_mirror(
|
||||
f"get_file_text {path}", lambda: self.mirror.get_file_text(path)
|
||||
)
|
||||
if mirror_text is None:
|
||||
self._raise_both_failed(f"get_file_text {path}")
|
||||
return mirror_text
|
||||
if text is not None:
|
||||
return text
|
||||
return self._try_mirror(
|
||||
f"get_file_text {path}", lambda: self.mirror.get_file_text(path)
|
||||
)
|
||||
|
||||
def put_file_text(self, path: str, text: str, *, message: str) -> None:
|
||||
"""Write to the primary, then best-effort to the mirror.
|
||||
|
||||
Raises:
|
||||
RemoteSyncError: If the *primary* write fails.
|
||||
"""
|
||||
self.primary.put_file_text(path, text, message=message)
|
||||
self._try_mirror(
|
||||
f"put_file_text {path}",
|
||||
lambda: self.mirror.put_file_text(path, text, message=message),
|
||||
)
|
||||
|
||||
def delete_file(self, path: str, *, message: str = "crdt_sync: delete") -> None:
|
||||
"""Delete from the primary, then best-effort from the mirror.
|
||||
|
||||
Raises:
|
||||
RemoteSyncError: If the *primary* delete fails.
|
||||
"""
|
||||
self.primary.delete_file(path, message=message)
|
||||
self._try_mirror(
|
||||
f"delete_file {path}",
|
||||
lambda: self.mirror.delete_file(path, message=message),
|
||||
)
|
||||
|
||||
def can_access_remote(self) -> bool:
|
||||
"""Return whether the **primary** is reachable.
|
||||
|
||||
Only the primary: a settings "Test connection" button must not report
|
||||
success because the backend being retired happens to answer.
|
||||
"""
|
||||
return self.primary.can_access_remote()
|
||||
|
||||
def get_string_map(self, path: str) -> dict[str, str]:
|
||||
"""Merge both backends' revision maps, primary winning on conflict.
|
||||
|
||||
A device that has not migrated publishes revisions only to the
|
||||
mirror; without this it would look revision-less and be
|
||||
re-downloaded every tick for the whole trial period.
|
||||
|
||||
Like the other reads, a primary failure degrades to the mirror rather
|
||||
than failing the tick: revisions are a cache, and losing them re-reads
|
||||
data rather than losing it.
|
||||
"""
|
||||
merged: dict[str, str] = {}
|
||||
mirror_read = getattr(self.mirror, "get_string_map", None)
|
||||
# A mirror that CANNOT do bulk reads contributes nothing, but that is
|
||||
# not a failure -- it has no revisions to give, which is a real answer.
|
||||
# Only a mirror that tried and threw leaves the read with no answer,
|
||||
# and only then can a failing primary make this raise.
|
||||
mirror_answered = mirror_read is None
|
||||
if mirror_read is not None:
|
||||
mirror_map = self._try_mirror(
|
||||
f"get_string_map {path}", lambda: mirror_read(path)
|
||||
)
|
||||
mirror_answered = mirror_map is not None
|
||||
merged.update(mirror_map or {})
|
||||
primary_read = getattr(self.primary, "get_string_map", None)
|
||||
if primary_read is not None:
|
||||
primary_map = self._try_primary_read(
|
||||
f"get_string_map {path}", lambda: primary_read(path)
|
||||
)
|
||||
if primary_map is None and not mirror_answered:
|
||||
self._raise_both_failed(f"get_string_map {path}")
|
||||
merged.update(primary_map or {})
|
||||
return merged
|
||||
|
||||
def _try_mirror(self, operation: str, run: Callable[[], _T]) -> _T | None:
|
||||
"""Run a mirror operation, reporting rather than propagating failure."""
|
||||
try:
|
||||
return run()
|
||||
except RemoteSyncError as error:
|
||||
self._on_mirror_failure(operation, error)
|
||||
return None
|
||||
|
||||
def _try_primary_read(self, operation: str, run: Callable[[], _T]) -> _T | None:
|
||||
"""Run a primary READ, reporting rather than propagating failure.
|
||||
|
||||
Reads only -- writes stay fail-closed, since a write the primary did
|
||||
not accept has not happened and must fail the tick.
|
||||
"""
|
||||
try:
|
||||
return run()
|
||||
except RemoteSyncError as error:
|
||||
self._on_primary_read_failure(operation, error)
|
||||
return None
|
||||
|
||||
def _on_primary_read_failure(self, operation: str, error: Exception) -> None:
|
||||
"""Warn that a read fell back to the mirror."""
|
||||
_logger.warning(
|
||||
"Primary backend failed during %s: %s — falling back to the mirror",
|
||||
operation,
|
||||
error,
|
||||
)
|
||||
|
||||
def _raise_both_failed(self, operation: str) -> None:
|
||||
"""Raise when neither backend could answer a read.
|
||||
|
||||
Fail closed rather than returning an empty result: an empty read is
|
||||
indistinguishable from "no data", and callers act on that by treating
|
||||
remote state as absent.
|
||||
"""
|
||||
msg = f"both primary and mirror backends failed during {operation}"
|
||||
raise RemoteSyncError(msg)
|
||||
|
||||
|
||||
def _log_mirror_failure(operation: str, error: Exception) -> None:
|
||||
"""Warn loudly, so a rotting fallback is visible before it is needed."""
|
||||
_logger.warning("Mirror backend failed during %s: %s", operation, error)
|
||||
@ -1,109 +0,0 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Downloading peers' logs, skipping the ones we already have.
|
||||
|
||||
Split from :mod:`crdt_sync._sync`, which keeps the push half and the
|
||||
orchestration. The revision map is the optimisation that makes a tick cheap:
|
||||
a peer whose published revision matches what we last merged is not fetched at
|
||||
all, so a quiet device costs one small read instead of one read per peer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from crdt_sync._revisions import revision_of
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from crdt_sync._log import Log
|
||||
from crdt_sync._remote import RemoteStore
|
||||
from crdt_sync._syncstate import SyncState
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _remote_revs(client: RemoteStore, revs_path: str) -> dict[str, str]:
|
||||
"""Return every peer's published revision, cheaply where possible.
|
||||
|
||||
Degrades to an empty map -- meaning "fetch everything", the old behaviour
|
||||
-- on any backend without a bulk-map read, so correctness never depends
|
||||
on the optimisation being available.
|
||||
"""
|
||||
get_string_map = getattr(client, "get_string_map", None)
|
||||
if get_string_map is None:
|
||||
return {}
|
||||
return get_string_map(revs_path)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _PullContext:
|
||||
"""Everything :func:`_pull_remote_logs` needs for one tick's pull.
|
||||
|
||||
A bundle rather than eight positional parameters: they are always passed
|
||||
together, from one call site, and several share a type -- exactly the
|
||||
shape where a positional mix-up is silent.
|
||||
"""
|
||||
|
||||
client: RemoteStore
|
||||
own_ids: frozenset[str]
|
||||
path_prefix: str
|
||||
filename: str
|
||||
decode: Callable[[str], Log]
|
||||
remote_revs: dict[str, str]
|
||||
state: SyncState
|
||||
|
||||
|
||||
def _pull_remote_logs(
|
||||
ctx: _PullContext,
|
||||
seen_revs: dict[str, str],
|
||||
) -> list[Log]:
|
||||
"""Return every other device's last-pushed log, skipping this one.
|
||||
|
||||
A device whose pushed file is corrupt or unparsable (e.g. an interrupted
|
||||
push) is logged and skipped, same as one that has never pushed at all --
|
||||
the remote is an external system boundary, and one bad device's file must
|
||||
not stall merging in every other device's.
|
||||
|
||||
``own_ids`` is a *set* rather than a single id because a device that has
|
||||
migrated from a role constant to a persisted uuid still owns the log it
|
||||
pushed under the old id. Matching only the current id would pull that
|
||||
file back and re-merge this device's own pre-migration history as though
|
||||
a peer had written it -- idempotent, but pure wasted transfer.
|
||||
|
||||
Records into ``seen_revs`` which peers are now merged in, so the next
|
||||
tick can skip them.
|
||||
"""
|
||||
remote_logs: list[Log] = []
|
||||
for other_device_id in ctx.client.list_directory(ctx.path_prefix):
|
||||
if other_device_id in ctx.own_ids:
|
||||
continue
|
||||
remote_rev = ctx.remote_revs.get(other_device_id)
|
||||
if remote_rev is not None and remote_rev == ctx.state.peer_revs.get(
|
||||
other_device_id
|
||||
):
|
||||
# Unchanged since we last merged it, and that merge is already
|
||||
# part of local_log -- so the (potentially hundreds of KB)
|
||||
# download is pure waste. Carry the revision forward so it stays
|
||||
# skipped next tick.
|
||||
seen_revs[other_device_id] = remote_rev
|
||||
continue
|
||||
text = ctx.client.get_file_text(
|
||||
f"{ctx.path_prefix}/{other_device_id}/{ctx.filename}"
|
||||
)
|
||||
if text is None:
|
||||
continue
|
||||
try:
|
||||
remote_logs.append(ctx.decode(text))
|
||||
except ValueError, KeyError, TypeError:
|
||||
_logger.warning(
|
||||
"Unparsable log pushed by device %r, skipping",
|
||||
other_device_id,
|
||||
)
|
||||
# Deliberately not recorded as seen: a corrupt push must be
|
||||
# retried next tick, not remembered as merged.
|
||||
continue
|
||||
seen_revs[other_device_id] = remote_rev or revision_of(text)
|
||||
return remote_logs
|
||||
@ -1,4 +1,3 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Record: a per-field LWW map with a sticky (monotonic) delete flag.
|
||||
|
||||
This is the unit CRDT merges operate on. It's deliberately domain-agnostic --
|
||||
|
||||
@ -1,83 +0,0 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Backend-neutral remote-storage contract that sync talks through.
|
||||
|
||||
Deliberately tiny: :func:`crdt_sync.sync_log` and every app-level sync
|
||||
service talk only to :class:`RemoteStore`, so swapping the storage backend
|
||||
(GitHub Contents API, Firebase Realtime Database, a dual-writing mirror of
|
||||
both) is a constructor change at the call site and nothing more.
|
||||
|
||||
Mirrors ``crdt_sync_dart``'s ``lib/src/remote_store.dart``; keep the two in
|
||||
step.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
|
||||
class RemoteSyncError(Exception):
|
||||
"""Raised for a remote-storage failure the caller must not ignore.
|
||||
|
||||
The backend-neutral base of the error hierarchy: catch this to handle
|
||||
"the sync transport failed" regardless of which backend is configured.
|
||||
Backends narrow it -- :class:`crdt_sync.GitHubSyncError` and
|
||||
:class:`crdt_sync.FirebaseSyncError` both derive from it -- so existing
|
||||
``except GitHubSyncError`` handlers keep working while a caller that has
|
||||
been migrated can catch the base type instead.
|
||||
"""
|
||||
|
||||
|
||||
class RemoteNotFoundError(RemoteSyncError):
|
||||
"""Raised when the configured remote itself is unreachable.
|
||||
|
||||
Distinguished from a missing *path* (nothing pushed there yet, which is
|
||||
benign -- it just means no other device has synced before) so the caller
|
||||
can tell "the repo/database is wrong or the credential isn't scoped to
|
||||
it" apart from "no other device has synced yet".
|
||||
"""
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class RemoteStore(Protocol):
|
||||
"""Dumb keyed storage of UTF-8 text blobs, with directory listing.
|
||||
|
||||
Implementations raise :class:`RemoteSyncError` (or a subclass) for any
|
||||
failure the caller must not silently ignore, and treat a missing path as
|
||||
a benign ``None`` / empty list rather than an error.
|
||||
|
||||
A ``Protocol`` rather than an ABC so the existing
|
||||
:class:`crdt_sync.GitHubSyncClient` satisfies it structurally, with no
|
||||
base class to add and no change to its own tests.
|
||||
"""
|
||||
|
||||
def list_directory(self, path: str) -> list[str]:
|
||||
"""Return the entry names directly under ``path``.
|
||||
|
||||
Returns an empty list when nothing has been written there yet.
|
||||
Includes both files and subdirectories: the
|
||||
``<path_prefix>/<device_id>/<filename>`` layout needs to discover
|
||||
device *directories*, not just files.
|
||||
"""
|
||||
|
||||
def get_file_text(self, path: str) -> str | None:
|
||||
"""Return the text stored at ``path``, or ``None`` if absent."""
|
||||
|
||||
def put_file_text(self, path: str, text: str, *, message: str) -> None:
|
||||
"""Create or replace the text at ``path``.
|
||||
|
||||
``message`` is a human-readable reason for the write. Backends that
|
||||
record one (GitHub commits) use it; backends that do not (Firebase)
|
||||
ignore it.
|
||||
"""
|
||||
|
||||
def delete_file(self, path: str, *, message: str = ...) -> None:
|
||||
"""Delete ``path``. A no-op when ``path`` does not exist."""
|
||||
|
||||
def can_access_remote(self) -> bool:
|
||||
"""Return whether the configured credential can reach the remote.
|
||||
|
||||
A lightweight connection test for a settings "Test connection"
|
||||
button: it probes the remote root, so it succeeds even before any
|
||||
file has been pushed. Never raises -- a network failure or missing
|
||||
remote returns ``False``.
|
||||
"""
|
||||
@ -1,38 +0,0 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""The revision string a device publishes alongside its log.
|
||||
|
||||
Split from :mod:`crdt_sync._sync` so both the push and pull halves can import
|
||||
it without importing each other. Pure: a revision is a function of the encoded
|
||||
log and nothing else, which is what lets two devices agree on whether they are
|
||||
looking at the same bytes.
|
||||
|
||||
Re-exported from :mod:`crdt_sync._sync`, so existing imports keep working.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
|
||||
def revision_of(encoded_log: str) -> str:
|
||||
"""Return the revision of a serialized log: a content hash.
|
||||
|
||||
A hash rather than the log's maximum HLC because merging a peer's *older*
|
||||
record changes the content without raising the maximum -- with three
|
||||
devices in one namespace (diet-guard has ``pc``, ``phone`` and
|
||||
``desktop``) a clock-based revision can miss a peer's merged state. It is
|
||||
also the same value used to suppress no-op pushes, so the two
|
||||
optimisations share one mechanism.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
encoded_log : str
|
||||
The serialized log, exactly as it would be pushed.
|
||||
|
||||
Returns:
|
||||
-------
|
||||
str
|
||||
A hex SHA-256 digest.
|
||||
|
||||
"""
|
||||
return hashlib.sha256(encoded_log.encode("utf-8")).hexdigest()
|
||||
@ -1,4 +1,3 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Local JSON persistence for a ``Log``.
|
||||
|
||||
Mirrors ``crdt_sync_dart``'s ``logToJson``/``FileLogPersistence`` serialization
|
||||
@ -54,7 +53,7 @@ def read_log(path: Path) -> Log:
|
||||
return {}
|
||||
try:
|
||||
return load_log(text)
|
||||
except ValueError, KeyError, TypeError:
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return {}
|
||||
|
||||
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Domain-agnostic pull/merge/push sync orchestration.
|
||||
|
||||
Generalizes diet_guard's original ``_sync.py`` (pull every other device's
|
||||
@ -6,179 +5,102 @@ pushed log, merge with the local one, push this device's own merged result
|
||||
back up) so any app can reuse the loop while keeping its own on-disk JSON
|
||||
shape via the ``encode``/``decode`` callbacks -- this module has no opinion
|
||||
on what a ``Record``'s fields actually mean.
|
||||
|
||||
Mirrors ``crdt_sync_dart``'s ``lib/src/sync.dart``; keep the two in step.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from crdt_sync._log import merge_logs
|
||||
from crdt_sync._pull import _pull_remote_logs, _PullContext, _remote_revs
|
||||
from crdt_sync._revisions import revision_of
|
||||
from crdt_sync._syncargs import LogCodec, RevisionTracking, SyncTarget
|
||||
from crdt_sync._syncstate import (
|
||||
FileSyncStateStore,
|
||||
MemorySyncStateStore,
|
||||
SyncState,
|
||||
SyncStateStore,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from crdt_sync._log import Log
|
||||
from collections.abc import Callable
|
||||
|
||||
# Named explicitly so the autofixer cannot prune an import that exists for its
|
||||
# re-export: crdt_sync/__init__.py imports all of these from here.
|
||||
__all__ = [
|
||||
"FileSyncStateStore",
|
||||
"LogCodec",
|
||||
"MemorySyncStateStore",
|
||||
"RevisionTracking",
|
||||
"SyncState",
|
||||
"SyncStateStore",
|
||||
"SyncTarget",
|
||||
"default_revs_path",
|
||||
"revision_of",
|
||||
"sync_log",
|
||||
]
|
||||
from crdt_sync._github import GitHubSyncClient
|
||||
from crdt_sync._log import Log
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_FILENAME = "log.json"
|
||||
|
||||
def default_revs_path(path_prefix: str) -> str:
|
||||
"""Return where revisions live for ``path_prefix``.
|
||||
|
||||
A ``revs`` sibling of the device directory, e.g.
|
||||
``diet-guard-sync/devices`` -> ``diet-guard-sync/revs`` and
|
||||
``todo-sync/notes`` -> ``todo-sync/revs``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path_prefix : str
|
||||
The directory holding one subdirectory per device.
|
||||
|
||||
Returns:
|
||||
-------
|
||||
str
|
||||
The revisions directory.
|
||||
def _pull_remote_logs(
|
||||
client: GitHubSyncClient,
|
||||
device_id: str,
|
||||
path_prefix: str,
|
||||
filename: str,
|
||||
decode: Callable[[str], Log],
|
||||
) -> list[Log]:
|
||||
"""Return every other device's last-pushed log, skipping this one.
|
||||
|
||||
A device whose pushed file is corrupt or unparsable (e.g. an interrupted
|
||||
push) is logged and skipped, same as one that has never pushed at all --
|
||||
GitHub is an external system boundary, and one bad device's file must not
|
||||
stall merging in every other device's.
|
||||
"""
|
||||
head, separator, _ = path_prefix.rpartition("/")
|
||||
return f"{head}/revs" if separator else f"{path_prefix}/revs"
|
||||
remote_logs: list[Log] = []
|
||||
for other_device_id in client.list_directory(path_prefix):
|
||||
if other_device_id == device_id:
|
||||
continue
|
||||
text = client.get_file_text(f"{path_prefix}/{other_device_id}/{filename}")
|
||||
if text is None:
|
||||
continue
|
||||
try:
|
||||
remote_logs.append(decode(text))
|
||||
except (ValueError, KeyError, TypeError):
|
||||
_logger.warning(
|
||||
"Unparsable log pushed by device %r, skipping",
|
||||
other_device_id,
|
||||
)
|
||||
return remote_logs
|
||||
|
||||
|
||||
def sync_log(
|
||||
target: SyncTarget,
|
||||
*,
|
||||
client: GitHubSyncClient,
|
||||
device_id: str,
|
||||
path_prefix: str,
|
||||
local_log: Log,
|
||||
codec: LogCodec,
|
||||
revisions: RevisionTracking | None = None,
|
||||
encode: Callable[[Log], str],
|
||||
decode: Callable[[str], Log],
|
||||
filename: str = _DEFAULT_FILENAME,
|
||||
commit_message: str = "crdt_sync: update log",
|
||||
) -> Log:
|
||||
"""Run one full sync tick: pull every other device's log, merge, push.
|
||||
|
||||
Pulls from ``<path_prefix>/<other-device-id>/<filename>`` for every
|
||||
device directory the remote reports under ``path_prefix``, merges each
|
||||
into ``local_log`` with :func:`crdt_sync.merge_logs`, then pushes this
|
||||
device directory GitHub reports under ``path_prefix``, merges each into
|
||||
``local_log`` with :func:`crdt_sync.merge_logs`, then pushes this
|
||||
device's own merged result to ``<path_prefix>/<device_id>/<filename>``.
|
||||
|
||||
Args:
|
||||
target: Which remote and device this tick is for.
|
||||
client: An authenticated :class:`GitHubSyncClient`.
|
||||
device_id: This device's identifier; also the directory name its own
|
||||
log is pushed under.
|
||||
path_prefix: The repo-relative directory holding one subdirectory
|
||||
per device (e.g. ``"devices"``).
|
||||
local_log: This device's current full log (including tombstones).
|
||||
codec: How to serialize and parse a log.
|
||||
revisions: Pass one to skip downloading peers that have not changed
|
||||
and skip pushing a log that has not changed. ``None`` fetches
|
||||
everything and pushes unconditionally.
|
||||
encode: Serializes a merged log for pushing.
|
||||
decode: Parses a remote device's pushed text back into a log.
|
||||
Raising ``ValueError``, ``KeyError``, or ``TypeError`` is treated
|
||||
as a corrupt/unparsable push, and that device is skipped for
|
||||
this tick rather than aborting the whole sync.
|
||||
filename: The file name each device pushes its log as.
|
||||
commit_message: The commit message used for this device's push.
|
||||
|
||||
Returns:
|
||||
The merged log, as pushed.
|
||||
|
||||
Notes:
|
||||
Revisions live at ``<revs_path>/<device_id>`` -- one small text node
|
||||
per device, each written only by its owner. Deliberately *not* one
|
||||
shared map per namespace: a whole-map write would erase every other
|
||||
device's entry, after which those peers would look permanently
|
||||
unchanged and never be fetched again. Per-device keys make that
|
||||
failure impossible rather than merely avoided.
|
||||
"""
|
||||
tracking = revisions if revisions is not None else RevisionTracking()
|
||||
state_store = tracking.state_store
|
||||
revs = (
|
||||
tracking.revs_path
|
||||
if tracking.revs_path is not None
|
||||
else default_revs_path(target.path_prefix)
|
||||
)
|
||||
state = state_store.load() if state_store is not None else SyncState()
|
||||
remote_revs = _remote_revs(target.client, revs) if state_store is not None else {}
|
||||
|
||||
seen_revs: dict[str, str] = {}
|
||||
merged = _merge_peers(
|
||||
_PullContext(
|
||||
client=target.client,
|
||||
own_ids=target.own_ids,
|
||||
path_prefix=target.path_prefix,
|
||||
filename=codec.filename,
|
||||
decode=codec.decode,
|
||||
remote_revs=remote_revs,
|
||||
state=state,
|
||||
),
|
||||
local_log,
|
||||
seen_revs,
|
||||
)
|
||||
|
||||
encoded = codec.encode(merged)
|
||||
revision = revision_of(encoded)
|
||||
if not (state_store is not None and revision == state.pushed_rev):
|
||||
_push(
|
||||
target,
|
||||
codec,
|
||||
encoded,
|
||||
_RevisionMarker(revs, revision) if state_store is not None else None,
|
||||
)
|
||||
if state_store is not None:
|
||||
state_store.save(SyncState(pushed_rev=revision, peer_revs=seen_revs))
|
||||
return merged
|
||||
|
||||
|
||||
def _merge_peers(
|
||||
ctx: _PullContext,
|
||||
local_log: Log,
|
||||
seen_revs: dict[str, str],
|
||||
) -> Log:
|
||||
"""Return ``local_log`` with every peer's pushed log merged in."""
|
||||
merged = dict(local_log)
|
||||
for remote_log in _pull_remote_logs(ctx, seen_revs):
|
||||
for remote_log in _pull_remote_logs(
|
||||
client, device_id, path_prefix, filename, decode
|
||||
):
|
||||
merged = merge_logs(merged, remote_log)
|
||||
return merged
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _RevisionMarker:
|
||||
"""The revision node a tracked push publishes after its log."""
|
||||
|
||||
revs_path: str
|
||||
revision: str
|
||||
|
||||
|
||||
def _push(
|
||||
target: SyncTarget,
|
||||
codec: LogCodec,
|
||||
encoded: str,
|
||||
marker: _RevisionMarker | None,
|
||||
) -> None:
|
||||
"""Push this device's merged log, then its revision marker."""
|
||||
target.client.put_file_text(
|
||||
f"{target.path_prefix}/{target.device_id}/{codec.filename}",
|
||||
encoded,
|
||||
message=codec.commit_message,
|
||||
client.put_file_text(
|
||||
f"{path_prefix}/{device_id}/{filename}",
|
||||
encode(merged),
|
||||
message=commit_message,
|
||||
)
|
||||
if marker is not None:
|
||||
# Published after the log, never before: a peer that cached
|
||||
# "seen rev X" against a log it never received would skip it
|
||||
# forever.
|
||||
target.client.put_file_text(
|
||||
f"{marker.revs_path}/{target.device_id}",
|
||||
marker.revision,
|
||||
message=codec.commit_message,
|
||||
)
|
||||
return merged
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user