mirror of
https://github.com/kuhyx/utils.git
synced 2026-11-12 11:11:03 +01:00
Compare commits
140 Commits
crdt_sync_
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 3174480624 | |||
| 3225af8365 | |||
| 6a40f7530b | |||
| edd0f7a788 | |||
| ba58ac41e9 | |||
| 90286f6cb2 | |||
| 9102c4b6ff | |||
| 47f2d239c4 | |||
| c8bb0808aa | |||
| e5990c9c15 | |||
| 84343aec2d | |||
| db1ce8c378 | |||
| 0ef5461805 | |||
| eb9bdc5796 | |||
| 307248b60b | |||
| 33791da6ff | |||
| eebeb0b510 | |||
| 2813a88db6 | |||
| cc09e56cd2 | |||
| 51e562a764 | |||
| b0fad2b2e5 | |||
| c83d41a594 | |||
| 2bbfbd9439 | |||
| 2d341dd41e | |||
| 3084080d74 | |||
| 9d6b17fd28 | |||
| ae56037e41 | |||
| eef7af4c86 | |||
| 9daa376254 | |||
| 17e300af0a | |||
| afcc1a47c7 | |||
| 0527bc680d | |||
| 21a0b5ce1c | |||
| 4c1ffd900b | |||
| 5a87868763 | |||
| fe4e34287a | |||
| 2f4f79bd6e | |||
| 51b747a364 | |||
| 89f464828a | |||
| eaf7282ac1 | |||
| 6641d85f05 | |||
| 46ab26fa15 | |||
| f71b0532fd | |||
| 3017bbf220 | |||
| fd81332f27 | |||
| ebedbf4b27 | |||
| d4fca83c9f | |||
| 3121639732 | |||
| 59b33949e4 | |||
| 027e67b197 | |||
| 8dfc4fb55f | |||
| 4fee294aa0 | |||
| 1a74345c40 | |||
| 71ea92f3c2 | |||
| 7f3f79f58d | |||
| 97d043d424 | |||
| 7af6afe51a | |||
| c2165e9fb8 | |||
| 26079c34c2 | |||
| 22fdf2c344 | |||
| de7d5d17eb | |||
| 640985a421 | |||
| 70c8aa8fbb | |||
| ba4dd58545 | |||
| b50e245ceb | |||
| 88e4be5042 | |||
| 737bf624ca | |||
| 716231a7f3 | |||
| 11cb1d81d9 | |||
| e0c9befdeb | |||
| 612b0b3c0a | |||
| f865508eb4 | |||
| 1fb04ce59d | |||
| 6f0070b353 | |||
| aec96eefd3 | |||
| 35569a6335 | |||
| a233268b66 | |||
| 20a825e249 | |||
| 2bbb428f2c | |||
| af3c98a50e | |||
| fd013abf67 | |||
| 0f332c9057 | |||
| 86b2621cd4 | |||
| fe1f4a22ab | |||
| db70ba204d | |||
| 5b36e84793 | |||
| ae29e67e28 | |||
| 4d75ce6d48 | |||
| 637428585c | |||
| c25ae4e0d1 | |||
| 98c013ea3f | |||
| 213537d0ef | |||
| 11c5b2e32b | |||
| c98da93339 | |||
| d45b1086c3 | |||
| 8398ca4823 | |||
| 087ed1c400 | |||
| e852e3ce48 | |||
| 6d80efb50a | |||
| cbcee98d94 | |||
| 82119bdd47 | |||
| d666e47e16 | |||
| 8a3076783e | |||
| 5d11f97093 | |||
| eb1c015315 | |||
| f99b77af89 | |||
| 1e96563522 | |||
| 28d3fb1361 | |||
| bce59127eb | |||
| 1bfacdf51f | |||
| c74815b8ab | |||
| dccfe14ae9 | |||
| 343f015c81 | |||
| decf5824e6 | |||
| 648bde44b1 | |||
| bbb7f0f0c0 | |||
| 4c417472ec | |||
| f9899b9c56 | |||
| 254c8ce240 | |||
| 5be9a22970 | |||
| 888e8ae57d | |||
| a27bbec59a | |||
| 1c9b47e35c | |||
| b3c9611af2 | |||
| def48e346b | |||
| 99f4e171e3 | |||
| 2c926928dc | |||
| a78c1e971b | |||
| 2531f9eeb3 | |||
| f47ed80584 | |||
| 2a9732a312 | |||
| e06df74caf | |||
| 2d2d8a41f8 | |||
| 6343499ed1 | |||
| ebf7ca624e | |||
| c22b14389a | |||
| c0e902e5cc | |||
| 28c243d066 | |||
| 96a0d8af07 | |||
| b730cffaf4 |
36
.github/workflows/data-transfer-flutter-tests.yml
vendored
Normal file
36
.github/workflows/data-transfer-flutter-tests.yml
vendored
Normal file
@ -0,0 +1,36 @@
|
||||
name: data_transfer_flutter tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['data_transfer_flutter/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['data_transfer_flutter/**']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: data_transfer_flutter
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Flutter
|
||||
uses: subosito/flutter-action@v2
|
||||
with:
|
||||
channel: stable
|
||||
|
||||
- name: Install dependencies
|
||||
run: flutter pub get
|
||||
|
||||
- name: Analyze
|
||||
run: flutter analyze
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: flutter test --coverage
|
||||
|
||||
- name: Enforce 100% line coverage
|
||||
run: scripts/check_coverage.sh coverage/lcov.info 100
|
||||
73
.github/workflows/focus-project-precommit.yml
vendored
Normal file
73
.github/workflows/focus-project-precommit.yml
vendored
Normal file
@ -0,0 +1,73 @@
|
||||
name: focus-project pre-commit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as focus_project: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['focus_project/**', '.github/workflows/focus-project-precommit.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['focus_project/**', '.github/workflows/focus-project-precommit.yml']
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
# Only pre-commit itself. Every linting hook runs in its own isolated
|
||||
# env -- ruff, mypy, pylint, bandit and codespell each come from their
|
||||
# own repo+rev, and the local hooks need nothing beyond bash and
|
||||
# python3 -- so installing the full requirements.txt spent 9s a run on
|
||||
# packages no pre-commit-stage hook imports. Verified by running all
|
||||
# three configs green in a venv holding pre-commit and nothing else.
|
||||
# The pin is read out of requirements.txt rather than repeated here, so
|
||||
# it cannot drift from what the dependency-freshness gate governs.
|
||||
- name: Install pre-commit
|
||||
run: |
|
||||
pip install "$(grep '^pre-commit==' focus_project/requirements.txt)"
|
||||
|
||||
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
|
||||
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
|
||||
# package jobs shared one key, raced to overwrite it with hook sets built
|
||||
# from four different configs, and editing a per-package config did not
|
||||
# invalidate it. The cache was therefore silently wrong rather than
|
||||
# missing, and the hooks reinstalled from scratch on every run (38-49s).
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: >-
|
||||
pre-commit-focus-project-${{ env.pythonLocation }}-${{
|
||||
hashFiles('focus_project/.pre-commit-config.yaml') }}
|
||||
# Prefix fallback, or this fix would evaporate on the very
|
||||
# event that motivated it: bumping a hook rev changes the
|
||||
# exact key, and a total miss means reinstalling every hook
|
||||
# again. pre-commit's store is keyed internally by hook
|
||||
# repo+rev, so a superseded entry is ignored disk rather
|
||||
# than a wrong hook, and cache still re-saves under the new
|
||||
# exact key after a prefix-only hit.
|
||||
restore-keys: pre-commit-focus-project-${{ env.pythonLocation }}-
|
||||
# mypy re-analyses the stdlib and every third-party stub from scratch
|
||||
# without this, which measured 10s of the hook step in CI against 0.25s
|
||||
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
|
||||
# the config, not the sources: mypy's own cache metadata records the
|
||||
# interpreter, the flag set and a hash per source file, so it re-checks
|
||||
# what changed and cannot pass stale code. Deliberately NOT keyed on
|
||||
# github.sha -- that re-saves 19MB x3 every commit and would evict the
|
||||
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
|
||||
# the expensive part, the dependency analysis, which is what costs 10s.
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: >-
|
||||
mypy-focus-project-${{ env.pythonLocation }}-${{
|
||||
hashFiles('focus_project/.pre-commit-config.yaml') }}
|
||||
restore-keys: mypy-focus-project-${{ env.pythonLocation }}-
|
||||
- name: Run the hooks
|
||||
run: >-
|
||||
pre-commit run --config focus_project/.pre-commit-config.yaml
|
||||
--all-files --show-diff-on-failure
|
||||
34
.github/workflows/focus-project-tests.yml
vendored
Normal file
34
.github/workflows/focus-project-tests.yml
vendored
Normal file
@ -0,0 +1,34 @@
|
||||
name: focus-project tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as focus_project: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['focus_project/**', '.github/workflows/focus-project-tests.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['focus_project/**', '.github/workflows/focus-project-tests.yml']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: focus_project
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: focus_project/requirements.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt && pip install --no-deps -e .
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: python -m pytest -v
|
||||
36
.github/workflows/google-account-token-tests.yml
vendored
Normal file
36
.github/workflows/google-account-token-tests.yml
vendored
Normal file
@ -0,0 +1,36 @@
|
||||
name: google_account_token tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths: ['google_account_token/**']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['google_account_token/**']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: google_account_token
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Flutter
|
||||
uses: subosito/flutter-action@v2
|
||||
with:
|
||||
channel: stable
|
||||
|
||||
- name: Install dependencies
|
||||
run: flutter pub get
|
||||
|
||||
- name: Analyze
|
||||
run: flutter analyze
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: flutter test --coverage
|
||||
|
||||
- name: Enforce 100% line coverage
|
||||
run: scripts/check_coverage.sh coverage/lcov.info 100
|
||||
73
.github/workflows/holds-precommit.yml
vendored
Normal file
73
.github/workflows/holds-precommit.yml
vendored
Normal file
@ -0,0 +1,73 @@
|
||||
name: holds pre-commit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as holds: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['holds/**', '.github/workflows/holds-precommit.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['holds/**', '.github/workflows/holds-precommit.yml']
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
# Only pre-commit itself. Every linting hook runs in its own isolated
|
||||
# env -- ruff, mypy, pylint, bandit and codespell each come from their
|
||||
# own repo+rev, and the local hooks need nothing beyond bash and
|
||||
# python3 -- so installing the full requirements.txt spent 9s a run on
|
||||
# packages no pre-commit-stage hook imports. Verified by running all
|
||||
# three configs green in a venv holding pre-commit and nothing else.
|
||||
# The pin is read out of requirements.txt rather than repeated here, so
|
||||
# it cannot drift from what the dependency-freshness gate governs.
|
||||
- name: Install pre-commit
|
||||
run: |
|
||||
pip install "$(grep '^pre-commit==' holds/requirements.txt)"
|
||||
|
||||
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
|
||||
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
|
||||
# package jobs shared one key, raced to overwrite it with hook sets built
|
||||
# from four different configs, and editing a per-package config did not
|
||||
# invalidate it. The cache was therefore silently wrong rather than
|
||||
# missing, and the hooks reinstalled from scratch on every run (38-49s).
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: >-
|
||||
pre-commit-holds-${{ env.pythonLocation }}-${{
|
||||
hashFiles('holds/.pre-commit-config.yaml') }}
|
||||
# Prefix fallback, or this fix would evaporate on the very
|
||||
# event that motivated it: bumping a hook rev changes the
|
||||
# exact key, and a total miss means reinstalling every hook
|
||||
# again. pre-commit's store is keyed internally by hook
|
||||
# repo+rev, so a superseded entry is ignored disk rather
|
||||
# than a wrong hook, and cache still re-saves under the new
|
||||
# exact key after a prefix-only hit.
|
||||
restore-keys: pre-commit-holds-${{ env.pythonLocation }}-
|
||||
# mypy re-analyses the stdlib and every third-party stub from scratch
|
||||
# without this, which measured 10s of the hook step in CI against 0.25s
|
||||
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
|
||||
# the config, not the sources: mypy's own cache metadata records the
|
||||
# interpreter, the flag set and a hash per source file, so it re-checks
|
||||
# what changed and cannot pass stale code. Deliberately NOT keyed on
|
||||
# github.sha -- that re-saves 19MB x3 every commit and would evict the
|
||||
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
|
||||
# the expensive part, the dependency analysis, which is what costs 10s.
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: >-
|
||||
mypy-holds-${{ env.pythonLocation }}-${{
|
||||
hashFiles('holds/.pre-commit-config.yaml') }}
|
||||
restore-keys: mypy-holds-${{ env.pythonLocation }}-
|
||||
- name: Run the hooks
|
||||
run: >-
|
||||
pre-commit run --config holds/.pre-commit-config.yaml
|
||||
--all-files --show-diff-on-failure
|
||||
34
.github/workflows/holds-tests.yml
vendored
Normal file
34
.github/workflows/holds-tests.yml
vendored
Normal file
@ -0,0 +1,34 @@
|
||||
name: holds tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as holds: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['holds/**', '.github/workflows/holds-tests.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['holds/**', '.github/workflows/holds-tests.yml']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: holds
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: holds/requirements.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt && pip install --no-deps -e .
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: python -m pytest -v
|
||||
73
.github/workflows/leetcode-coach-precommit.yml
vendored
Normal file
73
.github/workflows/leetcode-coach-precommit.yml
vendored
Normal file
@ -0,0 +1,73 @@
|
||||
name: leetcode-coach pre-commit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as leetcode_coach: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['leetcode_coach/**', '.github/workflows/leetcode-coach-precommit.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['leetcode_coach/**', '.github/workflows/leetcode-coach-precommit.yml']
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
# Only pre-commit itself. Every linting hook runs in its own isolated
|
||||
# env -- ruff, mypy, pylint, bandit and codespell each come from their
|
||||
# own repo+rev, and the local hooks need nothing beyond bash and
|
||||
# python3 -- so installing the full requirements.txt spent 9s a run on
|
||||
# packages no pre-commit-stage hook imports. Verified by running all
|
||||
# three configs green in a venv holding pre-commit and nothing else.
|
||||
# The pin is read out of requirements.txt rather than repeated here, so
|
||||
# it cannot drift from what the dependency-freshness gate governs.
|
||||
- name: Install pre-commit
|
||||
run: |
|
||||
pip install "$(grep '^pre-commit==' leetcode_coach/requirements.txt)"
|
||||
|
||||
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
|
||||
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
|
||||
# package jobs shared one key, raced to overwrite it with hook sets built
|
||||
# from four different configs, and editing a per-package config did not
|
||||
# invalidate it. The cache was therefore silently wrong rather than
|
||||
# missing, and the hooks reinstalled from scratch on every run (38-49s).
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: >-
|
||||
pre-commit-leetcode-coach-${{ env.pythonLocation }}-${{
|
||||
hashFiles('leetcode_coach/.pre-commit-config.yaml') }}
|
||||
# Prefix fallback, or this fix would evaporate on the very
|
||||
# event that motivated it: bumping a hook rev changes the
|
||||
# exact key, and a total miss means reinstalling every hook
|
||||
# again. pre-commit's store is keyed internally by hook
|
||||
# repo+rev, so a superseded entry is ignored disk rather
|
||||
# than a wrong hook, and cache still re-saves under the new
|
||||
# exact key after a prefix-only hit.
|
||||
restore-keys: pre-commit-leetcode-coach-${{ env.pythonLocation }}-
|
||||
# mypy re-analyses the stdlib and every third-party stub from scratch
|
||||
# without this, which measured 10s of the hook step in CI against 0.25s
|
||||
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
|
||||
# the config, not the sources: mypy's own cache metadata records the
|
||||
# interpreter, the flag set and a hash per source file, so it re-checks
|
||||
# what changed and cannot pass stale code. Deliberately NOT keyed on
|
||||
# github.sha -- that re-saves 19MB x3 every commit and would evict the
|
||||
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
|
||||
# the expensive part, the dependency analysis, which is what costs 10s.
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: >-
|
||||
mypy-leetcode-coach-${{ env.pythonLocation }}-${{
|
||||
hashFiles('leetcode_coach/.pre-commit-config.yaml') }}
|
||||
restore-keys: mypy-leetcode-coach-${{ env.pythonLocation }}-
|
||||
- name: Run the hooks
|
||||
run: >-
|
||||
pre-commit run --config leetcode_coach/.pre-commit-config.yaml
|
||||
--all-files --show-diff-on-failure
|
||||
34
.github/workflows/leetcode-coach-tests.yml
vendored
Normal file
34
.github/workflows/leetcode-coach-tests.yml
vendored
Normal file
@ -0,0 +1,34 @@
|
||||
name: leetcode-coach tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as leetcode_coach: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['leetcode_coach/**', '.github/workflows/leetcode-coach-tests.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['leetcode_coach/**', '.github/workflows/leetcode-coach-tests.yml']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: leetcode_coach
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: leetcode_coach/requirements.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt && pip install --no-deps -e .
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: python -m pytest -v
|
||||
7
.github/workflows/md-naming.yml
vendored
7
.github/workflows/md-naming.yml
vendored
@ -1,6 +1,6 @@
|
||||
name: md-naming
|
||||
|
||||
# The four markdown namespaces, enforced on push as well as at commit time.
|
||||
# The markdown namespaces (README / CLAUDE* / DOCS*, no TODO*), enforced on push as well as at commit time.
|
||||
# The local pre-commit hook can be skipped (a fresh clone that never ran
|
||||
# `pre-commit install` has no hooks at all), so the convention needs a second
|
||||
# gate that no one can forget to install.
|
||||
@ -31,8 +31,3 @@ jobs:
|
||||
|
||||
- name: Enforce the naming convention
|
||||
run: bash .utils/scripts/check_md_naming.sh --all
|
||||
|
||||
# Warns only: no machine can prove a task is finished, so an old TODO
|
||||
# gets a nudge, never a failure.
|
||||
- name: Warn about stale TODOs
|
||||
run: PYTHONPATH=.utils python3 .utils/md_naming/stale.py
|
||||
|
||||
2
.github/workflows/music-theory-tests.yml
vendored
2
.github/workflows/music-theory-tests.yml
vendored
@ -28,4 +28,4 @@ jobs:
|
||||
run: pip install -r requirements.txt
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: python -m pytest -v
|
||||
run: python -m pytest -v --cov=music_theory
|
||||
|
||||
73
.github/workflows/omniroute_offload-precommit.yml
vendored
Normal file
73
.github/workflows/omniroute_offload-precommit.yml
vendored
Normal file
@ -0,0 +1,73 @@
|
||||
name: omniroute_offload pre-commit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as omniroute_offload: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['omniroute_offload/**', '.github/workflows/omniroute_offload-precommit.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['omniroute_offload/**', '.github/workflows/omniroute_offload-precommit.yml']
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
# Only pre-commit itself. Every linting hook runs in its own isolated
|
||||
# env -- ruff, mypy, pylint, bandit and codespell each come from their
|
||||
# own repo+rev, and the local hooks need nothing beyond bash and
|
||||
# python3 -- so installing the full requirements.txt spent 9s a run on
|
||||
# packages no pre-commit-stage hook imports. Verified by running all
|
||||
# three configs green in a venv holding pre-commit and nothing else.
|
||||
# The pin is read out of requirements.txt rather than repeated here, so
|
||||
# it cannot drift from what the dependency-freshness gate governs.
|
||||
- name: Install pre-commit
|
||||
run: |
|
||||
pip install "$(grep '^pre-commit==' omniroute_offload/requirements.txt)"
|
||||
|
||||
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
|
||||
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
|
||||
# package jobs shared one key, raced to overwrite it with hook sets built
|
||||
# from four different configs, and editing a per-package config did not
|
||||
# invalidate it. The cache was therefore silently wrong rather than
|
||||
# missing, and the hooks reinstalled from scratch on every run (38-49s).
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: >-
|
||||
pre-commit-omniroute_offload-${{ env.pythonLocation }}-${{
|
||||
hashFiles('omniroute_offload/.pre-commit-config.yaml') }}
|
||||
# Prefix fallback, or this fix would evaporate on the very
|
||||
# event that motivated it: bumping a hook rev changes the
|
||||
# exact key, and a total miss means reinstalling every hook
|
||||
# again. pre-commit's store is keyed internally by hook
|
||||
# repo+rev, so a superseded entry is ignored disk rather
|
||||
# than a wrong hook, and cache still re-saves under the new
|
||||
# exact key after a prefix-only hit.
|
||||
restore-keys: pre-commit-omniroute_offload-${{ env.pythonLocation }}-
|
||||
# mypy re-analyses the stdlib and every third-party stub from scratch
|
||||
# without this, which measured 10s of the hook step in CI against 0.25s
|
||||
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
|
||||
# the config, not the sources: mypy's own cache metadata records the
|
||||
# interpreter, the flag set and a hash per source file, so it re-checks
|
||||
# what changed and cannot pass stale code. Deliberately NOT keyed on
|
||||
# github.sha -- that re-saves 19MB x3 every commit and would evict the
|
||||
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
|
||||
# the expensive part, the dependency analysis, which is what costs 10s.
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: >-
|
||||
mypy-omniroute_offload-${{ env.pythonLocation }}-${{
|
||||
hashFiles('omniroute_offload/.pre-commit-config.yaml') }}
|
||||
restore-keys: mypy-omniroute_offload-${{ env.pythonLocation }}-
|
||||
- name: Run the hooks
|
||||
run: >-
|
||||
pre-commit run --config omniroute_offload/.pre-commit-config.yaml
|
||||
--all-files --show-diff-on-failure
|
||||
5
.github/workflows/palette-drift.yml
vendored
5
.github/workflows/palette-drift.yml
vendored
@ -31,3 +31,8 @@ jobs:
|
||||
|
||||
- name: Categorical ramp still passes its constraints
|
||||
run: python3 unified-design-system/scripts/ramp_check.py
|
||||
|
||||
# Spacing, radius, type, shadow and motion: palette_check.py parses
|
||||
# hex only, so these scales were unguarded until structural_check.py.
|
||||
- name: Structural scales agree across all token sources
|
||||
run: python3 unified-design-system/scripts/structural_check.py
|
||||
|
||||
73
.github/workflows/sql-drills-precommit.yml
vendored
Normal file
73
.github/workflows/sql-drills-precommit.yml
vendored
Normal file
@ -0,0 +1,73 @@
|
||||
name: sql-drills pre-commit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as sql_drills: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['sql_drills/**', '.github/workflows/sql-drills-precommit.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['sql_drills/**', '.github/workflows/sql-drills-precommit.yml']
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
# Only pre-commit itself. Every linting hook runs in its own isolated
|
||||
# env -- ruff, mypy, pylint, bandit and codespell each come from their
|
||||
# own repo+rev, and the local hooks need nothing beyond bash and
|
||||
# python3 -- so installing the full requirements.txt spent 9s a run on
|
||||
# packages no pre-commit-stage hook imports. Verified by running all
|
||||
# three configs green in a venv holding pre-commit and nothing else.
|
||||
# The pin is read out of requirements.txt rather than repeated here, so
|
||||
# it cannot drift from what the dependency-freshness gate governs.
|
||||
- name: Install pre-commit
|
||||
run: |
|
||||
pip install "$(grep '^pre-commit==' sql_drills/requirements.txt)"
|
||||
|
||||
# Keyed on THIS package's config. pre-commit/action@v3.0.1 caches on
|
||||
# hashFiles('.pre-commit-config.yaml') -- the repo ROOT's -- so all three
|
||||
# package jobs shared one key, raced to overwrite it with hook sets built
|
||||
# from four different configs, and editing a per-package config did not
|
||||
# invalidate it. The cache was therefore silently wrong rather than
|
||||
# missing, and the hooks reinstalled from scratch on every run (38-49s).
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: >-
|
||||
pre-commit-sql-drills-${{ env.pythonLocation }}-${{
|
||||
hashFiles('sql_drills/.pre-commit-config.yaml') }}
|
||||
# Prefix fallback, or this fix would evaporate on the very
|
||||
# event that motivated it: bumping a hook rev changes the
|
||||
# exact key, and a total miss means reinstalling every hook
|
||||
# again. pre-commit's store is keyed internally by hook
|
||||
# repo+rev, so a superseded entry is ignored disk rather
|
||||
# than a wrong hook, and cache still re-saves under the new
|
||||
# exact key after a prefix-only hit.
|
||||
restore-keys: pre-commit-sql-drills-${{ env.pythonLocation }}-
|
||||
# mypy re-analyses the stdlib and every third-party stub from scratch
|
||||
# without this, which measured 10s of the hook step in CI against 0.25s
|
||||
# warm locally -- a 19MB cache dir does the whole difference. Keyed on
|
||||
# the config, not the sources: mypy's own cache metadata records the
|
||||
# interpreter, the flag set and a hash per source file, so it re-checks
|
||||
# what changed and cannot pass stale code. Deliberately NOT keyed on
|
||||
# github.sha -- that re-saves 19MB x3 every commit and would evict the
|
||||
# 68MB hook caches under the repo's 10GB LRU budget. A fixed key keeps
|
||||
# the expensive part, the dependency analysis, which is what costs 10s.
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: >-
|
||||
mypy-sql-drills-${{ env.pythonLocation }}-${{
|
||||
hashFiles('sql_drills/.pre-commit-config.yaml') }}
|
||||
restore-keys: mypy-sql-drills-${{ env.pythonLocation }}-
|
||||
- name: Run the hooks
|
||||
run: >-
|
||||
pre-commit run --config sql_drills/.pre-commit-config.yaml
|
||||
--all-files --show-diff-on-failure
|
||||
34
.github/workflows/sql-drills-tests.yml
vendored
Normal file
34
.github/workflows/sql-drills-tests.yml
vendored
Normal file
@ -0,0 +1,34 @@
|
||||
name: sql-drills tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# The workflow's own file counts as sql_drills: editing only this file would
|
||||
# otherwise trigger nothing, so a fix to the job itself could never prove
|
||||
# itself.
|
||||
paths: ['sql_drills/**', '.github/workflows/sql-drills-tests.yml']
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths: ['sql_drills/**', '.github/workflows/sql-drills-tests.yml']
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: sql_drills
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.x"
|
||||
cache: pip
|
||||
cache-dependency-path: sql_drills/requirements.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt && pip install --no-deps -e .
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: python -m pytest -v
|
||||
@ -13,7 +13,7 @@
|
||||
# linters, and carry this same hook so committing with one of those configs is
|
||||
# gated too. This root config is what `pre-commit install` wires up, and it is
|
||||
# the only one that covers the subprojects without a config of their own
|
||||
# (aur-publish/, guard-lib/, crdt_sync_dart/, file_length/, ...).
|
||||
# (guard-lib/, crdt_sync_dart/, file_length/, ...).
|
||||
#
|
||||
# The path is repo-relative on purpose. pre-commit always runs hooks from the
|
||||
# git root, so this resolves the same locally and on a CI runner -- an absolute
|
||||
@ -27,8 +27,8 @@ repos:
|
||||
language: system
|
||||
pass_filenames: true
|
||||
|
||||
# Four markdown namespaces (README / CLAUDE* / DOCS* / TODO*), plus the
|
||||
# rule that a TODO carries "REMOVE ME AFTER FINISH" and nothing else may.
|
||||
# Three markdown namespaces (README / CLAUDE* / DOCS*); TODO*.md is banned
|
||||
# (tasks live in the todo app).
|
||||
# Shared gate in ~/src/utils owns the namespaces and the exemption list, so
|
||||
# one repo's idea of a valid name cannot drift from every other repo's.
|
||||
# Since the 2026-09-11 reorganisation ~ holds eleven visible entries and
|
||||
@ -66,9 +66,14 @@ repos:
|
||||
# answer and passes with a loud banner; `--no-verify` is banned here, so
|
||||
# a hook that hard-failed on a flaky connection would leave no way to
|
||||
# commit at all. CI, always online, is the strict authority.
|
||||
#
|
||||
# Scoped to manifests this branch changed (vs merge-base with main), so
|
||||
# `--all-files` asks "did THIS change leave a stale pin?" and not "is
|
||||
# every project in the monorepo current?". CI `--all --strict` keeps
|
||||
# the repo-wide answer. See scripts/dep_freshness_changed.sh.
|
||||
- id: dependency-freshness
|
||||
name: dependencies on newest stable
|
||||
entry: bash scripts/check_dependency_freshness.sh
|
||||
entry: bash scripts/dep_freshness_changed.sh
|
||||
language: system
|
||||
pass_filenames: true
|
||||
files: >-
|
||||
|
||||
27
.pre-commit-hooks.yaml
Normal file
27
.pre-commit-hooks.yaml
Normal file
@ -0,0 +1,27 @@
|
||||
# Hooks other repos consume with `repo: https://github.com/kuhyx/utils`.
|
||||
# All three are post-* stages, so each clone must install them (see
|
||||
# todo_hooks/README.md). They never fail a git operation: the work has
|
||||
# already happened, so server/gh trouble prints one warning and exits 0.
|
||||
- id: todo-closer
|
||||
name: close Todo trailer tasks
|
||||
entry: scripts/todo_hooks.sh closer
|
||||
language: script
|
||||
stages: [post-commit]
|
||||
always_run: true
|
||||
pass_filenames: false
|
||||
|
||||
- id: todo-import-pr-merge
|
||||
name: import PR Todo sections (pull)
|
||||
entry: scripts/todo_hooks.sh import-pr post-merge
|
||||
language: script
|
||||
stages: [post-merge]
|
||||
always_run: true
|
||||
pass_filenames: false
|
||||
|
||||
- id: todo-import-pr-rewrite
|
||||
name: import PR Todo sections (pull --rebase)
|
||||
entry: scripts/todo_hooks.sh import-pr post-rewrite
|
||||
language: script
|
||||
stages: [post-rewrite]
|
||||
always_run: true
|
||||
pass_filenames: false
|
||||
@ -11,8 +11,9 @@ overwrite it with `repo_contract/templates/check_repo_contract.shim.sh`.
|
||||
## Commands
|
||||
|
||||
- run: n/a: collection of gates and libraries, no single entry point
|
||||
- test: `PYTHONPATH=. python -m pytest -q agent_ready/tests coverage_gaps/tests dep_freshness/tests file_length/tests flutter_web_drive/tests home_paths/tests md_naming/tests repo_contract/tests scripts/tests`
|
||||
- test: `PYTHONPATH=. python -m pytest -q agent_ready/tests coverage_gaps/tests dep_freshness/tests file_length/tests flutter_web_drive/tests home_paths/tests md_naming/tests repo_contract/tests scripts/tests unified-design-system/tests`
|
||||
- test-changed: `scripts/test_changed.sh`
|
||||
- lint: `ruff check agent_ready coverage_gaps flutter_web_drive repo_contract`
|
||||
- coverage: `PYTHONPATH=. python -m pytest -q agent_ready/tests flutter_web_drive/tests repo_contract/tests --cov=agent_ready --cov=flutter_web_drive --cov=repo_contract --cov-branch --cov-report=xml --cov-fail-under=100`
|
||||
- coverage: `PYTHONPATH=. python -m pytest -q agent_ready/tests flutter_web_drive/tests repo_contract/tests unified-design-system/tests --cov=agent_ready --cov=flutter_web_drive --cov=repo_contract --cov=conformance --cov=conformance_diff --cov-branch --cov-report=xml --cov-fail-under=100`
|
||||
- coverage-gaps: `coverage-gaps coverage.xml`
|
||||
- deploy: `git -C ~/src/utils pull --ff-only && ~/src/utils/scripts/deploy.sh` (installs the sickdays CLI + launcher from the live checkout, then exits 1 unless every released library's version tag is on origin and matches HEAD and the live `sickdays` runs the text-free checkout; `--verify-only` for the health check alone)
|
||||
|
||||
15
README.md
15
README.md
@ -42,6 +42,21 @@ dependencies:
|
||||
path: sync_settings_ui
|
||||
```
|
||||
|
||||
## google_account_token/
|
||||
|
||||
Flutter plugin (Android only; other platforms return null): one-tap Google ID
|
||||
token via `AccountManager`, replacing `google_sign_in` in the sync apps.
|
||||
|
||||
Consume via:
|
||||
```yaml
|
||||
dependencies:
|
||||
google_account_token:
|
||||
git:
|
||||
url: https://github.com/kuhyx/utils
|
||||
ref: google_account_token-v0.1.0
|
||||
path: google_account_token
|
||||
```
|
||||
|
||||
## crdt-sync/
|
||||
|
||||
Python CRDT sync library (package: `crdt_sync`, not published to PyPI). Same
|
||||
|
||||
@ -1,153 +0,0 @@
|
||||
# TODO — pylint `--fail-under=10` everywhere, and clear what that surfaces
|
||||
|
||||
REMOVE ME AFTER FINISH
|
||||
|
||||
## what
|
||||
|
||||
Ten pylint hooks across six repos still pass at `--fail-under=8.0`, which
|
||||
permanently tolerates two points of lint debt and lets a score *regress* from
|
||||
9.9 to 8.1 without failing anything. Move every one to `10`, and fix what
|
||||
that surfaces. Five other sites already run at `10`, so this is adopting an
|
||||
existing standard, not inventing one.
|
||||
|
||||
## the ten sites, exact
|
||||
|
||||
Measured 2026-09-12. "Score now" is that package's current rating under its
|
||||
own config, tests included.
|
||||
|
||||
| Repo / package | Site | Score now |
|
||||
|---|---|---|
|
||||
| `control-panel` | `.pre-commit-config.yaml:159` | 8.53 |
|
||||
| `leetcode-guard` | `.pre-commit-config.yaml:112` | 8.71 |
|
||||
| `screen-locker` | `.pre-commit-config.yaml:121` | 9.85 |
|
||||
| `screen-locker` | `.pre-commit-config.yaml:140` | (2nd pylint hook) |
|
||||
| `steam-backlog-enforcer` | `.pre-commit-config.yaml:124` | 8.76 |
|
||||
| `testsAndMisc/meta` | `.pre-commit-config.yaml:194` | 8.77 |
|
||||
| `utils/crdt-sync` | `.pre-commit-config.yaml:111` | 9.59 |
|
||||
| `utils/freedays` | `.pre-commit-config.yaml:109` | 8.55 |
|
||||
| `utils/gatelock` | `.pre-commit-config.yaml:109` | 9.18 |
|
||||
| `utils/music_theory` | `.pre-commit-config.yaml:113` | 8.54 |
|
||||
|
||||
`screen-locker` has **two** pylint hooks; both need the bump or the second one
|
||||
silently keeps the old bar.
|
||||
|
||||
Already at `10` — do not touch, these are the reference:
|
||||
`build-your-x`, `diet-guard`, `testsAndMisc` (root config),
|
||||
`wake-alarm`, `WUT_Computer_Science/.../translator/.pylintrc`.
|
||||
|
||||
## why the scores look bad, and why that is misleading
|
||||
|
||||
Every one of these configs runs `enable = "all"` with `disable = []` — the
|
||||
strictest pylint setting there is, including checks that are off by default.
|
||||
The raw findings are dominated by **test-file idioms**, not source defects.
|
||||
Measured on the four `utils` packages:
|
||||
|
||||
```
|
||||
crdt-sync 54 protected-access, 50 redefined-outer-name, 18 duplicate-code
|
||||
freedays 100 missing-function-docstring, 14 implicit-booleaness
|
||||
gatelock 147 redefined-outer-name, 142 protected-access
|
||||
music_theory 112 missing-function-docstring, 14 missing-module-docstring
|
||||
```
|
||||
|
||||
`redefined-outer-name` at that volume is pytest fixtures. `protected-access`
|
||||
is tests reaching into internals on purpose. Neither is a defect.
|
||||
|
||||
**The repos already at 10 solved this and wrote down why.** From
|
||||
`wake-alarm/pyproject.toml`, verbatim:
|
||||
|
||||
> `ignore = [".venv", "__pycache__", "tests"]` — "tests" is a basename match:
|
||||
> test suites intentionally use patterns (protected-access, magic-value
|
||||
> comparisons) that don't apply to source code, matching testsAndMisc's
|
||||
> pylint scope (tests are linted by ruff, not pylint there either).
|
||||
|
||||
So the fleet standard is: **pylint grades source, ruff grades tests.** The ten
|
||||
sites above are the outliers that never adopted it. Their `ruff` config
|
||||
already carries the equivalent test exemptions
|
||||
(`"**/tests/**/*.py" = ["ARG", "D", "PLC0415", "PLR2004", "S101", "SLF001"]`),
|
||||
so tests stay linted — by the tool that has the right rules for them.
|
||||
|
||||
## what is actually left after adopting the fleet config
|
||||
|
||||
Measured: `pylint --ignore=.venv,__pycache__,tests` plus wake-alarm's
|
||||
three-item `disable` list, run over source files only.
|
||||
|
||||
| Package | Score | Remaining findings |
|
||||
|---|---|---|
|
||||
| `crdt-sync` | 9.93 | 5 wrong-import-position, 3 duplicate-code, 1 too-many-locals, 1 invalid-name |
|
||||
| `freedays` | 9.63 | 3 import-error, 1 broad-exception-caught |
|
||||
| `gatelock` | 9.88 | 6 missing-function-docstring, 4 wrong-import-position, 3 too-many-instance-attributes, 3 import-outside-toplevel, 2 broad-exception-caught, 1 each too-many-arguments / invalid-name / duplicate-code / disallowed-name / chained-comparison |
|
||||
| `music_theory` | 9.42 | 10 import-error |
|
||||
|
||||
That is ~47 real findings across four packages, not the ~600 the raw run
|
||||
suggests. **`import-error` is almost certainly config, not code**: every repo
|
||||
already at 10 carries an `init-hook` the 8.0 repos lack —
|
||||
|
||||
```toml
|
||||
init-hook = "import sys; sys.path.insert(0, '.')"
|
||||
```
|
||||
|
||||
Try that first on `music_theory` and `freedays` before editing any import.
|
||||
|
||||
## must
|
||||
|
||||
- Every site at `--fail-under=10`. No `9.5`, no per-repo exception.
|
||||
- Adopt the reference config from `wake-alarm` / `diet-guard`: the `tests`
|
||||
basename ignore, the `init-hook`, and only then the residue fixes.
|
||||
- Fix findings in source. Prefer the fix to the disable, every time.
|
||||
- must not: add a `# pylint: disable=` comment, or extend `disable = [...]`
|
||||
beyond wake-alarm's three implicit-booleaness entries, **without asking
|
||||
kuhy first — every time, per `memories/code-quality.md`.** All seven
|
||||
"unavoidable" disables in a past konbini-67 session were reverted.
|
||||
- must not: exclude a source file from pylint to raise the score, or drop
|
||||
`enable = "all"`.
|
||||
- must not: touch the five sites already at `10`.
|
||||
- optional: `duplicate-code` findings may be genuine shared-code candidates
|
||||
for `utils`. Note them, do not force a refactor to hit the number.
|
||||
|
||||
## done
|
||||
|
||||
Both commands, from `~`:
|
||||
|
||||
```bash
|
||||
# 1. no site left below 10 (cov-fail-under is coverage, not pylint)
|
||||
! grep -rn --include=.pre-commit-config.yaml --include=.pylintrc \
|
||||
-E 'fail-under=(8|9)' ~/src ~/praca_magisterska 2>/dev/null \
|
||||
| grep -v cov-fail-under
|
||||
|
||||
# 2. every one of the nine packages passes its own pylint hook
|
||||
for d in ~/src/control-panel ~/src/leetcode-guard ~/src/screen-locker \
|
||||
~/src/steam-backlog-enforcer ~/src/testsAndMisc/meta \
|
||||
~/src/utils/crdt-sync ~/src/utils/freedays \
|
||||
~/src/utils/gatelock ~/src/utils/music_theory; do
|
||||
(cd "$d" && pre-commit run pylint --all-files) || echo "FAIL $d"
|
||||
done
|
||||
```
|
||||
|
||||
Command 1 prints nothing and exits 0; command 2 prints no `FAIL` line.
|
||||
|
||||
## verify
|
||||
|
||||
Desktop. Per repo, `pre-commit run --all-files` must be green — not just the
|
||||
pylint hook, since a fix that satisfies pylint can break `ruff`, `mypy` or the
|
||||
250-line cap. Then the test suite: these are behaviour-affecting edits
|
||||
(`broad-exception-caught` and `import-outside-toplevel` fixes especially), and
|
||||
every one of these packages holds a 100% coverage bar.
|
||||
|
||||
`utils` and `testsAndMisc` each have CI; push and confirm green rather than
|
||||
trusting the local run. `gatelock`'s Tk suite is flaky on a live X session —
|
||||
run it under `xvfb-run -a` and compare against a baseline worktree before
|
||||
blaming a change for a failure.
|
||||
|
||||
## read first
|
||||
|
||||
- `~/src/wake-alarm/pyproject.toml` — `[tool.pylint]`, the reference config.
|
||||
`diet-guard` is byte-similar; `build-your-x` also ignores `conftest.py`.
|
||||
- `~/.claude/memories/code-quality.md` — the suppression rule that governs
|
||||
every `disable` decision in this task.
|
||||
- `~/src/utils/rules/` is not involved; this is not a dependency change.
|
||||
|
||||
Each repo is its own git repo: commit per repo, work on `main`, stage
|
||||
narrowly. `utils` additionally runs the shared `md-naming` and file-length
|
||||
gates, so delete this file in that repo's final commit.
|
||||
|
||||
REMOVE ME AFTER FINISH
|
||||
@ -1,45 +0,0 @@
|
||||
# aur-publish
|
||||
|
||||
Hands-off publishing of kuhy's Flutter apps to the AUR.
|
||||
|
||||
```bash
|
||||
~/src/utils/aur-publish/publish.sh # wait for registration, then publish
|
||||
~/src/utils/aur-publish/publish.sh --dry-run # everything except the push
|
||||
~/src/utils/aur-publish/publish.sh --skip-wait # account already works
|
||||
~/src/utils/aur-publish/publish.sh --only habit-stack
|
||||
```
|
||||
|
||||
## What it does
|
||||
|
||||
1. Polls `/register` until AUR signup reopens, opens it in a browser and
|
||||
prints the SSH key to paste. **Signup is the only manual step** — it
|
||||
requires email confirmation.
|
||||
2. Waits for `ssh aur@aur.archlinux.org` to authenticate.
|
||||
3. Then, unattended, per package: resolve the newest `vX.Y.Z` tag → set
|
||||
`pkgver` → `updpkgsums` → `makepkg -C` → `namcap` → AUR rule check →
|
||||
`pacman -U` → launch check → `.SRCINFO` → commit → push.
|
||||
|
||||
Every package is built, installed and **launched** before it is pushed, so a
|
||||
broken package cannot reach the AUR. A package failing any gate is skipped and
|
||||
reported; the others still publish. Safe to re-run.
|
||||
|
||||
## anubis_gate.py
|
||||
|
||||
The AUR sits behind [Anubis](https://anubis.techaro.lol/), which answers HTML
|
||||
requests with a proof-of-work challenge page carrying **HTTP 200**. A plain
|
||||
`curl` therefore reports 200 for a page that is actually unreachable — which is
|
||||
exactly how "is registration back?" gets answered wrong. This module solves the
|
||||
challenge and reports the status of the *real* page behind it.
|
||||
|
||||
```bash
|
||||
python3 anubis_gate.py /register # prints the true status code
|
||||
```
|
||||
|
||||
## AUR rules enforced before pushing
|
||||
|
||||
The AUR does not review uploads, but per the submission guidelines "packages
|
||||
that violate the rules may be deleted without warning". `check_aur_rules()`
|
||||
blocks a push on the mechanically checkable ones: `# Maintainer:` line,
|
||||
`x86_64` in `arch`, no `replaces=` (reserved for renames — use `conflicts`),
|
||||
not already in an official repo, `pkgdesc`/`url`/`license` present, and
|
||||
`.SRCINFO` matching the PKGBUILD.
|
||||
@ -1,138 +0,0 @@
|
||||
"""Fetch an AUR URL from behind the Anubis anti-bot gate.
|
||||
|
||||
The AUR sits behind Anubis (https://anubis.techaro.lol/), which answers every
|
||||
HTML request with a proof-of-work challenge page carrying HTTP **200**. A bare
|
||||
`curl` therefore reports 200 for a page that is actually unreachable, which is
|
||||
exactly how "is /register back up?" gets answered wrong.
|
||||
|
||||
This module solves the challenge and reports the status of the *real* page
|
||||
behind it, so the caller can distinguish:
|
||||
|
||||
* 200 -> genuinely available
|
||||
* 503 -> reached the site; that endpoint is disabled server-side
|
||||
* other-> whatever the app actually said
|
||||
|
||||
Exit codes mirror the HTTP status so shell callers can branch on them:
|
||||
0 for 200, 1 for anything else, 2 for a local/transport failure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import http.cookiejar
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
BASE = "https://aur.archlinux.org"
|
||||
UA = "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0"
|
||||
CHALLENGE_RE = re.compile(
|
||||
r'<script id="anubis_challenge" type="application/json">(.*?)</script>',
|
||||
re.S,
|
||||
)
|
||||
PASS_PATH = "/.within.website/x/cmd/anubis/api/pass-challenge"
|
||||
# A solve at difficulty 4 takes ~0.01s; this ceiling only guards a server-side
|
||||
# difficulty spike, so the poller degrades instead of spinning forever.
|
||||
MAX_NONCE = 50_000_000
|
||||
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
"""Keep redirects visible instead of silently following them."""
|
||||
|
||||
def redirect_request(self, *_args, **_kwargs): # noqa: D102 - see class doc
|
||||
return None
|
||||
|
||||
|
||||
def _opener() -> urllib.request.OpenerDirector:
|
||||
jar = http.cookiejar.CookieJar()
|
||||
opener = urllib.request.build_opener(
|
||||
urllib.request.HTTPCookieProcessor(jar), _NoRedirect
|
||||
)
|
||||
opener.addheaders = [
|
||||
("User-Agent", UA),
|
||||
("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"),
|
||||
]
|
||||
return opener
|
||||
|
||||
|
||||
def _solve(random_data: str, difficulty: int) -> tuple[int, str, float]:
|
||||
"""Find a nonce whose sha256 digest starts with `difficulty` zero hex chars."""
|
||||
prefix = "0" * difficulty
|
||||
started = time.time()
|
||||
for nonce in range(MAX_NONCE):
|
||||
digest = hashlib.sha256(f"{random_data}{nonce}".encode()).hexdigest()
|
||||
if digest.startswith(prefix):
|
||||
return nonce, digest, time.time() - started
|
||||
raise RuntimeError(f"no nonce found below {MAX_NONCE} at difficulty {difficulty}")
|
||||
|
||||
|
||||
def fetch(path: str, timeout: int = 30) -> tuple[int, str]:
|
||||
"""Return (status, body) for `path`, clearing Anubis first if challenged."""
|
||||
opener = _opener()
|
||||
url = f"{BASE}{path}"
|
||||
|
||||
try:
|
||||
with opener.open(url, timeout=timeout) as resp:
|
||||
body = resp.read().decode("utf-8", "replace")
|
||||
status = resp.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, exc.read().decode("utf-8", "replace")
|
||||
|
||||
match = CHALLENGE_RE.search(body)
|
||||
if not match:
|
||||
return status, body
|
||||
|
||||
challenge = json.loads(match.group(1))["challenge"]
|
||||
nonce, digest, elapsed = _solve(
|
||||
challenge["randomData"], int(challenge["difficulty"])
|
||||
)
|
||||
query = urllib.parse.urlencode(
|
||||
{
|
||||
"id": challenge["id"],
|
||||
"response": digest,
|
||||
"nonce": str(nonce),
|
||||
"redir": path,
|
||||
"elapsedTime": str(int(elapsed * 1000)),
|
||||
}
|
||||
)
|
||||
try:
|
||||
opener.open(f"{BASE}{PASS_PATH}?{query}", timeout=timeout)
|
||||
except urllib.error.HTTPError:
|
||||
# A redirect or non-2xx here is fine; what matters is the cookie the
|
||||
# opener's jar just captured.
|
||||
pass
|
||||
|
||||
try:
|
||||
with opener.open(url, timeout=timeout) as resp:
|
||||
return resp.status, resp.read().decode("utf-8", "replace")
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, exc.read().decode("utf-8", "replace")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("path", help="AUR path, e.g. /register")
|
||||
parser.add_argument(
|
||||
"--print-body", action="store_true", help="write the page body to stdout"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
status, body = fetch(args.path)
|
||||
except (urllib.error.URLError, RuntimeError, TimeoutError) as exc:
|
||||
print(f"transport error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
print(status)
|
||||
if args.print_body:
|
||||
sys.stdout.write(body)
|
||||
return 0 if status == 200 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@ -1,107 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# ============================================================================
|
||||
# Building, verifying and pushing one package to the AUR.
|
||||
#
|
||||
# Sourced by publish.sh, never executed: these functions read the readonly
|
||||
# config (AUR_HOST, AUR_ROOT, DRY_RUN, PUBLISHED, FAILED) that the entry script declares, and
|
||||
# a `readonly` re-run inside a twice-sourced lib would abort under `set -e`.
|
||||
# ============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
publish_one() {
|
||||
local pkg="$1" repo="$2" wrapper="$3" binary="$4" expected="$5"
|
||||
local dir="$AUR_ROOT/$pkg"
|
||||
local version
|
||||
|
||||
log "$pkg"
|
||||
|
||||
[[ -d "$repo" ]] || { warn "$pkg: no repo at $repo"; return 1; }
|
||||
[[ -f "$dir/PKGBUILD" ]] || { warn "$pkg: no PKGBUILD at $dir"; return 1; }
|
||||
|
||||
version="$(newest_tag "$repo")"
|
||||
[[ -n "$version" ]] || { warn "$pkg: no vX.Y.Z tag on the remote"; return 1; }
|
||||
info "newest release tag: v$version"
|
||||
|
||||
cd "$dir"
|
||||
sed -i -E "s/^pkgver=.*/pkgver=$version/" PKGBUILD
|
||||
# The tarball for a tag contains the PKGBUILD, so the checksum can only be
|
||||
# computed after the tag exists — never committed ahead of time.
|
||||
updpkgsums >/dev/null 2>&1 || { warn "$pkg: updpkgsums failed (tag missing?)"; return 1; }
|
||||
|
||||
info "building"
|
||||
if ! env PATH=/usr/bin:/bin:/usr/local/bin makepkg -Cf --noconfirm >/dev/null 2>&1; then
|
||||
warn "$pkg: makepkg failed"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local built
|
||||
built="$(find . -maxdepth 1 -name "$pkg-$version-*.pkg.tar.zst" | head -1)"
|
||||
[[ -n "$built" ]] || { warn "$pkg: no package produced"; return 1; }
|
||||
|
||||
# namcap's "ELF files outside of a valid path ('opt/')" is a false positive
|
||||
# for /opt-installed apps, so only genuinely new errors are worth failing on.
|
||||
local findings
|
||||
findings="$(namcap "$built" 2>&1 | grep -E ' E: ' | grep -v "outside of a valid path" || true)"
|
||||
if [[ -n "$findings" ]]; then
|
||||
warn "$pkg: namcap errors:"
|
||||
printf ' %s\n' "$findings"
|
||||
return 1
|
||||
fi
|
||||
|
||||
info "installing and launching"
|
||||
sudo pacman -U --noconfirm "$built" >/dev/null 2>&1 \
|
||||
|| { warn "$pkg: pacman -U failed"; return 1; }
|
||||
verify_installed "$binary" "$expected" "$wrapper" || { warn "$pkg: launch check failed"; return 1; }
|
||||
|
||||
# Generated before the rule check, which verifies it matches the PKGBUILD.
|
||||
makepkg --printsrcinfo > .SRCINFO
|
||||
check_aur_rules "$pkg" || return 1
|
||||
|
||||
# The AUR repo is a separate remote living alongside the build dir; adding
|
||||
# it here (rather than cloning into a fresh path) keeps the verified
|
||||
# PKGBUILD and the thing being pushed byte-identical.
|
||||
if [[ ! -d .git ]]; then
|
||||
git init -q -b master .
|
||||
git remote add origin "ssh://aur@$AUR_HOST/$pkg.git"
|
||||
fi
|
||||
git remote get-url origin >/dev/null 2>&1 \
|
||||
|| git remote add origin "ssh://aur@$AUR_HOST/$pkg.git"
|
||||
|
||||
# Fetch first: an existing AUR package must be built on, not overwritten.
|
||||
if git fetch -q origin master 2>/dev/null; then
|
||||
git reset -q --soft FETCH_HEAD 2>/dev/null || true
|
||||
fi
|
||||
|
||||
cat > .gitignore <<'EOF'
|
||||
# Build products; the AUR tracks only the recipe.
|
||||
*.pkg.tar.zst
|
||||
*.tar.gz
|
||||
src/
|
||||
pkg/
|
||||
EOF
|
||||
|
||||
git add PKGBUILD .SRCINFO .gitignore
|
||||
if git diff --cached --quiet; then
|
||||
info "already up to date on the AUR; nothing to push"
|
||||
PUBLISHED+=("$pkg v$version (unchanged)")
|
||||
return 0
|
||||
fi
|
||||
|
||||
git -c user.name='Krzysztof Rudnicki' \
|
||||
-c user.email='krzysztofrudnicki0@gmail.com' \
|
||||
commit -q -m "$pkg $version-1"
|
||||
|
||||
if ((DRY_RUN)); then
|
||||
info "DRY RUN — not pushing"
|
||||
PUBLISHED+=("$pkg v$version (dry run)")
|
||||
return 0
|
||||
fi
|
||||
|
||||
info "pushing to the AUR"
|
||||
git push -q origin master 2>&1 | tail -3 || { warn "$pkg: push failed"; return 1; }
|
||||
PUBLISHED+=("$pkg v$version")
|
||||
return 0
|
||||
}
|
||||
|
||||
@ -1,107 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# ============================================================================
|
||||
# Waiting for AUR registration and SSH access to come up.
|
||||
#
|
||||
# Sourced by publish.sh, never executed: these functions read the readonly
|
||||
# config (AUR_HOST, SSH_KEY, GATE, POLL_SECONDS) that the entry script declares, and
|
||||
# a `readonly` re-run inside a twice-sourced lib would abort under `set -e`.
|
||||
# ============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
register_status() {
|
||||
python3 "$GATE" /register 2>/dev/null | head -1
|
||||
}
|
||||
|
||||
open_browser() {
|
||||
local url="$1"
|
||||
local browser
|
||||
for browser in xdg-open thorium-browser chromium google-chrome-stable firefox librewolf; do
|
||||
if command -v "$browser" >/dev/null 2>&1; then
|
||||
"$browser" "$url" >/dev/null 2>&1 &
|
||||
info "Opened $url in $browser"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
warn "No browser found; open this yourself: $url"
|
||||
}
|
||||
|
||||
ensure_key() {
|
||||
if [[ ! -f "$SSH_KEY" ]]; then
|
||||
log "Generating an AUR SSH key (none at $SSH_KEY)"
|
||||
ssh-keygen -t ed25519 -f "$SSH_KEY" -N '' -C 'kuhy@aur'
|
||||
fi
|
||||
if ! grep -q "Host $AUR_HOST" "$HOME/.ssh/config" 2>/dev/null; then
|
||||
log "Adding $AUR_HOST to ~/.ssh/config"
|
||||
mkdir -p "$HOME/.ssh"
|
||||
cat >> "$HOME/.ssh/config" <<EOF
|
||||
|
||||
# AUR uses a dedicated key so it stays separate from the GitHub identity.
|
||||
Host $AUR_HOST
|
||||
User aur
|
||||
IdentityFile $SSH_KEY
|
||||
IdentitiesOnly yes
|
||||
EOF
|
||||
chmod 600 "$HOME/.ssh/config"
|
||||
fi
|
||||
}
|
||||
|
||||
ssh_works() {
|
||||
# The AUR refuses interactive shells; "Interactive shell is disabled" on a
|
||||
# successful key auth is the success signal, not an error.
|
||||
ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new \
|
||||
-o ConnectTimeout=15 "aur@$AUR_HOST" help 2>&1 |
|
||||
grep -qiE 'interactive shell is disabled|Welcome to AUR'
|
||||
}
|
||||
|
||||
wait_for_registration() {
|
||||
local status
|
||||
status="$(register_status)"
|
||||
if [[ "$status" == "200" ]]; then
|
||||
log "AUR registration is OPEN"
|
||||
else
|
||||
log "AUR registration is closed (HTTP $status). Polling every ${POLL_SECONDS}s."
|
||||
info "Ctrl-C to stop; re-run any time, nothing is lost."
|
||||
while :; do
|
||||
sleep "$POLL_SECONDS"
|
||||
status="$(register_status)"
|
||||
if [[ "$status" == "200" ]]; then
|
||||
log "Registration is BACK (HTTP 200)"
|
||||
break
|
||||
fi
|
||||
printf ' %s still HTTP %s\n' "$(date +%H:%M)" "$status"
|
||||
done
|
||||
fi
|
||||
|
||||
open_browser "https://$AUR_HOST/register"
|
||||
cat <<EOF
|
||||
|
||||
------------------------------------------------------------------
|
||||
Sign up, and paste this into the "SSH Public Key" field:
|
||||
|
||||
$(sed 's/^/ /' "$SSH_KEY.pub")
|
||||
|
||||
You will need to confirm the address from your email before the
|
||||
account works. This script waits for that automatically.
|
||||
------------------------------------------------------------------
|
||||
EOF
|
||||
}
|
||||
|
||||
wait_for_ssh() {
|
||||
log "Waiting for the SSH key to authenticate against the AUR"
|
||||
local waited=0
|
||||
until ssh_works; do
|
||||
sleep 30
|
||||
waited=$((waited + 30))
|
||||
if ((waited % 300 == 0)); then
|
||||
info "still waiting ($((waited / 60)) min) — key not accepted yet"
|
||||
fi
|
||||
done
|
||||
log "SSH authentication OK"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 2: build, verify and publish one package
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@ -1,122 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# ============================================================================
|
||||
# Gates a package must pass before it is allowed near the AUR.
|
||||
#
|
||||
# Sourced by publish.sh, never executed: these functions read the readonly
|
||||
# config (AUR_ROOT, DRY_RUN) that the entry script declares, and
|
||||
# a `readonly` re-run inside a twice-sourced lib would abort under `set -e`.
|
||||
# ============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
newest_tag() {
|
||||
# Newest clean vX.Y.Z on the remote. Build tags (vX.Y.Z-build.N) and the
|
||||
# grafted upstream Flutter SDK tags in diet-guard are excluded on purpose.
|
||||
git -C "$1" ls-remote --tags origin 2>/dev/null |
|
||||
awk '{print $2}' |
|
||||
sed 's|refs/tags/||' |
|
||||
grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' |
|
||||
sed 's/^v//' |
|
||||
sort -V |
|
||||
tail -1
|
||||
}
|
||||
|
||||
verify_installed() {
|
||||
local binary="$1" expected="$2" wrapper="$3" output
|
||||
|
||||
# An instance left over from a previous run would make the wrapper print
|
||||
# "already running" and hand off, which passes trivially and turns the
|
||||
# strongest gate in this script into the weakest. Clear it first so the
|
||||
# check always exercises a real cold start.
|
||||
#
|
||||
# Resolved through /proc/<pid>/exe rather than any form of name or
|
||||
# command-line matching. Every argv-based approach selects the wrong
|
||||
# processes here:
|
||||
# * `pkill -x` compares against the kernel's 15-character comm field, and
|
||||
# two of these wrappers are longer (habit_stack_desktop is 19), so a
|
||||
# name match would silently never fire.
|
||||
# * `pkill -f`, and equally `ps -eo args` piped through a pattern, match
|
||||
# any process whose command line merely *mentions* the path — this
|
||||
# script, the shell running it, and the matching tool itself all
|
||||
# qualify, so the run kills its own shell instead of the app. A
|
||||
# $$/$PPID guard does not cover it, because the offending shell is
|
||||
# often a more distant ancestor.
|
||||
# /proc/<pid>/exe is a kernel-maintained symlink to the binary actually
|
||||
# being executed, so it cannot collide with an incidental mention.
|
||||
# The wrapper name is unique per app, so the /opt/<pkg>/ segment is left as
|
||||
# a glob instead of threading the package name through another parameter.
|
||||
#
|
||||
# The link is read without `-f`: it is already fully resolved, and by the
|
||||
# time this runs `pacman -U` has replaced the binary, so a leftover process
|
||||
# points at an unlinked inode that the kernel reports with a " (deleted)"
|
||||
# suffix. That suffix has to be stripped or the match silently misses the
|
||||
# one process this exists to kill.
|
||||
local pid exe killed=()
|
||||
for pid in /proc/[0-9]*; do
|
||||
pid="${pid#/proc/}"
|
||||
exe="$(readlink "/proc/$pid/exe" 2>/dev/null)" || continue
|
||||
exe="${exe% (deleted)}"
|
||||
[[ "$exe" == /opt/*/bin/"$wrapper" ]] || continue
|
||||
kill -TERM "$pid" 2>/dev/null && killed+=("$pid")
|
||||
done
|
||||
if ((${#killed[@]})); then
|
||||
info "stopped a running instance so the launch check is meaningful"
|
||||
# `kill` only reports that the signal was sent. Waiting for the process
|
||||
# to actually go keeps a slow exit from failing the cold start below on
|
||||
# a port that is still bound.
|
||||
local waited=0
|
||||
while ((waited < 100)) && kill -0 "${killed[@]}" 2>/dev/null; do
|
||||
sleep 0.1
|
||||
((waited++))
|
||||
done
|
||||
fi
|
||||
|
||||
output="$(timeout 15 "/usr/bin/$binary" 2>&1 | head -3 || true)"
|
||||
if grep -qF "$expected" <<<"$output"; then
|
||||
return 0
|
||||
fi
|
||||
warn "unexpected startup output: $output"
|
||||
return 1
|
||||
}
|
||||
|
||||
# The AUR does not review uploads, but "packages that violate the rules may be
|
||||
# deleted without warning" (AUR submission guidelines). These are the rules that
|
||||
# are mechanically checkable, enforced here so a violation blocks the push
|
||||
# rather than surfacing later as a deletion request.
|
||||
check_aur_rules() {
|
||||
local pkg="$1" problems=()
|
||||
|
||||
grep -q '^# Maintainer:' PKGBUILD \
|
||||
|| problems+=("missing '# Maintainer:' comment on line 1")
|
||||
|
||||
grep -qE "^arch=\(.*x86_64" PKGBUILD \
|
||||
|| problems+=("x86_64 not in arch=() — packages without it are not allowed")
|
||||
|
||||
# `replaces` makes pacman rip out the named package on the next -Sy, which
|
||||
# is only correct for a genuine rename. `conflicts` is the right tool.
|
||||
grep -q '^replaces=' PKGBUILD \
|
||||
&& problems+=("uses replaces=; AUR rules reserve it for renames — use conflicts=")
|
||||
|
||||
# A package already in an official repo must never be duplicated.
|
||||
if pacman -Si "$pkg" >/dev/null 2>&1; then
|
||||
problems+=("$pkg exists in an official repo — must not be submitted")
|
||||
fi
|
||||
|
||||
grep -qE '^(pkgdesc|url|license)=' PKGBUILD \
|
||||
|| problems+=("missing pkgdesc/url/license")
|
||||
|
||||
# A stale .SRCINFO is the single most common upload rejection.
|
||||
if ! diff -q <(makepkg --printsrcinfo) .SRCINFO >/dev/null 2>&1; then
|
||||
problems+=(".SRCINFO does not match PKGBUILD")
|
||||
fi
|
||||
|
||||
if ((${#problems[@]})); then
|
||||
warn "$pkg: AUR rule violations:"
|
||||
printf ' - %s\n' "${problems[@]}"
|
||||
return 1
|
||||
fi
|
||||
info "AUR rule checks passed"
|
||||
return 0
|
||||
}
|
||||
|
||||
@ -1,148 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# ============================================================================
|
||||
# Publish kuhy's three Flutter apps to the AUR, hands-off.
|
||||
#
|
||||
# Run this once AUR registration is back. It:
|
||||
# 1. Waits for https://aur.archlinux.org/register to stop returning 503,
|
||||
# then opens it in a browser and prints the SSH key to paste.
|
||||
# (Signup is the ONLY manual step: it needs an email confirmation, and
|
||||
# creating accounts is not something this script will do for you.)
|
||||
# 2. Waits for `ssh aur@aur.archlinux.org` to authenticate.
|
||||
# 3. From there, fully unattended, for each package:
|
||||
# newest release tag -> pkgver -> updpkgsums -> makepkg -C -> namcap
|
||||
# -> pacman -U -> launch check -> .SRCINFO -> clone AUR repo
|
||||
# -> commit -> push.
|
||||
#
|
||||
# Every package is BUILT, INSTALLED and LAUNCHED before it is pushed, so a
|
||||
# broken package cannot reach the AUR. A package that fails any gate is
|
||||
# skipped and reported; the others still publish.
|
||||
#
|
||||
# Idempotent and resumable: re-running skips work that is already done, and
|
||||
# an already-published package is updated rather than duplicated.
|
||||
# ============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
readonly SCRIPT_DIR
|
||||
readonly GATE="$SCRIPT_DIR/anubis_gate.py"
|
||||
readonly AUR_HOST="aur.archlinux.org"
|
||||
readonly SSH_KEY="$HOME/.ssh/aur"
|
||||
readonly AUR_ROOT="$HOME/sdk/aur"
|
||||
|
||||
# package:repo:wrapper-process:cli-name:expected-startup-string
|
||||
# The wrapper process name is NOT derivable from the package name
|
||||
# (diet-guard-app runs diet_guard_desktop), so it is listed explicitly.
|
||||
readonly PACKAGES=(
|
||||
"todo-flutter:$HOME/src/todo:todo_desktop:todo:serving on http://localhost:8730"
|
||||
"habit-stack:$HOME/src/habit_stack:habit_stack_desktop:habit-stack:serving on http://localhost:8731"
|
||||
"diet-guard-app:$HOME/src/diet-guard:diet_guard_desktop:diet-guard-app:serving on http://localhost:8732"
|
||||
)
|
||||
|
||||
POLL_SECONDS=300
|
||||
SKIP_WAIT=0
|
||||
DRY_RUN=0
|
||||
ONLY=""
|
||||
FAILED=()
|
||||
PUBLISHED=()
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $(basename "$0") [OPTIONS]
|
||||
|
||||
Options:
|
||||
--skip-wait Assume the account + SSH key already work; go straight
|
||||
to building and publishing.
|
||||
--dry-run Do everything except the final 'git push' to the AUR.
|
||||
--only NAME Publish just one package (todo-flutter, habit-stack,
|
||||
diet-guard-app).
|
||||
--poll SECONDS How often to re-check /register (default: $POLL_SECONDS).
|
||||
-h, --help Show this help.
|
||||
EOF
|
||||
exit 0
|
||||
}
|
||||
|
||||
log() { printf '\n\033[1;34m==>\033[0m %s\n' "$*"; }
|
||||
info() { printf ' %s\n' "$*"; }
|
||||
warn() { printf '\033[1;33m ! %s\033[0m\n' "$*" >&2; }
|
||||
die() { printf '\033[1;31m==> ERROR: %s\033[0m\n' "$*" >&2; exit 1; }
|
||||
|
||||
require_tools() {
|
||||
local missing=()
|
||||
local tool
|
||||
for tool in git makepkg updpkgsums namcap ssh python3 curl; do
|
||||
command -v "$tool" >/dev/null 2>&1 || missing+=("$tool")
|
||||
done
|
||||
if ((${#missing[@]})); then
|
||||
info "Installing missing tools: ${missing[*]}"
|
||||
# namcap/updpkgsums come from namcap + pacman-contrib; the rest are base.
|
||||
sudo pacman -S --needed --noconfirm namcap pacman-contrib git openssh python \
|
||||
|| die "could not install: ${missing[*]}"
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 1: wait for registration, then for the key to work
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# The rest lives in lib/, sourced (not executed) so every function sees the
|
||||
# readonly config above and the mutable FAILED/PUBLISHED arrays main() reads.
|
||||
# shellcheck source=lib/aur_registration.sh
|
||||
source "$SCRIPT_DIR/lib/aur_registration.sh"
|
||||
# shellcheck source=lib/aur_verify.sh
|
||||
source "$SCRIPT_DIR/lib/aur_verify.sh"
|
||||
# shellcheck source=lib/aur_publish_one.sh
|
||||
source "$SCRIPT_DIR/lib/aur_publish_one.sh"
|
||||
|
||||
main() {
|
||||
require_tools
|
||||
[[ -f "$GATE" ]] || die "missing $GATE"
|
||||
mkdir -p "$AUR_ROOT"
|
||||
|
||||
ensure_key
|
||||
if ((SKIP_WAIT)); then
|
||||
ssh_works || die "SSH key is not accepted yet; drop --skip-wait to wait for it"
|
||||
else
|
||||
wait_for_registration
|
||||
wait_for_ssh
|
||||
fi
|
||||
|
||||
local entry pkg repo wrapper binary expected
|
||||
for entry in "${PACKAGES[@]}"; do
|
||||
IFS=: read -r pkg repo wrapper binary expected <<<"$entry"
|
||||
[[ -z "$ONLY" || "$ONLY" == "$pkg" ]] || continue
|
||||
if ! publish_one "$pkg" "$repo" "$wrapper" "$binary" "$expected"; then
|
||||
FAILED+=("$pkg")
|
||||
fi
|
||||
done
|
||||
|
||||
log "Summary"
|
||||
if ((${#PUBLISHED[@]})); then
|
||||
printf ' published: %s\n' "${PUBLISHED[@]}"
|
||||
fi
|
||||
if ((${#FAILED[@]})); then
|
||||
printf '\033[1;31m FAILED: %s\033[0m\n' "${FAILED[@]}"
|
||||
info "Nothing broken was pushed; fix and re-run (safe to repeat)."
|
||||
return 1
|
||||
fi
|
||||
if ((${#PUBLISHED[@]} == 0)); then
|
||||
warn "nothing to do"
|
||||
return 1
|
||||
fi
|
||||
printf '\n Check: https://aur.archlinux.org/packages/todo-flutter\n'
|
||||
return 0
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--skip-wait) SKIP_WAIT=1; shift ;;
|
||||
--dry-run) DRY_RUN=1; shift ;;
|
||||
--only) ONLY="$2"; shift 2 ;;
|
||||
--poll) POLL_SECONDS="$2"; shift 2 ;;
|
||||
-h|--help) usage ;;
|
||||
*) die "Unknown option: $1" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
main "$@"
|
||||
@ -65,7 +65,7 @@ repos:
|
||||
types: [python]
|
||||
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.17.2
|
||||
rev: v0.17.4
|
||||
hooks:
|
||||
- id: ruff
|
||||
args: [--fix, --unsafe-fixes, --exit-non-zero-on-fix, --show-fixes]
|
||||
|
||||
@ -14,6 +14,30 @@ from crdt_sync._config import (
|
||||
mirror_client_for,
|
||||
)
|
||||
from crdt_sync._device import DeviceIdentity, load_device_identity
|
||||
from crdt_sync._e2e_cipher import (
|
||||
E2eCipher,
|
||||
E2eDecryptError,
|
||||
is_e2e_envelope,
|
||||
open_log_text,
|
||||
seal_log_text,
|
||||
)
|
||||
from crdt_sync._e2e_cleanup import clear_plaintext_prefix, unverified_blobs
|
||||
from crdt_sync._e2e_key import (
|
||||
Argon2Params,
|
||||
E2eHeaderError,
|
||||
E2eKey,
|
||||
E2eWrongPassphraseError,
|
||||
default_e2e_path,
|
||||
unlock_e2e,
|
||||
)
|
||||
from crdt_sync._e2e_store import (
|
||||
E2eLayout,
|
||||
E2eLockedError,
|
||||
EncryptingStore,
|
||||
copy_to_encrypted,
|
||||
open_encrypted_client,
|
||||
reset_e2e_copy_memo,
|
||||
)
|
||||
from crdt_sync._firebase import (
|
||||
DatabaseNotFoundError,
|
||||
FirebaseSyncClient,
|
||||
@ -51,10 +75,19 @@ __all__ = [
|
||||
"CONFIG_DIR",
|
||||
"CONFIG_FILE",
|
||||
"PASSWORD_FILE",
|
||||
"Argon2Params",
|
||||
"ConfigError",
|
||||
"CredentialStore",
|
||||
"DatabaseNotFoundError",
|
||||
"DeviceIdentity",
|
||||
"E2eCipher",
|
||||
"E2eDecryptError",
|
||||
"E2eHeaderError",
|
||||
"E2eKey",
|
||||
"E2eLayout",
|
||||
"E2eLockedError",
|
||||
"E2eWrongPassphraseError",
|
||||
"EncryptingStore",
|
||||
"Field",
|
||||
"FileCredentialStore",
|
||||
"FileSyncStateStore",
|
||||
@ -81,18 +114,28 @@ __all__ = [
|
||||
"SyncState",
|
||||
"SyncStateStore",
|
||||
"SyncTarget",
|
||||
"clear_plaintext_prefix",
|
||||
"copy_to_encrypted",
|
||||
"credential_store_for",
|
||||
"default_e2e_path",
|
||||
"default_revs_path",
|
||||
"dump_log",
|
||||
"firebase_client_for",
|
||||
"is_e2e_envelope",
|
||||
"load_device_identity",
|
||||
"load_log",
|
||||
"merge_field",
|
||||
"merge_logs",
|
||||
"merge_record",
|
||||
"mirror_client_for",
|
||||
"open_encrypted_client",
|
||||
"open_log_text",
|
||||
"read_log",
|
||||
"reset_e2e_copy_memo",
|
||||
"revision_of",
|
||||
"seal_log_text",
|
||||
"sync_log",
|
||||
"unlock_e2e",
|
||||
"unverified_blobs",
|
||||
"write_log",
|
||||
]
|
||||
|
||||
57
crdt-sync/crdt_sync/_e2e_cache.py
Normal file
57
crdt-sync/crdt_sync/_e2e_cache.py
Normal file
@ -0,0 +1,57 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""On-disk E2E state for short-lived daemons (one process per tick).
|
||||
|
||||
The derived key is cached ``0600`` beside the app's other state, so Argon2id
|
||||
runs once per machine rather than once a minute; it is no more sensitive
|
||||
than ``~/.config/crdt-sync/password``, which it is derived from. The copy
|
||||
memo records the old prefix's revisions last copied, so an unchanged old
|
||||
prefix is not re-scanned every tick.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
_MASTER = 32
|
||||
|
||||
|
||||
def _slug(old_prefix: str) -> str:
|
||||
return hashlib.sha256(old_prefix.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def read_cached_key(state_dir: Path, old_prefix: str) -> bytes | None:
|
||||
"""Return the cached 32-byte master key, or None."""
|
||||
path = state_dir / f"e2e-{_slug(old_prefix)}.key"
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
master = bytes.fromhex(path.read_text(encoding="utf-8").strip())
|
||||
except ValueError:
|
||||
return None
|
||||
return master if len(master) == _MASTER else None
|
||||
|
||||
|
||||
def write_cached_key(state_dir: Path, old_prefix: str, master: bytes) -> None:
|
||||
"""Store the master key owner-only."""
|
||||
state_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = state_dir / f"e2e-{_slug(old_prefix)}.key"
|
||||
path.touch(mode=0o600, exist_ok=True)
|
||||
path.chmod(0o600)
|
||||
path.write_text(master.hex(), encoding="utf-8")
|
||||
|
||||
|
||||
def read_copy_memo(state_dir: Path, old_prefix: str) -> str | None:
|
||||
"""Return the old-prefix fingerprint last copied, or None."""
|
||||
path = state_dir / f"e2e-{_slug(old_prefix)}.copied"
|
||||
return path.read_text(encoding="utf-8") if path.exists() else None
|
||||
|
||||
|
||||
def write_copy_memo(state_dir: Path, old_prefix: str, fingerprint: str) -> None:
|
||||
"""Record that the old prefix was copied at ``fingerprint``."""
|
||||
state_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = state_dir / f"e2e-{_slug(old_prefix)}.copied"
|
||||
path.write_text(fingerprint, encoding="utf-8")
|
||||
202
crdt-sync/crdt_sync/_e2e_cipher.py
Normal file
202
crdt-sync/crdt_sync/_e2e_cipher.py
Normal file
@ -0,0 +1,202 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""XChaCha20-Poly1305 blob cipher and per-record envelope.
|
||||
|
||||
Byte-compatible with ``crdt_sync_dart``'s ``E2eCipher`` and
|
||||
``sealLogText``/``openLogText``: a blob is base64 ``[version][nonce][ct][tag]``
|
||||
with ``[version] + utf8(path)`` as AAD; an envelope keeps record ids, the
|
||||
tombstone and the newest HLC clear and seals each record's fields.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import secrets
|
||||
from typing import cast
|
||||
|
||||
from nacl.bindings import (
|
||||
crypto_aead_xchacha20poly1305_ietf_decrypt,
|
||||
crypto_aead_xchacha20poly1305_ietf_encrypt,
|
||||
)
|
||||
from nacl.exceptions import CryptoError
|
||||
|
||||
from crdt_sync._hlc import Hlc
|
||||
|
||||
E2E_FORMAT_VERSION = 1
|
||||
_NONCE = 24
|
||||
_TAG = 16
|
||||
_KEY = 32
|
||||
|
||||
|
||||
class E2eDecryptError(ValueError):
|
||||
"""A blob that is tampered, under another key, or in an unknown format.
|
||||
|
||||
A ``ValueError`` so the pull loop skips the peer instead of aborting.
|
||||
"""
|
||||
|
||||
|
||||
def is_plaintext(stored: str) -> bool:
|
||||
"""Return whether ``stored`` is legacy JSON rather than a sealed blob."""
|
||||
return stored.lstrip().startswith(("{", "["))
|
||||
|
||||
|
||||
def _aad(version: int, path: str) -> bytes:
|
||||
return bytes([version]) + path.encode()
|
||||
|
||||
|
||||
class E2eCipher:
|
||||
"""Seals and opens blobs bound to their remote path."""
|
||||
|
||||
def __init__(self, key: bytes, *, accept_plaintext: bool = True) -> None:
|
||||
"""Wrap a 32-byte ``key``; ``accept_plaintext`` reads old peers."""
|
||||
if len(key) != _KEY:
|
||||
msg = "key must be 32 bytes"
|
||||
raise ValueError(msg)
|
||||
self._key = bytes(key)
|
||||
self.accept_plaintext = accept_plaintext
|
||||
|
||||
def seal(self, plaintext: str, *, path: str) -> str:
|
||||
"""Return ``plaintext`` sealed for ``path``."""
|
||||
nonce = secrets.token_bytes(_NONCE)
|
||||
box = crypto_aead_xchacha20poly1305_ietf_encrypt(
|
||||
plaintext.encode(), _aad(E2E_FORMAT_VERSION, path), nonce, self._key
|
||||
)
|
||||
return base64.b64encode(bytes([E2E_FORMAT_VERSION]) + nonce + box).decode()
|
||||
|
||||
def open(self, stored: str, *, path: str) -> str:
|
||||
"""Return the plaintext of ``stored``; raise :class:`E2eDecryptError`."""
|
||||
if is_plaintext(stored):
|
||||
if self.accept_plaintext:
|
||||
return stored
|
||||
msg = f"plaintext blob at {path} after migration"
|
||||
raise E2eDecryptError(msg)
|
||||
try:
|
||||
raw = base64.b64decode(stored, validate=True)
|
||||
except binascii.Error as exc:
|
||||
msg = f"blob at {path} is neither JSON nor base64"
|
||||
raise E2eDecryptError(msg) from exc
|
||||
if not raw:
|
||||
msg = f"empty blob at {path}"
|
||||
raise E2eDecryptError(msg)
|
||||
if raw[0] != E2E_FORMAT_VERSION:
|
||||
msg = f"blob at {path} uses format {raw[0]}; update the app"
|
||||
raise E2eDecryptError(msg)
|
||||
if len(raw) < 1 + _NONCE + _TAG:
|
||||
msg = f"blob at {path} is truncated"
|
||||
raise E2eDecryptError(msg)
|
||||
try:
|
||||
clear = crypto_aead_xchacha20poly1305_ietf_decrypt(
|
||||
raw[1 + _NONCE :], _aad(raw[0], path), raw[1 : 1 + _NONCE], self._key
|
||||
)
|
||||
except CryptoError as exc:
|
||||
msg = f"blob at {path} failed authentication"
|
||||
raise E2eDecryptError(msg) from exc
|
||||
return clear.decode()
|
||||
|
||||
|
||||
def is_e2e_envelope(text: str) -> bool:
|
||||
"""Return whether ``text`` is an envelope (integer ``"v"``)."""
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except ValueError:
|
||||
return False
|
||||
version = data.get("v") if isinstance(data, dict) else None
|
||||
return isinstance(version, int) and not isinstance(version, bool)
|
||||
|
||||
|
||||
def _record_aad(
|
||||
path: str, rid: str, deleted: object, deleted_hlc: object, newest: object
|
||||
) -> str:
|
||||
flag = 1 if deleted is True else 0
|
||||
return f"{path}#{rid}#{flag}#{deleted_hlc or ''}#{newest or ''}"
|
||||
|
||||
|
||||
def _as_log(text: str) -> dict[str, dict[str, object]] | None:
|
||||
try:
|
||||
raw = json.loads(text)
|
||||
except ValueError:
|
||||
return None
|
||||
if not isinstance(raw, dict):
|
||||
return None
|
||||
for rid, rec in raw.items():
|
||||
if not isinstance(rec, dict) or rec.get("id") != rid:
|
||||
return None
|
||||
if not isinstance(rec.get("fields"), dict):
|
||||
return None
|
||||
return cast("dict[str, dict[str, object]]", raw)
|
||||
|
||||
|
||||
def _newest(fields: dict[str, list[object]]) -> str | None:
|
||||
clocks = [Hlc.from_str(str(pair[1])) for pair in fields.values()]
|
||||
return max(clocks).to_str() if clocks else None
|
||||
|
||||
|
||||
def seal_log_text(text: str, cipher: E2eCipher, *, path: str) -> str:
|
||||
"""Seal a log (``dump_log`` shape) as an envelope; anything else whole."""
|
||||
log = _as_log(text)
|
||||
if log is None:
|
||||
return json.dumps(
|
||||
{"v": E2E_FORMAT_VERSION, "b": cipher.seal(text, path=path)},
|
||||
separators=(",", ":"),
|
||||
)
|
||||
records: dict[str, object] = {}
|
||||
for rid, rec in log.items():
|
||||
fields = cast("dict[str, list[object]]", rec["fields"])
|
||||
deleted = rec.get("deleted") is True
|
||||
deleted_hlc = rec.get("deleted_hlc")
|
||||
newest = _newest(fields)
|
||||
records[rid] = {
|
||||
"d": deleted,
|
||||
"dh": deleted_hlc,
|
||||
"h": newest,
|
||||
"p": cipher.seal(
|
||||
json.dumps(fields, separators=(",", ":"), ensure_ascii=False),
|
||||
path=_record_aad(path, rid, deleted, deleted_hlc, newest),
|
||||
),
|
||||
}
|
||||
return json.dumps(
|
||||
{"v": E2E_FORMAT_VERSION, "r": records},
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
|
||||
def open_log_text(text: str, cipher: E2eCipher, *, path: str) -> str:
|
||||
"""Open an envelope back to the ``dump_log`` shape."""
|
||||
if not is_e2e_envelope(text):
|
||||
if cipher.accept_plaintext:
|
||||
return text
|
||||
msg = f"plaintext blob at {path} after migration"
|
||||
raise E2eDecryptError(msg)
|
||||
data = json.loads(text)
|
||||
if data["v"] != E2E_FORMAT_VERSION:
|
||||
msg = f"blob at {path} uses format {data['v']}; update the app"
|
||||
raise E2eDecryptError(msg)
|
||||
whole = data.get("b")
|
||||
if isinstance(whole, str):
|
||||
return cipher.open(whole, path=path)
|
||||
records = data.get("r")
|
||||
if not isinstance(records, dict):
|
||||
msg = f"envelope at {path} is mis-shaped"
|
||||
raise E2eDecryptError(msg)
|
||||
out: dict[str, object] = {}
|
||||
for rid, rec in records.items():
|
||||
if (
|
||||
not isinstance(rec, dict)
|
||||
or not isinstance(rec.get("p"), str)
|
||||
or not isinstance(rec.get("d"), bool)
|
||||
):
|
||||
msg = f"record {rid} at {path} is mis-shaped"
|
||||
raise E2eDecryptError(msg)
|
||||
clear = cipher.open(
|
||||
rec["p"],
|
||||
path=_record_aad(path, rid, rec["d"], rec.get("dh"), rec.get("h")),
|
||||
)
|
||||
out[rid] = {
|
||||
"id": rid,
|
||||
"fields": json.loads(clear),
|
||||
"deleted": rec["d"],
|
||||
"deleted_hlc": rec.get("dh"),
|
||||
}
|
||||
return json.dumps(out, separators=(",", ":"), ensure_ascii=False)
|
||||
79
crdt-sync/crdt_sync/_e2e_cleanup.py
Normal file
79
crdt-sync/crdt_sync/_e2e_cleanup.py
Normal file
@ -0,0 +1,79 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Clear an app's old plaintext prefix once its encrypted copy is proven.
|
||||
|
||||
The migration never deletes the old prefix by itself: an old build may still
|
||||
write there. Run this once every device runs an encrypting build. It never
|
||||
creates a header (a typo'd passphrase must not lock the app for good) and
|
||||
deletes nothing unless every old blob is contained in the encrypted copy.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from crdt_sync._e2e_key import derive_key, read_header
|
||||
from crdt_sync._e2e_store import E2eLayout, E2eLockedError, EncryptingStore, _blobs
|
||||
from crdt_sync._log import merge_logs
|
||||
from crdt_sync._store import dump_log, load_log
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from crdt_sync._e2e_key import E2eKey
|
||||
from crdt_sync._remote import RemoteStore
|
||||
|
||||
|
||||
def unverified_blobs(
|
||||
store: RemoteStore, key: E2eKey, old_prefix: str, layout: E2eLayout
|
||||
) -> list[str]:
|
||||
"""Return old blobs (relative paths) not fully contained in the copy."""
|
||||
dest = EncryptingStore(store, key, layout)
|
||||
missing: list[str] = []
|
||||
for rel, text in sorted(_blobs(store, old_prefix).items()):
|
||||
try:
|
||||
source = load_log(text)
|
||||
copied = dest.get_file_text(f"{layout.prefix}/{rel}")
|
||||
current = load_log(copied) if copied is not None else None
|
||||
except ValueError, KeyError, TypeError:
|
||||
missing.append(rel)
|
||||
continue
|
||||
if not source:
|
||||
continue
|
||||
if current is None or dump_log(merge_logs(current, source)) != dump_log(
|
||||
current
|
||||
):
|
||||
missing.append(rel)
|
||||
return missing
|
||||
|
||||
|
||||
def _existing_key(store: RemoteStore, layout: E2eLayout, passphrase: str) -> E2eKey:
|
||||
header = read_header(store, layout.e2e_path)
|
||||
if header is None:
|
||||
msg = f"no e2e header at {layout.e2e_path}: nothing was migrated"
|
||||
raise E2eLockedError(msg)
|
||||
key = derive_key(passphrase, header.salt, header.params)
|
||||
if not hmac.compare_digest(key.check_for(header.salt), header.check):
|
||||
msg = "wrong crdt-sync passphrase"
|
||||
raise E2eLockedError(msg)
|
||||
return key
|
||||
|
||||
|
||||
def clear_plaintext_prefix(
|
||||
store: RemoteStore, old_prefix: str, *, passphrase: str, apply: bool
|
||||
) -> list[str]:
|
||||
"""Return the old paths to delete; delete them only when ``apply``.
|
||||
|
||||
Raises:
|
||||
E2eLockedError: No header, a wrong passphrase, or a copy that does
|
||||
not yet hold every old record.
|
||||
"""
|
||||
layout = E2eLayout.of(old_prefix)
|
||||
key = _existing_key(store, layout, passphrase)
|
||||
missing = unverified_blobs(store, key, old_prefix, layout)
|
||||
if missing:
|
||||
msg = f"copy not verified for {', '.join(missing)}"
|
||||
raise E2eLockedError(msg)
|
||||
paths = [f"{old_prefix}/{rel}" for rel in sorted(_blobs(store, old_prefix))]
|
||||
if apply:
|
||||
for path in paths:
|
||||
store.delete_file(path, message="e2e: clear plaintext prefix")
|
||||
return paths
|
||||
191
crdt-sync/crdt_sync/_e2e_key.py
Normal file
191
crdt-sync/crdt_sync/_e2e_key.py
Normal file
@ -0,0 +1,191 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Argon2id key from the shared crdt-sync password, plus HKDF subkeys.
|
||||
|
||||
Byte-compatible with ``crdt_sync_dart``'s ``E2eKey``/``unlockE2e``: the clear
|
||||
header at ``e2e_path`` holds salt, Argon2id cost and a check MAC, never a key.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import secrets
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from argon2.low_level import Type, hash_secret_raw
|
||||
|
||||
from crdt_sync._e2e_cipher import E2eCipher
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from crdt_sync._remote import RemoteStore
|
||||
|
||||
_SALT = 16
|
||||
_MASTER = 32
|
||||
_HEADER_VERSION = 1
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Argon2Params:
|
||||
"""Argon2id cost, stored in the header so it can be raised later."""
|
||||
|
||||
memory_kib: int = 19456
|
||||
iterations: int = 2
|
||||
parallelism: int = 1
|
||||
|
||||
|
||||
class E2eHeaderError(ValueError):
|
||||
"""The header exists but cannot be parsed; it is never overwritten."""
|
||||
|
||||
|
||||
class E2eWrongPassphraseError(Exception):
|
||||
"""The passphrase does not match the header's check."""
|
||||
|
||||
|
||||
def default_e2e_path(path_prefix: str) -> str:
|
||||
"""Return the header path: a sibling of the device dirs, like ``revs``."""
|
||||
head, sep, _ = path_prefix.rpartition("/")
|
||||
return f"{head}/e2e" if sep else f"{path_prefix}/e2e"
|
||||
|
||||
|
||||
def _hkdf(master: bytes, label: str) -> bytes:
|
||||
prk = hmac.new(b"", master, hashlib.sha256).digest()
|
||||
info = f"crdt_sync e2e {label}".encode()
|
||||
return hmac.new(prk, info + b"\x01", hashlib.sha256).digest()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class E2eKey:
|
||||
"""A 32-byte master key and its subkeys. Never written to the remote."""
|
||||
|
||||
master: bytes
|
||||
enc: bytes
|
||||
rev: bytes
|
||||
check: bytes
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, master: bytes) -> E2eKey:
|
||||
"""Expand a cached 32-byte master key."""
|
||||
if len(master) != _MASTER:
|
||||
msg = "master key must be 32 bytes"
|
||||
raise ValueError(msg)
|
||||
return cls(
|
||||
master, _hkdf(master, "enc"), _hkdf(master, "rev"), _hkdf(master, "check")
|
||||
)
|
||||
|
||||
def cipher(self, *, accept_plaintext: bool = True) -> E2eCipher:
|
||||
"""Return the blob cipher for this key."""
|
||||
return E2eCipher(self.enc, accept_plaintext=accept_plaintext)
|
||||
|
||||
def revision(self, text: str) -> str:
|
||||
"""Return the keyed revision (hex HMAC-SHA256) of ``text``."""
|
||||
return hmac.new(self.rev, text.encode(), hashlib.sha256).hexdigest()
|
||||
|
||||
def check_for(self, salt: bytes) -> str:
|
||||
"""Return the header check value for ``salt``."""
|
||||
mac = hmac.new(self.check, salt, hashlib.sha256).digest()
|
||||
return base64.b64encode(mac).decode()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Header:
|
||||
salt: bytes
|
||||
params: Argon2Params
|
||||
check: str
|
||||
|
||||
@classmethod
|
||||
def parse(cls, text: str) -> _Header:
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except ValueError as exc:
|
||||
msg = "e2e header is not JSON"
|
||||
raise E2eHeaderError(msg) from exc
|
||||
if not isinstance(data, dict) or data.get("v") != _HEADER_VERSION:
|
||||
msg = "e2e header is not a known version"
|
||||
raise E2eHeaderError(msg)
|
||||
if data.get("kdf") != "argon2id":
|
||||
msg = "e2e header is malformed"
|
||||
raise E2eHeaderError(msg)
|
||||
try:
|
||||
return cls(
|
||||
base64.b64decode(data["salt"]),
|
||||
Argon2Params(int(data["m"]), int(data["t"]), int(data["p"])),
|
||||
str(data["check"]),
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
msg = "e2e header is malformed"
|
||||
raise E2eHeaderError(msg) from exc
|
||||
|
||||
def encode(self) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"v": _HEADER_VERSION,
|
||||
"kdf": "argon2id",
|
||||
"salt": base64.b64encode(self.salt).decode(),
|
||||
"m": self.params.memory_kib,
|
||||
"t": self.params.iterations,
|
||||
"p": self.params.parallelism,
|
||||
"check": self.check,
|
||||
},
|
||||
separators=(",", ":"),
|
||||
)
|
||||
|
||||
|
||||
def derive_key(passphrase: str, salt: bytes, params: Argon2Params) -> E2eKey:
|
||||
"""Run Argon2id (v1.3) and expand the subkeys."""
|
||||
master = hash_secret_raw(
|
||||
secret=passphrase.encode(),
|
||||
salt=salt,
|
||||
time_cost=params.iterations,
|
||||
memory_cost=params.memory_kib,
|
||||
parallelism=params.parallelism,
|
||||
hash_len=32,
|
||||
type=Type.ID,
|
||||
)
|
||||
return E2eKey.from_bytes(master)
|
||||
|
||||
|
||||
def read_header(store: RemoteStore, path: str) -> _Header | None:
|
||||
"""Return the parsed header at ``path``, or None when absent."""
|
||||
text = store.get_file_text(path)
|
||||
return None if text is None else _Header.parse(text)
|
||||
|
||||
|
||||
def key_matches(key: E2eKey, store: RemoteStore, path: str) -> bool:
|
||||
"""Return whether ``key`` matches the header at ``path``."""
|
||||
header = read_header(store, path)
|
||||
return header is not None and hmac.compare_digest(
|
||||
key.check_for(header.salt), header.check
|
||||
)
|
||||
|
||||
|
||||
def unlock_e2e(
|
||||
store: RemoteStore,
|
||||
*,
|
||||
path_prefix: str,
|
||||
passphrase: str,
|
||||
params: Argon2Params | None = None,
|
||||
e2e_path: str | None = None,
|
||||
) -> E2eKey:
|
||||
"""Derive the key from ``passphrase``, creating the header on first use."""
|
||||
if not passphrase:
|
||||
msg = "passphrase must not be empty"
|
||||
raise ValueError(msg)
|
||||
path = e2e_path or default_e2e_path(path_prefix)
|
||||
header = read_header(store, path)
|
||||
if header is None:
|
||||
salt = secrets.token_bytes(_SALT)
|
||||
cost = params or Argon2Params()
|
||||
key = derive_key(passphrase, salt, cost)
|
||||
encoded = _Header(salt, cost, key.check_for(salt)).encode()
|
||||
store.put_file_text(path, encoded, message=path)
|
||||
stored = store.get_file_text(path) or encoded
|
||||
if stored == encoded:
|
||||
return key
|
||||
header = _Header.parse(stored)
|
||||
key = derive_key(passphrase, header.salt, header.params)
|
||||
if not hmac.compare_digest(key.check_for(header.salt), header.check):
|
||||
raise E2eWrongPassphraseError
|
||||
return key
|
||||
267
crdt-sync/crdt_sync/_e2e_store.py
Normal file
267
crdt-sync/crdt_sync/_e2e_store.py
Normal file
@ -0,0 +1,267 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Encrypted remote store, ``-e2e`` layout and the one-time copy.
|
||||
|
||||
Mirrors ``crdt_sync_dart``'s ``EncryptingStore``, ``E2eLayout``,
|
||||
``copyToEncrypted`` and ``openEncryptedStore`` so Python daemons and Flutter
|
||||
apps share one encrypted prefix per app.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from crdt_sync._e2e_cache import (
|
||||
read_cached_key,
|
||||
read_copy_memo,
|
||||
write_cached_key,
|
||||
write_copy_memo,
|
||||
)
|
||||
from crdt_sync._e2e_cipher import E2eDecryptError, open_log_text, seal_log_text
|
||||
from crdt_sync._e2e_key import (
|
||||
Argon2Params,
|
||||
E2eHeaderError,
|
||||
E2eKey,
|
||||
E2eWrongPassphraseError,
|
||||
key_matches,
|
||||
unlock_e2e,
|
||||
)
|
||||
from crdt_sync._log import merge_logs
|
||||
from crdt_sync._store import dump_log, load_log
|
||||
from crdt_sync._sync import default_revs_path
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
from crdt_sync._remote import RemoteStore
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class E2eLockedError(Exception):
|
||||
"""No usable key: the caller must not sync at all (fail closed)."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class E2eLayout:
|
||||
"""Where an app's encrypted data lives, derived from its old prefix."""
|
||||
|
||||
prefix: str
|
||||
revs_path: str
|
||||
e2e_path: str
|
||||
|
||||
@classmethod
|
||||
def of(cls, old_prefix: str) -> E2eLayout:
|
||||
"""Return ``<old>-e2e`` with revs and header kept out of old paths."""
|
||||
prefix = f"{old_prefix}-e2e"
|
||||
parent, sep, _ = old_prefix.rpartition("/")
|
||||
if not sep:
|
||||
return cls(prefix, f"{prefix}/revs", f"{prefix}/e2e")
|
||||
return cls(prefix, f"{parent}/revs-e2e", f"{parent}/e2e")
|
||||
|
||||
|
||||
class EncryptingStore:
|
||||
"""Seals log blobs under the layout's prefix and keys its revisions."""
|
||||
|
||||
def __init__(self, inner: RemoteStore, key: E2eKey, layout: E2eLayout) -> None:
|
||||
"""Wrap ``inner``; other paths pass through unchanged."""
|
||||
self._inner = inner
|
||||
self.key = key
|
||||
self._layout = layout
|
||||
self._cipher = key.cipher()
|
||||
|
||||
def _is_rev(self, path: str) -> bool:
|
||||
return path.startswith(f"{self._layout.revs_path}/")
|
||||
|
||||
def _is_log(self, path: str) -> bool:
|
||||
return (
|
||||
not self._is_rev(path)
|
||||
and path != self._layout.e2e_path
|
||||
and path.startswith(f"{self._layout.prefix}/")
|
||||
)
|
||||
|
||||
def list_directory(self, path: str) -> list[str]:
|
||||
"""Delegate."""
|
||||
return self._inner.list_directory(path)
|
||||
|
||||
def get_file_text(self, path: str) -> str | None:
|
||||
"""Read and open a sealed log; other paths pass through."""
|
||||
text = self._inner.get_file_text(path)
|
||||
if text is None or not self._is_log(path):
|
||||
return text
|
||||
return open_log_text(text, self._cipher, path=path)
|
||||
|
||||
def put_file_text(self, path: str, text: str, *, message: str) -> None:
|
||||
"""Seal logs and key revisions before writing."""
|
||||
if self._is_log(path):
|
||||
text = seal_log_text(text, self._cipher, path=path)
|
||||
elif self._is_rev(path):
|
||||
text = self.key.revision(text)
|
||||
self._inner.put_file_text(path, text, message=message)
|
||||
|
||||
def get_string_map(self, path: str) -> dict[str, str]:
|
||||
"""Bulk read; sealed entries come back opened, bad ones dropped."""
|
||||
reader = getattr(self._inner, "get_string_map", None)
|
||||
if reader is None:
|
||||
return {}
|
||||
out: dict[str, str] = {}
|
||||
for name, value in reader(path).items():
|
||||
full = f"{path}/{name}"
|
||||
if not self._is_log(full):
|
||||
out[name] = value
|
||||
continue
|
||||
try:
|
||||
out[name] = open_log_text(value, self._cipher, path=full)
|
||||
except E2eDecryptError:
|
||||
_logger.warning("Undecryptable entry %s dropped", full)
|
||||
return out
|
||||
|
||||
def delete_file(self, path: str, *, message: str = "") -> None:
|
||||
"""Delegate."""
|
||||
self._inner.delete_file(path, message=message)
|
||||
|
||||
def can_access_remote(self) -> bool:
|
||||
"""Delegate."""
|
||||
return self._inner.can_access_remote()
|
||||
|
||||
|
||||
def _blobs(store: RemoteStore, root: str) -> dict[str, str]:
|
||||
"""Every text blob under ``root`` (flat leaves and ``<dir>/<file>``)."""
|
||||
skip = {default_revs_path(root), f"{root}/e2e", f"{root}/revs"}
|
||||
out: dict[str, str] = {}
|
||||
for name in store.list_directory(root):
|
||||
full = f"{root}/{name}"
|
||||
if full in skip:
|
||||
continue
|
||||
children = store.list_directory(full)
|
||||
if not children:
|
||||
text = store.get_file_text(full)
|
||||
if text is not None:
|
||||
out[name] = text
|
||||
continue
|
||||
for child in children:
|
||||
text = store.get_file_text(f"{full}/{child}")
|
||||
if text is not None:
|
||||
out[f"{name}/{child}"] = text
|
||||
return out
|
||||
|
||||
|
||||
def copy_to_encrypted(
|
||||
store: RemoteStore, key: E2eKey, old_prefix: str, layout: E2eLayout
|
||||
) -> int:
|
||||
"""Merge every readable old log into its encrypted twin; return writes."""
|
||||
dest = EncryptingStore(store, key, layout)
|
||||
copied = 0
|
||||
for rel, text in _blobs(store, old_prefix).items():
|
||||
try:
|
||||
source = load_log(text)
|
||||
except ValueError, KeyError, TypeError:
|
||||
_logger.warning("Unreadable old log %s/%s not copied", old_prefix, rel)
|
||||
continue
|
||||
if not source:
|
||||
continue
|
||||
path = f"{layout.prefix}/{rel}"
|
||||
existing = dest.get_file_text(path)
|
||||
try:
|
||||
current = load_log(existing) if existing is not None else {}
|
||||
except ValueError, KeyError, TypeError:
|
||||
# Not a sync log (an app's own blob): never overwrite it.
|
||||
_logger.warning("Non-log blob at %s left as is", path)
|
||||
continue
|
||||
merged = merge_logs(current, source)
|
||||
if existing is not None and dump_log(merged) == dump_log(current):
|
||||
continue
|
||||
dest.put_file_text(path, dump_log(merged), message="e2e copy")
|
||||
copied += 1
|
||||
return copied
|
||||
|
||||
|
||||
_copied: dict[str, str] = {}
|
||||
|
||||
|
||||
def reset_e2e_copy_memo() -> None:
|
||||
"""Forget which old prefixes were copied; for tests."""
|
||||
_copied.clear()
|
||||
|
||||
|
||||
def _fingerprint(store: RemoteStore, old_prefix: str) -> str:
|
||||
reader = getattr(store, "get_string_map", None)
|
||||
if reader is None:
|
||||
return ""
|
||||
revs = reader(default_revs_path(old_prefix))
|
||||
return ";".join(f"{k}={revs[k]}" for k in sorted(revs))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _OpenRequest:
|
||||
old_prefix: str
|
||||
layout: E2eLayout
|
||||
passphrase: str
|
||||
params: Argon2Params | None
|
||||
state_dir: Path | None
|
||||
|
||||
|
||||
def _unlock(store: RemoteStore, req: _OpenRequest) -> E2eKey:
|
||||
old_prefix, layout, state_dir = req.old_prefix, req.layout, req.state_dir
|
||||
if state_dir is not None:
|
||||
cached = read_cached_key(state_dir, old_prefix)
|
||||
if cached is not None:
|
||||
key = E2eKey.from_bytes(cached)
|
||||
if key_matches(key, store, layout.e2e_path):
|
||||
return key
|
||||
if not req.passphrase:
|
||||
msg = "no crdt-sync passphrase on this device"
|
||||
raise E2eLockedError(msg)
|
||||
try:
|
||||
key = unlock_e2e(
|
||||
store,
|
||||
path_prefix=layout.prefix,
|
||||
passphrase=req.passphrase,
|
||||
params=req.params,
|
||||
e2e_path=layout.e2e_path,
|
||||
)
|
||||
except (E2eWrongPassphraseError, E2eHeaderError) as exc:
|
||||
raise E2eLockedError(str(exc) or type(exc).__name__) from exc
|
||||
if state_dir is not None:
|
||||
write_cached_key(state_dir, old_prefix, key.master)
|
||||
return key
|
||||
|
||||
|
||||
def open_encrypted_client(
|
||||
store: RemoteStore,
|
||||
old_prefix: str,
|
||||
*,
|
||||
passphrase: str,
|
||||
params: Argon2Params | None = None,
|
||||
state_dir: Path | None = None,
|
||||
) -> tuple[EncryptingStore, E2eLayout]:
|
||||
"""Unlock, copy the old prefix when it changed, and return the store.
|
||||
|
||||
Args:
|
||||
store: The outermost remote (above any mirror).
|
||||
old_prefix: The app's former plaintext prefix.
|
||||
passphrase: The shared crdt-sync password.
|
||||
params: Argon2id cost for a header created now; tests lower it.
|
||||
state_dir: Where a short-lived daemon keeps its derived key (0600)
|
||||
and copy memo between runs; None keeps both in memory only.
|
||||
|
||||
Raises:
|
||||
E2eLockedError: No passphrase, a wrong one, or an unreadable header.
|
||||
"""
|
||||
layout = E2eLayout.of(old_prefix)
|
||||
key = _unlock(
|
||||
store, _OpenRequest(old_prefix, layout, passphrase, params, state_dir)
|
||||
)
|
||||
fingerprint = _fingerprint(store, old_prefix)
|
||||
last = (
|
||||
read_copy_memo(state_dir, old_prefix)
|
||||
if state_dir is not None
|
||||
else _copied.get(old_prefix)
|
||||
)
|
||||
if last != fingerprint:
|
||||
copy_to_encrypted(store, key, old_prefix, layout)
|
||||
_copied[old_prefix] = fingerprint
|
||||
if state_dir is not None:
|
||||
write_copy_memo(state_dir, old_prefix, fingerprint)
|
||||
return EncryptingStore(store, key, layout), layout
|
||||
@ -13,6 +13,7 @@ from dataclasses import dataclass
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from crdt_sync._e2e_cipher import E2eDecryptError
|
||||
from crdt_sync._revisions import revision_of
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@ -90,9 +91,16 @@ def _pull_remote_logs(
|
||||
# skipped next tick.
|
||||
seen_revs[other_device_id] = remote_rev
|
||||
continue
|
||||
text = ctx.client.get_file_text(
|
||||
f"{ctx.path_prefix}/{other_device_id}/{ctx.filename}"
|
||||
)
|
||||
try:
|
||||
text = ctx.client.get_file_text(
|
||||
f"{ctx.path_prefix}/{other_device_id}/{ctx.filename}"
|
||||
)
|
||||
except E2eDecryptError:
|
||||
_logger.warning(
|
||||
"Undecryptable log from device %r (wrong key or newer app)",
|
||||
other_device_id,
|
||||
)
|
||||
continue
|
||||
if text is None:
|
||||
continue
|
||||
try:
|
||||
|
||||
10
crdt-sync/crdt_sync/tests/fixtures/e2e_dart_vectors.json
vendored
Normal file
10
crdt-sync/crdt_sync/tests/fixtures/e2e_dart_vectors.json
vendored
Normal file
@ -0,0 +1,10 @@
|
||||
{
|
||||
"passphrase": "correct horse ✓",
|
||||
"header": "{\"v\":1,\"kdf\":\"argon2id\",\"salt\":\"HGW+FZJe6hd71He22BrlUA==\",\"m\":64,\"t\":2,\"p\":1,\"check\":\"okKnATkNoZKvjSGfrgQGOjdRw5NORh2lC32zuBq7V6I=\"}",
|
||||
"master_hex": "cad5afe8ddc6e21d2f39eb196de454db6b6d1f60629619ce8539f03a4bb0b086",
|
||||
"revision_of_abc": "e7f845e1b0a7ea2b571fd3e3bdfb60641e650d63171bf73b6356c7909d2748c8",
|
||||
"path": "vec/notes-e2e/phone/log.json",
|
||||
"log": "{\"n1\":{\"id\":\"n1\",\"fields\":{\"title\":[\"buy milk\",\"1970-01-01T00:00:01.000Z-0000-node-a\"]},\"deleted\":false,\"deleted_hlc\":null},\"n2\":{\"id\":\"n2\",\"fields\":{\"title\":[\"zażółć\",\"1970-01-01T00:00:01.000Z-0000-node-a\"]},\"deleted\":false,\"deleted_hlc\":null}}",
|
||||
"envelope": "{\"v\":1,\"r\":{\"n1\":{\"d\":false,\"dh\":null,\"h\":\"1970-01-01T00:00:01.000Z-0000-node-a\",\"p\":\"AR1zODx4TD67X3TGa0N7EV/V4FcZAaqc41jxZVtskFu3Pqb/pNlqm6db66rQUaMyNLtTPa4HMOJzZomBRyaSHjaPb4oNxgVPkMeIF5IbuI/GA30gR0sz/MSvYmfZ5wSnlBo1+fpH\"},\"n2\":{\"d\":false,\"dh\":null,\"h\":\"1970-01-01T00:00:01.000Z-0000-node-a\",\"p\":\"Aa0omlBGnmlsItzq7A21G0A1tqT8xMOJE0wgxcxJ1c3hVpRmNt4kq2IPNNGuW8v4A7fvs2FrnGX4A+NVtXPpu41ckKSShTqk8Ml6zTuxyrT85g50sgzbM7PKFczeEaS+ITWDj0N04u0=\"}}}",
|
||||
"whole": "{\"v\":1,\"b\":\"AYdAwPehG84oHBRhRr5dP5VmleBGzQC1ryV5QHTCqBZE9ZsOZMMkJiP40+KzpnfIc2g=\"}"
|
||||
}
|
||||
146
crdt-sync/crdt_sync/tests/test_e2e_cipher.py
Normal file
146
crdt-sync/crdt_sync/tests/test_e2e_cipher.py
Normal file
@ -0,0 +1,146 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""E2eCipher blob errors and the per-record envelope's edge shapes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from crdt_sync import (
|
||||
E2eCipher,
|
||||
E2eDecryptError,
|
||||
is_e2e_envelope,
|
||||
open_log_text,
|
||||
seal_log_text,
|
||||
)
|
||||
|
||||
PATH = "app/devices-e2e/pc/log.json"
|
||||
CLOCK = "2026-01-01T00:00:00.000Z-0000-pc"
|
||||
|
||||
|
||||
def _cipher(*, accept_plaintext: bool = True) -> E2eCipher:
|
||||
return E2eCipher(bytes(range(32)), accept_plaintext=accept_plaintext)
|
||||
|
||||
|
||||
def _blob(raw: bytes) -> str:
|
||||
return base64.b64encode(raw).decode()
|
||||
|
||||
|
||||
def _log(**records: object) -> str:
|
||||
return json.dumps(records)
|
||||
|
||||
|
||||
def _record(rid: str = "a", **extra: object) -> dict[str, object]:
|
||||
return {
|
||||
"id": rid,
|
||||
"fields": {"title": ["x", CLOCK]},
|
||||
"deleted": False,
|
||||
"deleted_hlc": None,
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
def test_key_must_be_32_bytes() -> None:
|
||||
with pytest.raises(ValueError, match="32 bytes"):
|
||||
E2eCipher(b"short")
|
||||
|
||||
|
||||
def test_blob_round_trips() -> None:
|
||||
cipher = _cipher()
|
||||
assert cipher.open(cipher.seal("héllo", path=PATH), path=PATH) == "héllo"
|
||||
|
||||
|
||||
def test_plaintext_is_read_only_while_accepted() -> None:
|
||||
assert _cipher().open('{"a": 1}', path=PATH) == '{"a": 1}'
|
||||
with pytest.raises(E2eDecryptError, match="after migration"):
|
||||
_cipher(accept_plaintext=False).open("[1]", path=PATH)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("stored", "message"),
|
||||
[
|
||||
("!!not base64!!", "neither JSON nor base64"),
|
||||
("", "empty blob"),
|
||||
(_blob(b"\x02" + bytes(60)), "uses format 2"),
|
||||
(_blob(b"\x01" + bytes(10)), "truncated"),
|
||||
(_blob(b"\x01" + bytes(60)), "failed authentication"),
|
||||
],
|
||||
)
|
||||
def test_bad_blobs_are_refused(stored: str, message: str) -> None:
|
||||
with pytest.raises(E2eDecryptError, match=message):
|
||||
_cipher().open(stored, path=PATH)
|
||||
|
||||
|
||||
def test_blob_is_bound_to_its_path() -> None:
|
||||
sealed = _cipher().seal("secret", path=PATH)
|
||||
with pytest.raises(E2eDecryptError, match="failed authentication"):
|
||||
_cipher().open(sealed, path="other")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["not json", "[1]", '{"v": true}', '{"v": "1"}', "{}"])
|
||||
def test_non_envelopes_are_detected(text: str) -> None:
|
||||
assert not is_e2e_envelope(text)
|
||||
|
||||
|
||||
def test_envelope_is_detected() -> None:
|
||||
assert is_e2e_envelope('{"v": 1}')
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"not json",
|
||||
"[1]",
|
||||
_log(a=[1]),
|
||||
_log(a=_record("b")),
|
||||
_log(a={"id": "a", "fields": []}),
|
||||
],
|
||||
)
|
||||
def test_non_log_text_is_sealed_whole(text: str) -> None:
|
||||
sealed = seal_log_text(text, _cipher(), path=PATH)
|
||||
assert "b" in json.loads(sealed)
|
||||
assert open_log_text(sealed, _cipher(), path=PATH) == text
|
||||
|
||||
|
||||
def test_record_without_fields_has_no_newest_clock() -> None:
|
||||
empty = {"id": "a", "fields": {}, "deleted": True, "deleted_hlc": CLOCK}
|
||||
sealed = seal_log_text(_log(a=empty), _cipher(), path=PATH)
|
||||
assert json.loads(sealed)["r"]["a"]["h"] is None
|
||||
opened = json.loads(open_log_text(sealed, _cipher(), path=PATH))
|
||||
assert opened == {"a": empty}
|
||||
|
||||
|
||||
def test_plaintext_log_passes_or_fails_by_policy() -> None:
|
||||
text = _log(a=_record())
|
||||
assert open_log_text(text, _cipher(), path=PATH) == text
|
||||
with pytest.raises(E2eDecryptError, match="after migration"):
|
||||
open_log_text(text, _cipher(accept_plaintext=False), path=PATH)
|
||||
|
||||
|
||||
def test_unknown_envelope_version_is_refused() -> None:
|
||||
with pytest.raises(E2eDecryptError, match="uses format 2"):
|
||||
open_log_text('{"v": 2, "r": {}}', _cipher(), path=PATH)
|
||||
|
||||
|
||||
def test_envelope_without_records_is_misshaped() -> None:
|
||||
with pytest.raises(E2eDecryptError, match="mis-shaped"):
|
||||
open_log_text('{"v": 1, "b": 5, "r": []}', _cipher(), path=PATH)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"record",
|
||||
[[], {"p": "x", "d": 1}, {"d": False}, {"p": 3, "d": False}],
|
||||
)
|
||||
def test_misshaped_record_is_refused(record: object) -> None:
|
||||
text = json.dumps({"v": 1, "r": {"a": record}})
|
||||
with pytest.raises(E2eDecryptError, match="record a"):
|
||||
open_log_text(text, _cipher(), path=PATH)
|
||||
|
||||
|
||||
def test_tampered_record_metadata_fails_authentication() -> None:
|
||||
sealed = json.loads(seal_log_text(_log(a=_record()), _cipher(), path=PATH))
|
||||
sealed["r"]["a"]["d"] = True
|
||||
with pytest.raises(E2eDecryptError, match="failed authentication"):
|
||||
open_log_text(json.dumps(sealed), _cipher(), path=PATH)
|
||||
108
crdt-sync/crdt_sync/tests/test_e2e_cleanup.py
Normal file
108
crdt-sync/crdt_sync/tests/test_e2e_cleanup.py
Normal file
@ -0,0 +1,108 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Clearing an old plaintext prefix only after the encrypted copy is proven."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from crdt_sync import (
|
||||
Argon2Params,
|
||||
E2eLayout,
|
||||
E2eLockedError,
|
||||
clear_plaintext_prefix,
|
||||
copy_to_encrypted,
|
||||
unlock_e2e,
|
||||
unverified_blobs,
|
||||
)
|
||||
from crdt_sync.tests.conftest import FakeStore
|
||||
from tool import e2e_cleanup
|
||||
|
||||
CHEAP = Argon2Params(8, 1, 1)
|
||||
PHRASE = "pw"
|
||||
WRONG = PHRASE + "x"
|
||||
OLD = "app/devices"
|
||||
LAYOUT = E2eLayout.of(OLD)
|
||||
|
||||
|
||||
def _log(title: str, stamp: str = "2026-01-01T00:00:00.000Z") -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"n": {
|
||||
"id": "n",
|
||||
"fields": {"t": [title, f"{stamp}-0000-a"]},
|
||||
"deleted": False,
|
||||
"deleted_hlc": None,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _seeded() -> FakeStore:
|
||||
remote = FakeStore({f"{OLD}/pc/log.json": _log("x"), "app/revs/pc": "r"})
|
||||
key = unlock_e2e(
|
||||
remote,
|
||||
path_prefix=LAYOUT.prefix,
|
||||
passphrase=PHRASE,
|
||||
params=CHEAP,
|
||||
e2e_path=LAYOUT.e2e_path,
|
||||
)
|
||||
copy_to_encrypted(remote, key, OLD, LAYOUT)
|
||||
return remote
|
||||
|
||||
|
||||
def test_dry_run_lists_and_deletes_nothing() -> None:
|
||||
remote = _seeded()
|
||||
paths = clear_plaintext_prefix(remote, OLD, passphrase=PHRASE, apply=False)
|
||||
assert paths == [f"{OLD}/pc/log.json"]
|
||||
assert f"{OLD}/pc/log.json" in remote.files
|
||||
|
||||
|
||||
def test_apply_deletes_only_after_verification() -> None:
|
||||
remote = _seeded()
|
||||
clear_plaintext_prefix(remote, OLD, passphrase=PHRASE, apply=True)
|
||||
assert f"{OLD}/pc/log.json" not in remote.files
|
||||
assert f"{LAYOUT.prefix}/pc/log.json" in remote.files
|
||||
|
||||
|
||||
def test_unverified_copy_refuses() -> None:
|
||||
remote = _seeded()
|
||||
remote.files[f"{OLD}/pc/log.json"] = _log("newer", "2026-02-01T00:00:00.000Z")
|
||||
remote.files[f"{OLD}/bad/log.json"] = "{oops"
|
||||
key = unlock_e2e(
|
||||
remote, path_prefix=LAYOUT.prefix, passphrase=PHRASE, e2e_path=LAYOUT.e2e_path
|
||||
)
|
||||
assert unverified_blobs(remote, key, OLD, LAYOUT) == ["bad/log.json", "pc/log.json"]
|
||||
with pytest.raises(E2eLockedError, match="not verified"):
|
||||
clear_plaintext_prefix(remote, OLD, passphrase=PHRASE, apply=True)
|
||||
assert f"{OLD}/pc/log.json" in remote.files
|
||||
|
||||
|
||||
def test_empty_logs_need_no_copy() -> None:
|
||||
remote = _seeded()
|
||||
remote.files[f"{OLD}/tab/log.json"] = "{}"
|
||||
paths = clear_plaintext_prefix(remote, OLD, passphrase=PHRASE, apply=False)
|
||||
assert f"{OLD}/tab/log.json" in paths
|
||||
|
||||
|
||||
def test_never_creates_a_header_and_wrong_passphrase_locks() -> None:
|
||||
remote = FakeStore({f"{OLD}/pc/log.json": _log("x")})
|
||||
with pytest.raises(E2eLockedError, match="no e2e header"):
|
||||
clear_plaintext_prefix(remote, OLD, passphrase=PHRASE, apply=False)
|
||||
assert LAYOUT.e2e_path not in remote.files
|
||||
seeded = _seeded()
|
||||
with pytest.raises(E2eLockedError):
|
||||
clear_plaintext_prefix(seeded, OLD, passphrase=WRONG, apply=False)
|
||||
|
||||
|
||||
def test_tool_main(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
remote = _seeded()
|
||||
out = io.StringIO()
|
||||
monkeypatch.setattr("sys.stdout", out)
|
||||
assert e2e_cleanup.run(remote, OLD, PHRASE, apply=False) == 0
|
||||
assert "would delete 1" in out.getvalue()
|
||||
assert e2e_cleanup.run(remote, OLD, PHRASE, apply=True) == 0
|
||||
assert "deleted 1" in out.getvalue()
|
||||
assert e2e_cleanup.run(FakeStore(), OLD, PHRASE, apply=True) == 1
|
||||
74
crdt-sync/crdt_sync/tests/test_e2e_interop.py
Normal file
74
crdt-sync/crdt_sync/tests/test_e2e_interop.py
Normal file
@ -0,0 +1,74 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Cross-language golden vectors: Python opens what Dart sealed."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from crdt_sync import (
|
||||
E2eDecryptError,
|
||||
E2eKey,
|
||||
E2eWrongPassphraseError,
|
||||
open_log_text,
|
||||
seal_log_text,
|
||||
unlock_e2e,
|
||||
)
|
||||
from crdt_sync.tests.conftest import FakeStore as MemoryRemote
|
||||
|
||||
WRONG = "nope"
|
||||
VECTORS = json.loads(
|
||||
(Path(__file__).parent / "fixtures" / "e2e_dart_vectors.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _key() -> E2eKey:
|
||||
remote = MemoryRemote({"vec/e2e": VECTORS["header"]})
|
||||
return unlock_e2e(
|
||||
remote,
|
||||
path_prefix="vec/notes-e2e",
|
||||
passphrase=str(VECTORS["passphrase"]),
|
||||
e2e_path="vec/e2e",
|
||||
)
|
||||
|
||||
|
||||
def test_argon2id_and_hkdf_match_dart() -> None:
|
||||
assert _key().master.hex() == VECTORS["master_hex"]
|
||||
|
||||
|
||||
def test_keyed_revision_matches_dart() -> None:
|
||||
assert _key().revision("abc") == VECTORS["revision_of_abc"]
|
||||
|
||||
|
||||
def test_opens_a_dart_envelope() -> None:
|
||||
opened = open_log_text(VECTORS["envelope"], _key().cipher(), path=VECTORS["path"])
|
||||
assert json.loads(opened) == json.loads(VECTORS["log"])
|
||||
|
||||
|
||||
def test_opens_a_dart_whole_blob() -> None:
|
||||
opened = open_log_text(VECTORS["whole"], _key().cipher(), path=VECTORS["path"])
|
||||
assert opened == "not a log"
|
||||
|
||||
|
||||
def test_wrong_passphrase_is_refused() -> None:
|
||||
remote = MemoryRemote({"vec/e2e": VECTORS["header"]})
|
||||
with pytest.raises(E2eWrongPassphraseError):
|
||||
unlock_e2e(remote, path_prefix="x", passphrase=WRONG, e2e_path="vec/e2e")
|
||||
|
||||
|
||||
def test_a_moved_envelope_is_refused() -> None:
|
||||
with pytest.raises(E2eDecryptError):
|
||||
open_log_text(VECTORS["envelope"], _key().cipher(), path="vec/other")
|
||||
|
||||
|
||||
def test_python_round_trip() -> None:
|
||||
cipher = _key().cipher()
|
||||
sealed = seal_log_text(VECTORS["log"], cipher, path=VECTORS["path"])
|
||||
assert "buy milk" not in sealed
|
||||
assert json.loads(open_log_text(sealed, cipher, path=VECTORS["path"])) == (
|
||||
json.loads(VECTORS["log"])
|
||||
)
|
||||
123
crdt-sync/crdt_sync/tests/test_e2e_key.py
Normal file
123
crdt-sync/crdt_sync/tests/test_e2e_key.py
Normal file
@ -0,0 +1,123 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Argon2id key, header parsing and unlock edge cases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from crdt_sync import (
|
||||
Argon2Params,
|
||||
E2eHeaderError,
|
||||
E2eKey,
|
||||
E2eWrongPassphraseError,
|
||||
default_e2e_path,
|
||||
unlock_e2e,
|
||||
)
|
||||
from crdt_sync._e2e_key import key_matches, read_header
|
||||
from crdt_sync.tests.conftest import FakeStore
|
||||
|
||||
CHEAP = Argon2Params(8, 1, 1)
|
||||
HEADER = {
|
||||
"v": 1,
|
||||
"kdf": "argon2id",
|
||||
"salt": "AAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"m": 8,
|
||||
"t": 1,
|
||||
"p": 1,
|
||||
"check": "c",
|
||||
}
|
||||
|
||||
|
||||
def _unlock(remote: FakeStore, phrase: str = "pw") -> E2eKey:
|
||||
return unlock_e2e(
|
||||
remote,
|
||||
path_prefix="app/devices-e2e",
|
||||
passphrase=phrase,
|
||||
params=CHEAP,
|
||||
e2e_path="app/e2e",
|
||||
)
|
||||
|
||||
|
||||
class RacedStore(FakeStore):
|
||||
"""Another device's header wins the first write."""
|
||||
|
||||
def __init__(self, winner: str) -> None:
|
||||
super().__init__()
|
||||
self._winner = winner
|
||||
|
||||
def put_file_text(self, path: str, text: str, *, message: str) -> None:
|
||||
super().put_file_text(path, self._winner, message=message)
|
||||
|
||||
|
||||
def test_default_path_is_a_sibling_or_nested() -> None:
|
||||
assert default_e2e_path("app/devices-e2e") == "app/e2e"
|
||||
assert default_e2e_path("flat") == "flat/e2e"
|
||||
|
||||
|
||||
def test_master_key_must_be_32_bytes() -> None:
|
||||
with pytest.raises(ValueError, match="32 bytes"):
|
||||
E2eKey.from_bytes(b"short")
|
||||
|
||||
|
||||
def test_unlock_refuses_an_empty_passphrase() -> None:
|
||||
with pytest.raises(ValueError, match="empty"):
|
||||
_unlock(FakeStore(), "")
|
||||
|
||||
|
||||
def test_unlock_creates_then_reuses_the_header() -> None:
|
||||
remote = FakeStore()
|
||||
first = _unlock(remote)
|
||||
assert _unlock(remote).master == first.master
|
||||
assert key_matches(first, remote, "app/e2e")
|
||||
assert not key_matches(_unlock(FakeStore(), "other"), remote, "app/e2e")
|
||||
|
||||
|
||||
def test_missing_header_matches_nothing() -> None:
|
||||
key = E2eKey.from_bytes(bytes(32))
|
||||
assert read_header(FakeStore(), "app/e2e") is None
|
||||
assert not key_matches(key, FakeStore(), "app/e2e")
|
||||
|
||||
|
||||
def test_wrong_passphrase_is_refused() -> None:
|
||||
remote = FakeStore()
|
||||
_unlock(remote)
|
||||
with pytest.raises(E2eWrongPassphraseError):
|
||||
_unlock(remote, "nope")
|
||||
|
||||
|
||||
def test_losing_the_header_race_adopts_the_winner() -> None:
|
||||
winner = FakeStore()
|
||||
key = _unlock(winner)
|
||||
raced = RacedStore(winner.files["app/e2e"])
|
||||
assert _unlock(raced).master == key.master
|
||||
with pytest.raises(E2eWrongPassphraseError):
|
||||
_unlock(RacedStore(winner.files["app/e2e"]), "nope")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "message"),
|
||||
[
|
||||
("not json", "not JSON"),
|
||||
("[1]", "known version"),
|
||||
(json.dumps({**HEADER, "v": 2}), "known version"),
|
||||
(json.dumps({**HEADER, "kdf": "scrypt"}), "malformed"),
|
||||
(json.dumps({k: v for k, v in HEADER.items() if k != "m"}), "malformed"),
|
||||
(json.dumps({**HEADER, "m": "x"}), "malformed"),
|
||||
(json.dumps({**HEADER, "salt": "A"}), "malformed"),
|
||||
],
|
||||
)
|
||||
def test_bad_headers_are_refused(text: str, message: str) -> None:
|
||||
remote = FakeStore({"app/e2e": text})
|
||||
with pytest.raises(E2eHeaderError, match=message):
|
||||
read_header(remote, "app/e2e")
|
||||
with pytest.raises(E2eHeaderError, match=message):
|
||||
_unlock(remote)
|
||||
|
||||
|
||||
def test_a_good_header_parses() -> None:
|
||||
header = read_header(FakeStore({"app/e2e": json.dumps(HEADER)}), "app/e2e")
|
||||
assert header is not None
|
||||
assert header.params == CHEAP
|
||||
assert header.check == "c"
|
||||
101
crdt-sync/crdt_sync/tests/test_e2e_state_dir.py
Normal file
101
crdt-sync/crdt_sync/tests/test_e2e_state_dir.py
Normal file
@ -0,0 +1,101 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""A per-minute daemon must not pay Argon2id or a full copy every tick."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from crdt_sync import Argon2Params, E2eLockedError, open_encrypted_client
|
||||
from crdt_sync import _e2e_key as key_mod
|
||||
from crdt_sync import _e2e_store as store_mod
|
||||
from crdt_sync.tests.conftest import FakeStore
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
from crdt_sync import E2eKey, RemoteStore
|
||||
|
||||
CHEAP = Argon2Params(8, 1, 1)
|
||||
PHRASE = "pw"
|
||||
WRONG = PHRASE + "x"
|
||||
LOG = json.dumps(
|
||||
{
|
||||
"a": {
|
||||
"id": "a",
|
||||
"fields": {"t": ["x", "2026-01-01T00:00:00.000Z-0000-a"]},
|
||||
"deleted": False,
|
||||
"deleted_hlc": None,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_state_dir_caches_key_and_copy_across_processes(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
remote = FakeStore({"app/devices/pc/log.json": LOG, "app/revs/pc": "r1"})
|
||||
derives: list[int] = []
|
||||
real = key_mod.unlock_e2e
|
||||
|
||||
def counting(
|
||||
remote: RemoteStore,
|
||||
*,
|
||||
path_prefix: str,
|
||||
passphrase: str,
|
||||
params: Argon2Params | None = None,
|
||||
e2e_path: str | None = None,
|
||||
) -> E2eKey:
|
||||
derives.append(1)
|
||||
return real(
|
||||
remote,
|
||||
path_prefix=path_prefix,
|
||||
passphrase=passphrase,
|
||||
params=params,
|
||||
e2e_path=e2e_path,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(store_mod, "unlock_e2e", counting)
|
||||
for _ in range(3):
|
||||
store_mod.reset_e2e_copy_memo() # a fresh process each tick
|
||||
remote.writes.clear()
|
||||
open_encrypted_client(
|
||||
remote, "app/devices", passphrase=PHRASE, params=CHEAP, state_dir=tmp_path
|
||||
)
|
||||
assert derives == [1]
|
||||
assert remote.writes == []
|
||||
key_files = list(tmp_path.glob("e2e-*.key"))
|
||||
assert len(key_files) == 1
|
||||
assert key_files[0].stat().st_mode & 0o777 == 0o600
|
||||
|
||||
|
||||
def test_stale_cached_key_rederives(tmp_path: Path) -> None:
|
||||
remote = FakeStore()
|
||||
open_encrypted_client(
|
||||
remote, "app/devices", passphrase=PHRASE, params=CHEAP, state_dir=tmp_path
|
||||
)
|
||||
for key_file in tmp_path.glob("e2e-*.key"):
|
||||
key_file.write_text("00" * 32, encoding="utf-8")
|
||||
store, _ = open_encrypted_client(
|
||||
remote, "app/devices", passphrase=PHRASE, params=CHEAP, state_dir=tmp_path
|
||||
)
|
||||
assert next(tmp_path.glob("e2e-*.key")).read_text(encoding="utf-8") == (
|
||||
store.key.master.hex()
|
||||
)
|
||||
|
||||
|
||||
def test_garbage_cached_key_is_ignored_and_wrong_passphrase_locks(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
remote = FakeStore()
|
||||
open_encrypted_client(
|
||||
remote, "app/devices", passphrase=PHRASE, params=CHEAP, state_dir=tmp_path
|
||||
)
|
||||
for key_file in tmp_path.glob("e2e-*.key"):
|
||||
key_file.write_text("not hex", encoding="utf-8")
|
||||
with pytest.raises(E2eLockedError):
|
||||
open_encrypted_client(
|
||||
remote, "app/devices", passphrase=WRONG, params=CHEAP, state_dir=tmp_path
|
||||
)
|
||||
118
crdt-sync/crdt_sync/tests/test_e2e_store.py
Normal file
118
crdt-sync/crdt_sync/tests/test_e2e_store.py
Normal file
@ -0,0 +1,118 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""EncryptingStore, layout, one-time copy and open, mirroring the Dart side."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from crdt_sync import (
|
||||
Argon2Params,
|
||||
E2eLayout,
|
||||
E2eLockedError,
|
||||
EncryptingStore,
|
||||
LogCodec,
|
||||
SyncTarget,
|
||||
copy_to_encrypted,
|
||||
dump_log,
|
||||
load_log,
|
||||
open_encrypted_client,
|
||||
reset_e2e_copy_memo,
|
||||
sync_log,
|
||||
unlock_e2e,
|
||||
)
|
||||
from crdt_sync.tests.conftest import FakeStore
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from crdt_sync import Hlc
|
||||
|
||||
PHRASE = "pw"
|
||||
WRONG = "bad"
|
||||
CHEAP = Argon2Params(8, 1, 1)
|
||||
OLD = "app/devices"
|
||||
|
||||
|
||||
def _log_text(make_hlc: Callable[..., Hlc], **titles: str) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
rid: {
|
||||
"id": rid,
|
||||
"fields": {"title": [t, make_hlc(1000).to_str()]},
|
||||
"deleted": False,
|
||||
"deleted_hlc": None,
|
||||
}
|
||||
for rid, t in titles.items()
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _memo() -> None:
|
||||
reset_e2e_copy_memo()
|
||||
|
||||
|
||||
def test_layouts() -> None:
|
||||
assert E2eLayout.of("app/devices") == E2eLayout(
|
||||
"app/devices-e2e", "app/revs-e2e", "app/e2e"
|
||||
)
|
||||
assert E2eLayout.of("settings-sync") == E2eLayout(
|
||||
"settings-sync-e2e", "settings-sync-e2e/revs", "settings-sync-e2e/e2e"
|
||||
)
|
||||
|
||||
|
||||
def test_sync_through_store_is_ciphertext_and_merges(
|
||||
make_hlc: Callable[..., Hlc],
|
||||
) -> None:
|
||||
remote = FakeStore()
|
||||
layout = E2eLayout.of(OLD)
|
||||
key = unlock_e2e(
|
||||
remote,
|
||||
path_prefix=layout.prefix,
|
||||
passphrase=PHRASE,
|
||||
params=CHEAP,
|
||||
e2e_path=layout.e2e_path,
|
||||
)
|
||||
store = EncryptingStore(remote, key, layout)
|
||||
codec = LogCodec(encode=dump_log, decode=load_log, filename="log.json")
|
||||
phone = load_log(_log_text(make_hlc, n1="buy milk"))
|
||||
sync_log(SyncTarget(store, "phone", layout.prefix), phone, codec)
|
||||
blob = remote.files[f"{layout.prefix}/phone/log.json"]
|
||||
assert "buy milk" not in blob
|
||||
assert '"title"' not in blob
|
||||
merged = sync_log(SyncTarget(store, "pc", layout.prefix), {}, codec)
|
||||
assert set(merged) == {"n1"}
|
||||
|
||||
|
||||
def test_open_copies_old_prefix_once_and_locks_without_password(
|
||||
make_hlc: Callable[..., Hlc],
|
||||
) -> None:
|
||||
remote = FakeStore({f"{OLD}/pc/log.json": _log_text(make_hlc, a="x")})
|
||||
with pytest.raises(E2eLockedError):
|
||||
open_encrypted_client(remote, OLD, passphrase="", params=CHEAP)
|
||||
store, layout = open_encrypted_client(remote, OLD, passphrase=PHRASE, params=CHEAP)
|
||||
text = store.get_file_text(f"{layout.prefix}/pc/log.json")
|
||||
assert text is not None
|
||||
assert set(load_log(text)) == {"a"}
|
||||
assert "x" in remote.files[f"{OLD}/pc/log.json"]
|
||||
with pytest.raises(E2eLockedError):
|
||||
open_encrypted_client(remote, OLD, passphrase=WRONG, params=CHEAP)
|
||||
|
||||
|
||||
def test_copy_is_idempotent(make_hlc: Callable[..., Hlc]) -> None:
|
||||
remote = FakeStore({f"{OLD}/pc/log.json": _log_text(make_hlc, a="x")})
|
||||
layout = E2eLayout.of(OLD)
|
||||
key = unlock_e2e(
|
||||
remote,
|
||||
path_prefix=layout.prefix,
|
||||
passphrase=PHRASE,
|
||||
params=CHEAP,
|
||||
e2e_path=layout.e2e_path,
|
||||
)
|
||||
assert copy_to_encrypted(remote, key, OLD, layout) == 1
|
||||
remote.writes.clear()
|
||||
assert copy_to_encrypted(remote, key, OLD, layout) == 0
|
||||
assert remote.writes == []
|
||||
217
crdt-sync/crdt_sync/tests/test_e2e_store_edges.py
Normal file
217
crdt-sync/crdt_sync/tests/test_e2e_store_edges.py
Normal file
@ -0,0 +1,217 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""EncryptingStore passthroughs, bulk reads and copy/open edge cases."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from crdt_sync import (
|
||||
Argon2Params,
|
||||
E2eLayout,
|
||||
E2eLockedError,
|
||||
EncryptingStore,
|
||||
copy_to_encrypted,
|
||||
load_log,
|
||||
open_encrypted_client,
|
||||
reset_e2e_copy_memo,
|
||||
unlock_e2e,
|
||||
)
|
||||
from crdt_sync._e2e_store import _blobs, _fingerprint
|
||||
from crdt_sync.tests.conftest import FakeStore, FakeStoreWithoutBulkRead
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from crdt_sync import E2eKey, Hlc
|
||||
|
||||
PHRASE = "pw"
|
||||
CHEAP = Argon2Params(8, 1, 1)
|
||||
OLD = "app/devices"
|
||||
LAYOUT = E2eLayout.of(OLD)
|
||||
CLOCK = "2026-01-01T00:00:00.000Z-0000-pc"
|
||||
|
||||
|
||||
def _log(title: str = "x", rid: str = "a") -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
rid: {
|
||||
"id": rid,
|
||||
"fields": {"title": [title, CLOCK]},
|
||||
"deleted": False,
|
||||
"deleted_hlc": None,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _key(remote: FakeStore) -> E2eKey:
|
||||
return unlock_e2e(
|
||||
remote,
|
||||
path_prefix=LAYOUT.prefix,
|
||||
passphrase=PHRASE,
|
||||
params=CHEAP,
|
||||
e2e_path=LAYOUT.e2e_path,
|
||||
)
|
||||
|
||||
|
||||
class HidingStore(FakeStore):
|
||||
"""Lists every file but reads ``hidden`` paths as absent."""
|
||||
|
||||
def __init__(self, files: dict[str, str], hidden: set[str]) -> None:
|
||||
super().__init__(files)
|
||||
self._hidden = hidden
|
||||
|
||||
def get_file_text(self, path: str) -> str | None:
|
||||
if path in self._hidden:
|
||||
return None
|
||||
return super().get_file_text(path)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _memo() -> None:
|
||||
reset_e2e_copy_memo()
|
||||
|
||||
|
||||
def test_passthroughs_and_keyed_revisions() -> None:
|
||||
remote = FakeStore({"other/file": "raw"})
|
||||
store = EncryptingStore(remote, _key(remote), LAYOUT)
|
||||
assert store.list_directory("other") == ["file"]
|
||||
assert store.get_file_text("other/file") == "raw"
|
||||
assert store.get_file_text(f"{LAYOUT.prefix}/pc/log.json") is None
|
||||
assert store.can_access_remote()
|
||||
store.put_file_text(f"{LAYOUT.revs_path}/pc", "text", message="m")
|
||||
assert remote.files[f"{LAYOUT.revs_path}/pc"] == store.key.revision("text")
|
||||
store.put_file_text("other/file", "plain", message="m")
|
||||
assert remote.files["other/file"] == "plain"
|
||||
store.delete_file("other/file")
|
||||
assert "other/file" not in remote.files
|
||||
|
||||
|
||||
def test_bulk_read_opens_sealed_and_drops_bad(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
remote = FakeStore()
|
||||
store = EncryptingStore(remote, _key(remote), LAYOUT)
|
||||
store.put_file_text(f"{LAYOUT.prefix}/pc", _log(), message="m")
|
||||
remote.files[f"{LAYOUT.prefix}/bad"] = '{"v": 2, "r": {}}'
|
||||
remote.files[f"{LAYOUT.revs_path}/pc"] = "rev"
|
||||
with caplog.at_level(logging.WARNING):
|
||||
out = store.get_string_map(LAYOUT.prefix)
|
||||
assert set(out) == {"pc"}
|
||||
assert set(load_log(out["pc"])) == {"a"}
|
||||
assert "Undecryptable entry" in caplog.text
|
||||
assert store.get_string_map(LAYOUT.revs_path) == {"pc": "rev"}
|
||||
|
||||
|
||||
def test_bulk_read_is_empty_without_inner_support() -> None:
|
||||
remote = FakeStoreWithoutBulkRead()
|
||||
store = EncryptingStore(remote, _key(FakeStore()), LAYOUT)
|
||||
assert store.get_string_map(LAYOUT.prefix) == {}
|
||||
assert _fingerprint(remote, OLD) == ""
|
||||
|
||||
|
||||
def test_blobs_cover_flat_nested_and_skipped_paths() -> None:
|
||||
remote = HidingStore(
|
||||
{
|
||||
f"{OLD}/flat": "f",
|
||||
f"{OLD}/gone": "g",
|
||||
f"{OLD}/pc/log.json": "n",
|
||||
f"{OLD}/pc/hidden.json": "h",
|
||||
"app/revs/pc": "r",
|
||||
f"{OLD}/e2e": "h",
|
||||
f"{OLD}/revs": "r",
|
||||
},
|
||||
{f"{OLD}/gone", f"{OLD}/pc/hidden.json"},
|
||||
)
|
||||
assert _blobs(remote, OLD) == {"flat": "f", "pc/log.json": "n"}
|
||||
|
||||
|
||||
def test_copy_skips_unreadable_and_flat_logs(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
remote = FakeStore({f"{OLD}/flat": _log(), f"{OLD}/pc/log.json": "not json"})
|
||||
with caplog.at_level(logging.WARNING):
|
||||
copied = copy_to_encrypted(remote, _key(remote), OLD, LAYOUT)
|
||||
assert copied == 1
|
||||
assert "Unreadable old log" in caplog.text
|
||||
assert f"{LAYOUT.prefix}/flat" in remote.files
|
||||
assert f"{LAYOUT.prefix}/pc/log.json" not in remote.files
|
||||
|
||||
|
||||
def test_changed_old_prefix_is_copied_again() -> None:
|
||||
remote = FakeStore({f"{OLD}/pc/log.json": _log("one")})
|
||||
open_encrypted_client(remote, OLD, passphrase=PHRASE, params=CHEAP)
|
||||
remote.files[f"{OLD}/pc/log.json"] = _log("two", rid="b")
|
||||
remote.files["app/revs/pc"] = "newer"
|
||||
store, layout = open_encrypted_client(remote, OLD, passphrase=PHRASE, params=CHEAP)
|
||||
text = store.get_file_text(f"{layout.prefix}/pc/log.json")
|
||||
assert text is not None
|
||||
assert set(load_log(text)) == {"a", "b"}
|
||||
|
||||
|
||||
def test_unchanged_prefix_is_not_copied_twice() -> None:
|
||||
remote = FakeStore({f"{OLD}/pc/log.json": _log()})
|
||||
open_encrypted_client(remote, OLD, passphrase=PHRASE, params=CHEAP)
|
||||
remote.writes.clear()
|
||||
open_encrypted_client(remote, OLD, passphrase=PHRASE, params=CHEAP)
|
||||
assert remote.writes == []
|
||||
|
||||
|
||||
def test_unreadable_header_locks_the_client() -> None:
|
||||
remote = FakeStore({LAYOUT.e2e_path: "not json"})
|
||||
with pytest.raises(E2eLockedError, match="not JSON"):
|
||||
open_encrypted_client(remote, OLD, passphrase=PHRASE, params=CHEAP)
|
||||
|
||||
|
||||
def test_copy_never_overwrites_non_log_and_skips_empty() -> None:
|
||||
layout = E2eLayout.of("app/devices")
|
||||
remote = FakeStore(
|
||||
{
|
||||
"app/devices/pc/current.json": "{}",
|
||||
"app/devices-e2e/pc/current.json": '{"photos":["a"]}',
|
||||
"app/devices/tab/current.json": "{}",
|
||||
}
|
||||
)
|
||||
key = unlock_e2e(
|
||||
remote,
|
||||
path_prefix=layout.prefix,
|
||||
passphrase="x" * 8,
|
||||
params=Argon2Params(8, 1, 1),
|
||||
e2e_path=layout.e2e_path,
|
||||
)
|
||||
assert copy_to_encrypted(remote, key, "app/devices", layout) == 0
|
||||
assert remote.files["app/devices-e2e/pc/current.json"] == '{"photos":["a"]}'
|
||||
assert "app/devices-e2e/tab/current.json" not in remote.files
|
||||
|
||||
|
||||
def test_copy_leaves_a_non_log_destination_alone(
|
||||
make_hlc: Callable[..., Hlc],
|
||||
) -> None:
|
||||
layout = E2eLayout.of("app/devices")
|
||||
log = {
|
||||
"n": {
|
||||
"id": "n",
|
||||
"fields": {"t": ["x", make_hlc(1).to_str()]},
|
||||
"deleted": False,
|
||||
"deleted_hlc": None,
|
||||
}
|
||||
}
|
||||
remote = FakeStore(
|
||||
{
|
||||
"app/devices/pc/current.json": json.dumps(log),
|
||||
"app/devices-e2e/pc/current.json": '{"photos":["a"]}',
|
||||
}
|
||||
)
|
||||
key = unlock_e2e(
|
||||
remote,
|
||||
path_prefix=layout.prefix,
|
||||
passphrase="x" * 8,
|
||||
params=Argon2Params(8, 1, 1),
|
||||
e2e_path=layout.e2e_path,
|
||||
)
|
||||
assert copy_to_encrypted(remote, key, "app/devices", layout) == 0
|
||||
assert remote.files["app/devices-e2e/pc/current.json"] == '{"photos":["a"]}'
|
||||
32
crdt-sync/crdt_sync/tests/test_sync_e2e_skip.py
Normal file
32
crdt-sync/crdt_sync/tests/test_sync_e2e_skip.py
Normal file
@ -0,0 +1,32 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""A peer whose log cannot be decrypted is skipped, not fatal."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from crdt_sync import (
|
||||
E2eDecryptError,
|
||||
GitHubSyncClient,
|
||||
LogCodec,
|
||||
SyncTarget,
|
||||
dump_log,
|
||||
load_log,
|
||||
sync_log,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import pytest
|
||||
|
||||
|
||||
def test_undecryptable_peer_is_skipped(caplog: pytest.LogCaptureFixture) -> None:
|
||||
client = MagicMock(spec=GitHubSyncClient)
|
||||
client.list_directory.return_value = ["phone"]
|
||||
client.get_file_text.side_effect = E2eDecryptError("wrong key")
|
||||
codec = LogCodec(encode=dump_log, decode=load_log, filename="log.json")
|
||||
with caplog.at_level(logging.WARNING):
|
||||
merged = sync_log(SyncTarget(client, "pc", "devices"), {}, codec)
|
||||
assert merged == {}
|
||||
assert "Undecryptable log from device 'phone'" in caplog.text
|
||||
120
crdt-sync/crdt_sync/tests/test_tool_client_probe.py
Normal file
120
crdt-sync/crdt_sync/tests/test_tool_client_probe.py
Normal file
@ -0,0 +1,120 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Tests for ``tool/client_probe``: the click-free "does the client work" check."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import requests
|
||||
|
||||
from tool import client_probe as probe
|
||||
from tool.desktop_client import DesktopClient
|
||||
|
||||
_HELD = "stored-client-credential"
|
||||
_CLIENT = DesktopClient(
|
||||
client_id="1-abc.apps.googleusercontent.com",
|
||||
client_secret=_HELD,
|
||||
project_id="kuhy-syncs",
|
||||
)
|
||||
|
||||
|
||||
def _location(payload: bytes) -> str:
|
||||
encoded = base64.urlsafe_b64encode(payload).decode().rstrip("=")
|
||||
return f"https://accounts.google.com/signin/oauth/error?authError={encoded}&x=1"
|
||||
|
||||
|
||||
def test_auth_location_without_error_is_ok() -> None:
|
||||
result = probe.classify_auth_location("https://accounts.google.com/v3/signin")
|
||||
assert result.ok
|
||||
|
||||
|
||||
def test_auth_location_none_is_ok() -> None:
|
||||
assert probe.classify_auth_location(None).ok
|
||||
|
||||
|
||||
def test_auth_error_is_decoded() -> None:
|
||||
location = _location(b"\n\x0finvalid_request\x12\x02 Custom URI scheme is bad")
|
||||
result = probe.classify_auth_location(location)
|
||||
assert not result.ok
|
||||
assert "invalid_request" in result.detail
|
||||
assert "Custom URI scheme" in result.detail
|
||||
|
||||
|
||||
def test_undecodable_auth_error_is_still_a_failure() -> None:
|
||||
result = probe.classify_auth_location(
|
||||
"https://accounts.google.com/signin/oauth/error?authError=!!!"
|
||||
)
|
||||
assert not result.ok
|
||||
assert "authError=!!!" in result.detail
|
||||
|
||||
|
||||
def test_token_invalid_grant_means_client_and_secret_are_good() -> None:
|
||||
assert probe.classify_token(400, {"error": "invalid_grant"}).ok
|
||||
|
||||
|
||||
def test_token_client_not_found_means_still_propagating() -> None:
|
||||
result = probe.classify_token(
|
||||
401,
|
||||
{
|
||||
"error": "invalid_client",
|
||||
"error_description": "The OAuth client was not found.",
|
||||
},
|
||||
)
|
||||
assert not result.ok
|
||||
assert "--verify-only" in result.detail
|
||||
|
||||
|
||||
def test_token_invalid_client_means_wrong_secret() -> None:
|
||||
result = probe.classify_token(
|
||||
401, {"error": "invalid_client", "error_description": "Unauthorized"}
|
||||
)
|
||||
assert not result.ok
|
||||
assert "secret" in result.detail
|
||||
|
||||
|
||||
def test_token_other_error_is_reported_verbatim() -> None:
|
||||
result = probe.classify_token(500, {"error": "backend_error"})
|
||||
assert not result.ok
|
||||
assert "backend_error" in result.detail
|
||||
|
||||
|
||||
def test_token_non_json_body_is_reported() -> None:
|
||||
result = probe.classify_token(502, None)
|
||||
assert not result.ok
|
||||
assert "502" in result.detail
|
||||
|
||||
|
||||
def _fakes(location: str | None, status: int, body: object) -> tuple:
|
||||
get = MagicMock(
|
||||
return_value=MagicMock(headers={"Location": location} if location else {})
|
||||
)
|
||||
post_response = MagicMock(status_code=status)
|
||||
if body is None:
|
||||
post_response.json.side_effect = ValueError
|
||||
else:
|
||||
post_response.json.return_value = body
|
||||
return get, MagicMock(return_value=post_response)
|
||||
|
||||
|
||||
def test_probe_client_runs_both_checks() -> None:
|
||||
get, post = _fakes(None, 400, {"error": "invalid_grant"})
|
||||
results = probe.probe_client(_CLIENT, get=get, post=post)
|
||||
assert [r.ok for r in results] == [True, True]
|
||||
assert get.call_args.kwargs["allow_redirects"] is False
|
||||
assert post.call_args.kwargs["data"]["client_secret"] == _HELD
|
||||
|
||||
|
||||
def test_probe_client_turns_network_errors_into_failures() -> None:
|
||||
get = MagicMock(side_effect=requests.ConnectionError("down"))
|
||||
post = MagicMock(side_effect=requests.ConnectionError("down"))
|
||||
results = probe.probe_client(_CLIENT, get=get, post=post)
|
||||
assert [r.ok for r in results] == [False, False]
|
||||
assert "down" in results[0].detail
|
||||
|
||||
|
||||
def test_probe_client_tolerates_a_non_json_token_reply() -> None:
|
||||
get, post = _fakes(None, 502, None)
|
||||
results = probe.probe_client(_CLIENT, get=get, post=post)
|
||||
assert results[1].ok is False
|
||||
assert "502" in results[1].detail
|
||||
186
crdt-sync/crdt_sync/tests/test_tool_desktop_client.py
Normal file
186
crdt-sync/crdt_sync/tests/test_tool_desktop_client.py
Normal file
@ -0,0 +1,186 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Tests for ``tool/desktop_client``: picking up and validating the client JSON.
|
||||
|
||||
The Android client JSON Google offers for download has the same top-level
|
||||
``installed`` key and project as a Desktop one, so the negative fixture here is
|
||||
that exact shape: it must be refused, or the wrong client gets stored.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from tool import desktop_client as dc
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
_CLIENT_ID = "1-abc.apps.googleusercontent.com"
|
||||
_HELD = "the-desktop-client-credential"
|
||||
|
||||
|
||||
def _installed(**overrides: object) -> dict[str, object]:
|
||||
body: dict[str, object] = {
|
||||
"client_id": _CLIENT_ID,
|
||||
"project_id": "kuhy-syncs",
|
||||
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
|
||||
"token_uri": "https://oauth2.googleapis.com/token",
|
||||
"client_secret": _HELD,
|
||||
"redirect_uris": ["http://localhost"],
|
||||
}
|
||||
body.update(overrides)
|
||||
return body
|
||||
|
||||
|
||||
def _text(**overrides: object) -> str:
|
||||
return json.dumps({"installed": _installed(**overrides)})
|
||||
|
||||
|
||||
def _android_text() -> str:
|
||||
body = _installed()
|
||||
del body["client_secret"]
|
||||
del body["redirect_uris"]
|
||||
return json.dumps({"installed": body})
|
||||
|
||||
|
||||
def test_valid_desktop_json_parses() -> None:
|
||||
client = dc.parse_client_json(_text())
|
||||
assert client == dc.DesktopClient(_CLIENT_ID, _HELD, "kuhy-syncs")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "needle"),
|
||||
[
|
||||
("{not json", "not valid JSON"),
|
||||
("[]", "JSON object"),
|
||||
(json.dumps({"web": _installed()}), "'installed'"),
|
||||
(_text(project_id="other"), "other"),
|
||||
(_text(client_id="plain"), "client_id"),
|
||||
(_android_text(), "client_secret"),
|
||||
(_text(redirect_uris=["https://x.example"]), "http://localhost"),
|
||||
(json.dumps({"installed": "nope"}), "'installed'"),
|
||||
],
|
||||
)
|
||||
def test_invalid_json_is_refused_with_a_reason(text: str, needle: str) -> None:
|
||||
with pytest.raises(dc.ClientFileError, match=needle):
|
||||
dc.parse_client_json(text)
|
||||
|
||||
|
||||
def _touch(directory: Path, name: str, text: str, mtime: float) -> Path:
|
||||
path = directory / name
|
||||
path.write_text(text, encoding="utf-8")
|
||||
os.utime(path, (mtime, mtime))
|
||||
return path
|
||||
|
||||
|
||||
def _wait(directory: Path, since: float, skipped: list[str]) -> dc.Pickup | None:
|
||||
return dc.wait_for_client_json(
|
||||
directory, since, attempts=3, pause=lambda: None, report=skipped.append
|
||||
)
|
||||
|
||||
|
||||
def test_pickup_ignores_old_files_and_the_android_json(tmp_path: Path) -> None:
|
||||
_touch(tmp_path, "client_secret_old.json", _text(), 50.0)
|
||||
_touch(tmp_path, "client_secret_android.json", _android_text(), 150.0)
|
||||
_touch(tmp_path, "notes.json", _text(), 150.0)
|
||||
skipped: list[str] = []
|
||||
assert _wait(tmp_path, 100.0, skipped) is None
|
||||
assert len(skipped) == 1
|
||||
assert "client_secret_android.json" in skipped[0]
|
||||
assert "client_secret" in skipped[0]
|
||||
|
||||
|
||||
def test_pickup_takes_the_newest_valid_file(tmp_path: Path) -> None:
|
||||
_touch(
|
||||
tmp_path,
|
||||
"client_secret_a.json",
|
||||
_text(client_id="1-a.apps.googleusercontent.com"),
|
||||
120.0,
|
||||
)
|
||||
newest = _touch(tmp_path, "client_secret_b.json", _text(), 130.0)
|
||||
found = _wait(tmp_path, 100.0, [])
|
||||
assert found is not None
|
||||
assert found.path == newest
|
||||
assert found.client.client_id == _CLIENT_ID
|
||||
|
||||
|
||||
def test_pickup_waits_for_the_file_to_appear(tmp_path: Path) -> None:
|
||||
def pause() -> None:
|
||||
_touch(tmp_path, "client_secret_late.json", _text(), 500.0)
|
||||
|
||||
found = dc.wait_for_client_json(tmp_path, 100.0, attempts=3, pause=pause)
|
||||
assert found is not None
|
||||
|
||||
|
||||
def test_pickup_pauses_between_attempts_only(tmp_path: Path) -> None:
|
||||
pauses: list[int] = []
|
||||
result = dc.wait_for_client_json(
|
||||
tmp_path, 0.0, attempts=3, pause=lambda: pauses.append(1)
|
||||
)
|
||||
assert result is None
|
||||
assert len(pauses) == 2
|
||||
|
||||
|
||||
def test_store_client_is_private_and_consumes_the_download(tmp_path: Path) -> None:
|
||||
source = _touch(tmp_path, "client_secret_x.json", _text(), 1.0)
|
||||
dest = tmp_path / "conf" / "desktop_client.json"
|
||||
dc.store_client(source, dest)
|
||||
assert stat.S_IMODE(dest.stat().st_mode) == 0o600
|
||||
assert dc.load_client(dest).client_id == _CLIENT_ID
|
||||
assert not source.exists()
|
||||
|
||||
|
||||
def test_store_client_overwrites_and_tightens_an_existing_file(tmp_path: Path) -> None:
|
||||
dest = tmp_path / "desktop_client.json"
|
||||
dest.write_text("old", encoding="utf-8")
|
||||
dest.chmod(0o644)
|
||||
dc.store_client(_touch(tmp_path, "client_secret_y.json", _text(), 1.0), dest)
|
||||
assert stat.S_IMODE(dest.stat().st_mode) == 0o600
|
||||
assert dest.read_text(encoding="utf-8") == _text()
|
||||
|
||||
|
||||
def test_load_client_points_at_the_register_script_when_missing(tmp_path: Path) -> None:
|
||||
with pytest.raises(dc.ClientFileError, match=r"register_desktop_client\.sh"):
|
||||
dc.load_client(tmp_path / "absent.json")
|
||||
|
||||
|
||||
def test_main_pickup_stores_the_client(tmp_path: Path) -> None:
|
||||
_touch(tmp_path, "client_secret_z.json", _text(), 200.0)
|
||||
dest = tmp_path / "out" / "desktop_client.json"
|
||||
code = dc.main(
|
||||
[
|
||||
"pickup",
|
||||
"--since",
|
||||
"100",
|
||||
"--timeout",
|
||||
"5",
|
||||
"--downloads",
|
||||
str(tmp_path),
|
||||
"--dest",
|
||||
str(dest),
|
||||
]
|
||||
)
|
||||
assert code == 0
|
||||
assert dc.load_client(dest).project_id == "kuhy-syncs"
|
||||
|
||||
|
||||
def test_main_pickup_times_out(tmp_path: Path) -> None:
|
||||
code = dc.main(
|
||||
[
|
||||
"pickup",
|
||||
"--since",
|
||||
"100",
|
||||
"--timeout",
|
||||
"0",
|
||||
"--downloads",
|
||||
str(tmp_path),
|
||||
"--dest",
|
||||
str(tmp_path / "d.json"),
|
||||
]
|
||||
)
|
||||
assert code == 1
|
||||
62
crdt-sync/crdt_sync/tests/test_tool_e2e_audit.py
Normal file
62
crdt-sync/crdt_sync/tests/test_tool_e2e_audit.py
Normal file
@ -0,0 +1,62 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""The RTDB plaintext audit behind the E2E done check."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from crdt_sync import E2eCipher, seal_log_text
|
||||
from tool import e2e_audit
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import pytest
|
||||
|
||||
LOG = json.dumps(
|
||||
{
|
||||
"n": {
|
||||
"id": "n",
|
||||
"fields": {"title": ["x", "2026-01-01T00:00:00.000Z-0000-a"]},
|
||||
"deleted": False,
|
||||
"deleted_hlc": None,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _sealed() -> str:
|
||||
return seal_log_text(LOG, E2eCipher(bytes(32)), path="a/devices-e2e/p/log.json")
|
||||
|
||||
|
||||
def test_sealed_tree_is_clean() -> None:
|
||||
tree = {
|
||||
"devices-e2e": {"p": {"log~2Ejson": _sealed()}},
|
||||
"revs-e2e": {"p": "ab" * 32},
|
||||
"e2e": '{"v":1,"kdf":"argon2id"}',
|
||||
}
|
||||
assert e2e_audit.plaintext_paths(tree) == []
|
||||
|
||||
|
||||
def test_plaintext_leaf_is_reported_with_decoded_key() -> None:
|
||||
tree = {"devices": {"p": {"log~2Ejson": LOG}}, "settings": {"k": '{"a":1}'}}
|
||||
assert e2e_audit.plaintext_paths(tree) == [
|
||||
"devices/p/log.json",
|
||||
"settings/k",
|
||||
]
|
||||
|
||||
|
||||
def test_raw_json_scalars_count_as_plaintext() -> None:
|
||||
assert e2e_audit.plaintext_paths({"a": 5, "b": True, "c": None}) == ["a", "b"]
|
||||
|
||||
|
||||
def test_main_reads_stdin_and_sets_exit_code(
|
||||
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps({"x": LOG})))
|
||||
assert e2e_audit.main(["todo-sync"]) == 1
|
||||
assert "todo-sync/x" in capsys.readouterr().out
|
||||
monkeypatch.setattr("sys.stdin", io.StringIO("null"))
|
||||
assert e2e_audit.main(["todo-sync"]) == 0
|
||||
monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps({"x": _sealed()})))
|
||||
assert e2e_audit.main([]) == 0
|
||||
41
crdt-sync/crdt_sync/tests/test_tool_e2e_vectors.py
Normal file
41
crdt-sync/crdt_sync/tests/test_tool_e2e_vectors.py
Normal file
@ -0,0 +1,41 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""tool/e2e_vectors writes vectors that Python itself can open."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import runpy
|
||||
|
||||
import pytest
|
||||
|
||||
from crdt_sync import open_log_text, unlock_e2e
|
||||
from crdt_sync.tests.conftest import FakeStore
|
||||
from tool import e2e_vectors
|
||||
|
||||
|
||||
def test_vectors_open_with_the_stated_passphrase() -> None:
|
||||
vec = e2e_vectors.vectors()
|
||||
key = unlock_e2e(
|
||||
FakeStore({"vec/e2e": str(vec["header"])}),
|
||||
path_prefix="vec/notes-e2e",
|
||||
passphrase=str(vec["passphrase"]),
|
||||
e2e_path="vec/e2e",
|
||||
)
|
||||
assert key.master.hex() == vec["master_hex"]
|
||||
path = str(vec["path"])
|
||||
opened = open_log_text(str(vec["envelope"]), key.cipher(), path=path)
|
||||
assert json.loads(opened) == json.loads(str(vec["log"]))
|
||||
assert open_log_text(str(vec["whole"]), key.cipher(), path=path) == "not a log"
|
||||
|
||||
|
||||
def test_main_prints_json(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
assert e2e_vectors.main() == 0
|
||||
assert "master_hex" in json.loads(capsys.readouterr().out)
|
||||
|
||||
|
||||
def test_module_entry_point(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
runpy.run_path(str(Path(e2e_vectors.__file__)), run_name="__main__")
|
||||
assert exc.value.code == 0
|
||||
assert "envelope" in capsys.readouterr().out
|
||||
64
crdt-sync/crdt_sync/tests/test_tool_pkce.py
Normal file
64
crdt-sync/crdt_sync/tests/test_tool_pkce.py
Normal file
@ -0,0 +1,64 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Tests for ``tool/_pkce``: the verifier/challenge pair, consent URL, browser."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
import urllib.parse
|
||||
|
||||
from tool import _pkce
|
||||
|
||||
# The worked example from RFC 7636 appendix B.
|
||||
_RFC_VERIFIER = "dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk"
|
||||
_RFC_CHALLENGE = "E9Melhoa2OwvFrEMTJguCHaoeK1t8URWbuGJSstw-cM"
|
||||
|
||||
|
||||
def test_challenge_matches_the_rfc_vector() -> None:
|
||||
assert _pkce.challenge_for(_RFC_VERIFIER) == _RFC_CHALLENGE
|
||||
|
||||
|
||||
def test_make_pkce_is_random_and_self_consistent() -> None:
|
||||
first, second = _pkce.make_pkce(), _pkce.make_pkce()
|
||||
assert first.verifier != second.verifier
|
||||
assert 43 <= len(first.verifier) <= 128
|
||||
assert first.challenge == _pkce.challenge_for(first.verifier)
|
||||
|
||||
|
||||
def test_consent_url_carries_every_pkce_field() -> None:
|
||||
url = _pkce.build_consent_url(
|
||||
client_id="cid",
|
||||
redirect_uri="http://127.0.0.1:5555",
|
||||
challenge="chal",
|
||||
state="st",
|
||||
login_hint="me@example.com",
|
||||
)
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
query = {k: v[0] for k, v in urllib.parse.parse_qs(parsed.query).items()}
|
||||
assert parsed.netloc == "accounts.google.com"
|
||||
assert query == {
|
||||
"client_id": "cid",
|
||||
"redirect_uri": "http://127.0.0.1:5555",
|
||||
"response_type": "code",
|
||||
"scope": "openid email profile",
|
||||
"state": "st",
|
||||
"code_challenge": "chal",
|
||||
"code_challenge_method": "S256",
|
||||
"login_hint": "me@example.com",
|
||||
"prompt": "select_account",
|
||||
}
|
||||
|
||||
|
||||
def test_open_detached_spawns_a_new_session() -> None:
|
||||
popen = MagicMock()
|
||||
which = MagicMock(side_effect=lambda name: f"/usr/bin/{name}")
|
||||
assert _pkce.open_detached("https://x", popen=popen, which=which) is True
|
||||
args, kwargs = popen.call_args
|
||||
assert args[0] == ["/usr/bin/setsid", "/usr/bin/xdg-open", "https://x"]
|
||||
assert kwargs["start_new_session"] is True
|
||||
|
||||
|
||||
def test_open_detached_reports_missing_tools() -> None:
|
||||
popen = MagicMock()
|
||||
which = MagicMock(return_value=None)
|
||||
assert _pkce.open_detached("https://x", popen=popen, which=which) is False
|
||||
popen.assert_not_called()
|
||||
38
crdt-sync/crdt_sync/tests/test_tool_seeded_apps_todo_mcp.py
Normal file
38
crdt-sync/crdt_sync/tests/test_tool_seeded_apps_todo_mcp.py
Normal file
@ -0,0 +1,38 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""The todo MCP server's desktop session is seeded by ``seed_session``.
|
||||
|
||||
Split from :mod:`test_tool_seed_session_main` for the 250-line cap.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from crdt_sync._config import credential_store_for
|
||||
from crdt_sync._credentials import FirebaseCredentials
|
||||
from tool import seed_session as ss
|
||||
|
||||
_APP = "todo-mcp"
|
||||
|
||||
|
||||
def test_todo_mcp_is_a_default_app() -> None:
|
||||
"""``~/src/todo/mcp`` reads ``~/.config/todo-mcp/firebase_auth.json`` once
|
||||
at start; ``--app todo-mcp`` is rejected by argparse unless it is listed."""
|
||||
assert _APP in ss.DEFAULT_APPS
|
||||
|
||||
|
||||
def test_the_seed_lands_where_the_todo_mcp_server_reads_it(tmp_path: Path) -> None:
|
||||
"""The Dart server's ``seedCredentials`` path, in the usual shape."""
|
||||
with patch.object(Path, "home", return_value=tmp_path):
|
||||
credential_store_for(_APP).save(
|
||||
FirebaseCredentials("fake-id", "fake-refresh", datetime.now(UTC))
|
||||
)
|
||||
written = tmp_path / ".config" / _APP / "firebase_auth.json"
|
||||
assert set(json.loads(written.read_text())) == {
|
||||
"id_token",
|
||||
"refresh_token",
|
||||
"expires_at",
|
||||
}
|
||||
156
crdt-sync/crdt_sync/tests/test_tool_verify_desktop_client.py
Normal file
156
crdt-sync/crdt_sync/tests/test_tool_verify_desktop_client.py
Normal file
@ -0,0 +1,156 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Tests for ``tool/verify_desktop_client``: the PKCE loopback + Firebase check.
|
||||
|
||||
No browser, no network, no socket: the consent step, the code exchange and
|
||||
Firebase's REST endpoint are all faked at the module boundary.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import contextlib
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from crdt_sync._autherrors import FirebaseAuthError
|
||||
from crdt_sync._config import FirebaseConfig
|
||||
from tool import verify_desktop_client as vdc
|
||||
from tool._token_exchange import TokenError, _exchange_code
|
||||
from tool.desktop_client import DesktopClient
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
_HELD = "stored-client-credential"
|
||||
_CLIENT = DesktopClient("1-abc.apps.googleusercontent.com", _HELD, "kuhy-syncs")
|
||||
_CONFIG = FirebaseConfig("api", "https://db", "kuhy-syncs", "uid-1", "me@example.com")
|
||||
_ID_VALUE = (
|
||||
"header."
|
||||
+ base64.urlsafe_b64encode(json.dumps({"aud": _CLIENT.client_id}).encode())
|
||||
.decode()
|
||||
.rstrip("=")
|
||||
+ ".sig"
|
||||
)
|
||||
|
||||
|
||||
def test_audience_of_reads_the_aud_claim() -> None:
|
||||
assert vdc.audience_of(_ID_VALUE) == _CLIENT.client_id
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad", ["", "a.b", "a.!!!.c", "a." + base64.b64encode(b"[]").decode() + ".c"]
|
||||
)
|
||||
def test_audience_of_survives_garbage(bad: str) -> None:
|
||||
assert vdc.audience_of(bad) == "?"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("message", "needle"),
|
||||
[
|
||||
(
|
||||
"failed: HTTP 400 (INVALID_IDP_RESPONSE : id_token audience mismatch)",
|
||||
"Safelist",
|
||||
),
|
||||
("signed in as the wrong account: Google resolved to uid 'x'", "login_hint"),
|
||||
("anything else", ""),
|
||||
],
|
||||
)
|
||||
def test_hint_for(message: str, needle: str) -> None:
|
||||
hint = vdc.hint_for(message)
|
||||
assert (needle in hint) if needle else hint == ""
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _flow(result: dict[str, str]) -> Iterator[MagicMock]:
|
||||
"""Fake the socket, the callback and the exchange; yield the exchange."""
|
||||
with (
|
||||
patch.object(vdc, "_free_port", return_value=5555),
|
||||
patch.object(
|
||||
vdc, "_start_callback_server", return_value=(MagicMock(), MagicMock())
|
||||
),
|
||||
patch.object(vdc._CallbackHandler, "result", result),
|
||||
patch.object(vdc, "_exchange_code", return_value=_ID_VALUE) as exchange,
|
||||
patch.object(
|
||||
vdc.secrets,
|
||||
"token_urlsafe",
|
||||
side_effect=lambda nbytes: "st" if nbytes == 16 else "v" * 86,
|
||||
),
|
||||
):
|
||||
yield exchange
|
||||
|
||||
|
||||
def test_fetch_id_token_sends_pkce_and_the_secret() -> None:
|
||||
opener = MagicMock(return_value=True)
|
||||
with _flow({"code": "abc", "state": "st"}) as exchange:
|
||||
token = vdc.fetch_id_token(_CLIENT, "me@example.com", opener=opener)
|
||||
assert token == _ID_VALUE
|
||||
url = opener.call_args.args[0]
|
||||
assert "redirect_uri=http%3A%2F%2F127.0.0.1%3A5555" in url
|
||||
assert "code_challenge_method=S256" in url
|
||||
code, client_id, held, redirect, verifier = exchange.call_args.args
|
||||
assert (code, client_id, held) == ("abc", _CLIENT.client_id, _HELD)
|
||||
assert redirect == "http://127.0.0.1:5555"
|
||||
assert len(verifier) >= 43
|
||||
|
||||
|
||||
def test_fetch_id_token_without_a_browser_still_waits() -> None:
|
||||
opener = MagicMock(return_value=False)
|
||||
with _flow({"code": "abc", "state": "st"}):
|
||||
token = vdc.fetch_id_token(_CLIENT, "", open_browser=False, opener=opener)
|
||||
assert token == _ID_VALUE
|
||||
opener.assert_not_called()
|
||||
|
||||
|
||||
def test_fetch_id_token_reports_a_refused_consent() -> None:
|
||||
with (
|
||||
_flow({"error": "access_denied"}),
|
||||
pytest.raises(TokenError, match="access_denied"),
|
||||
):
|
||||
vdc.fetch_id_token(_CLIENT, "", opener=MagicMock())
|
||||
|
||||
|
||||
def _firebase_reply(*, ok: bool, body: dict[str, object]) -> MagicMock:
|
||||
response = MagicMock(ok=ok, status_code=200 if ok else 400)
|
||||
response.json.return_value = body
|
||||
return response
|
||||
|
||||
|
||||
def test_sign_in_returns_the_email_and_writes_no_file() -> None:
|
||||
reply = _firebase_reply(
|
||||
ok=True,
|
||||
body={
|
||||
"localId": "uid-1",
|
||||
"email": "me@example.com",
|
||||
"idToken": "i",
|
||||
"refreshToken": "r",
|
||||
"expiresIn": "3600",
|
||||
},
|
||||
)
|
||||
with patch("crdt_sync._firebase_auth.requests.post", return_value=reply) as post:
|
||||
assert vdc.sign_in(_CONFIG, _ID_VALUE) == "me@example.com"
|
||||
assert "signInWithIdp" in post.call_args.args[0]
|
||||
|
||||
|
||||
def test_sign_in_surfaces_the_exact_firebase_error() -> None:
|
||||
reply = _firebase_reply(
|
||||
ok=False,
|
||||
body={
|
||||
"error": {"message": "INVALID_IDP_RESPONSE : id_token audience mismatch"}
|
||||
},
|
||||
)
|
||||
with (
|
||||
patch("crdt_sync._firebase_auth.requests.post", return_value=reply),
|
||||
pytest.raises(FirebaseAuthError, match="audience mismatch"),
|
||||
):
|
||||
vdc.sign_in(_CONFIG, _ID_VALUE)
|
||||
|
||||
|
||||
def test_exchange_sends_the_pkce_verifier() -> None:
|
||||
reply = MagicMock(ok=True)
|
||||
reply.json.return_value = {"id_token": _ID_VALUE}
|
||||
with patch("tool._token_exchange.requests.post", return_value=reply) as post:
|
||||
_exchange_code("c", "id", _HELD, "http://127.0.0.1:1", "ver")
|
||||
assert post.call_args.kwargs["data"]["code_verifier"] == "ver"
|
||||
@ -0,0 +1,125 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Tests for ``verify_desktop_client.main``: every exit path and its message."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from crdt_sync._autherrors import FirebaseAuthError
|
||||
from crdt_sync._config import ConfigError, FirebaseConfig
|
||||
from tool import verify_desktop_client as vdc
|
||||
from tool._token_exchange import TokenError
|
||||
from tool.client_probe import ProbeResult
|
||||
from tool.desktop_client import ClientFileError, DesktopClient
|
||||
|
||||
_HELD = "stored-client-credential"
|
||||
_CLIENT = DesktopClient("1-abc.apps.googleusercontent.com", _HELD, "kuhy-syncs")
|
||||
_CONFIG = FirebaseConfig("api", "https://db", "kuhy-syncs", "uid-1", "me@example.com")
|
||||
_ID_VALUE = (
|
||||
"header."
|
||||
+ base64.urlsafe_b64encode(json.dumps({"aud": _CLIENT.client_id}).encode())
|
||||
.decode()
|
||||
.rstrip("=")
|
||||
+ ".sig"
|
||||
)
|
||||
|
||||
|
||||
def _run(
|
||||
*,
|
||||
probe_ok: bool = True,
|
||||
token: object = _ID_VALUE,
|
||||
sign_in: object = "me@example.com",
|
||||
argv: list[str] | None = None,
|
||||
) -> tuple[int, MagicMock]:
|
||||
fetch = MagicMock(
|
||||
side_effect=token if isinstance(token, Exception) else None, return_value=token
|
||||
)
|
||||
signer = MagicMock(
|
||||
side_effect=sign_in if isinstance(sign_in, Exception) else None,
|
||||
return_value=sign_in,
|
||||
)
|
||||
with (
|
||||
patch.object(vdc.FirebaseConfig, "load", return_value=_CONFIG),
|
||||
patch.object(vdc, "load_client", return_value=_CLIENT),
|
||||
patch.object(
|
||||
vdc, "probe_client", return_value=[ProbeResult(probe_ok, "probed")]
|
||||
),
|
||||
patch.object(vdc, "fetch_id_token", fetch),
|
||||
patch.object(vdc, "sign_in", signer),
|
||||
):
|
||||
return vdc.main(argv or []), fetch
|
||||
|
||||
|
||||
def test_main_passes_and_names_the_uid(caplog: pytest.LogCaptureFixture) -> None:
|
||||
caplog.set_level(logging.INFO)
|
||||
code, _ = _run()
|
||||
assert code == 0
|
||||
assert "PASS" in caplog.text
|
||||
assert "uid-1" in caplog.text
|
||||
assert "me@example.com" in caplog.text
|
||||
|
||||
|
||||
def test_main_stops_before_consent_when_the_probe_fails(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
caplog.set_level(logging.INFO)
|
||||
code, fetch = _run(probe_ok=False)
|
||||
assert code == 1
|
||||
assert "FAIL" in caplog.text
|
||||
fetch.assert_not_called()
|
||||
|
||||
|
||||
def test_main_probe_only_never_opens_consent() -> None:
|
||||
code, fetch = _run(argv=["--probe-only"])
|
||||
assert code == 0
|
||||
fetch.assert_not_called()
|
||||
|
||||
|
||||
def test_main_skip_probe_goes_straight_to_consent() -> None:
|
||||
code, fetch = _run(probe_ok=False, argv=["--skip-probe", "--no-browser"])
|
||||
assert code == 0
|
||||
assert fetch.call_args.kwargs["open_browser"] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kwargs", "needle"),
|
||||
[
|
||||
(
|
||||
{"token": TokenError("token exchange failed: HTTP 400 invalid_grant")},
|
||||
"invalid_grant",
|
||||
),
|
||||
(
|
||||
{
|
||||
"sign_in": FirebaseAuthError(
|
||||
"INVALID_IDP_RESPONSE : id_token audience mismatch"
|
||||
)
|
||||
},
|
||||
"Safelist",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_main_prints_the_exact_error_and_exits_1(
|
||||
kwargs: dict[str, object], needle: str, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
caplog.set_level(logging.INFO)
|
||||
code, _ = _run(**kwargs)
|
||||
assert code == 1
|
||||
assert "FAIL" in caplog.text
|
||||
assert needle in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error", [ConfigError("no config"), ClientFileError("no client")]
|
||||
)
|
||||
def test_main_reports_setup_errors(
|
||||
error: Exception, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
caplog.set_level(logging.INFO)
|
||||
with patch.object(vdc.FirebaseConfig, "load", side_effect=error):
|
||||
assert vdc.main([]) == 1
|
||||
assert "FAIL" in caplog.text
|
||||
@ -1,9 +1,13 @@
|
||||
[project]
|
||||
name = "crdt-sync"
|
||||
version = "0.9.2"
|
||||
version = "0.11.1"
|
||||
description = "Shared CRDT merge scheme + GitHub-Contents-API sync transport for cross-device apps"
|
||||
requires-python = ">=3.14"
|
||||
dependencies = ["requests==2.34.2"]
|
||||
dependencies = [
|
||||
"argon2-cffi==25.1.0",
|
||||
"pynacl==1.6.2",
|
||||
"requests==2.34.2",
|
||||
]
|
||||
|
||||
# Ship only the library. `tool/` is a package too (it has an __init__.py so
|
||||
# its scripts import cleanly), and setuptools' flat-layout auto-discovery
|
||||
|
||||
@ -1,12 +1,14 @@
|
||||
# Exact pins, enforced by the shared dependency-freshness gate: a `>=`
|
||||
# floor is satisfied by anything newer, so it tells a checker nothing
|
||||
# about whether this repo is actually current.
|
||||
argon2-cffi==25.1.0
|
||||
bandit[toml]==1.9.4
|
||||
mypy==2.4.0
|
||||
mypy==2.4.1
|
||||
pre-commit==4.7.0
|
||||
pylint==4.1.3
|
||||
pytest==9.1.1
|
||||
pynacl==1.6.2
|
||||
pytest==9.2.0
|
||||
pytest-cov==7.1.0
|
||||
requests==2.34.2
|
||||
ruff==0.17.2
|
||||
types-requests==2.33.0.20261006
|
||||
ruff==0.17.4
|
||||
types-requests==2.33.0.20261030
|
||||
|
||||
85
crdt-sync/tool/_pkce.py
Normal file
85
crdt-sync/tool/_pkce.py
Normal file
@ -0,0 +1,85 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""PKCE pair, consent URL and detached browser launch for the loopback flow."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import secrets
|
||||
import shutil
|
||||
import subprocess
|
||||
from typing import TYPE_CHECKING
|
||||
import urllib.parse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
AUTH_ENDPOINT = "https://accounts.google.com/o/oauth2/v2/auth"
|
||||
SCOPES = "openid email profile"
|
||||
_VERIFIER_BYTES = 64
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Pkce:
|
||||
"""A code verifier and the S256 challenge derived from it."""
|
||||
|
||||
verifier: str
|
||||
challenge: str
|
||||
|
||||
|
||||
def challenge_for(verifier: str) -> str:
|
||||
"""Return the unpadded base64url SHA-256 of ``verifier`` (RFC 7636 S256)."""
|
||||
digest = hashlib.sha256(verifier.encode("ascii")).digest()
|
||||
return base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def make_pkce() -> Pkce:
|
||||
"""Return a fresh 86-char verifier and its challenge."""
|
||||
verifier = secrets.token_urlsafe(_VERIFIER_BYTES)
|
||||
return Pkce(verifier=verifier, challenge=challenge_for(verifier))
|
||||
|
||||
|
||||
def build_consent_url(
|
||||
*,
|
||||
client_id: str,
|
||||
redirect_uri: str,
|
||||
challenge: str,
|
||||
state: str,
|
||||
login_hint: str,
|
||||
) -> str:
|
||||
"""Return the consent URL, with the sync account preselected."""
|
||||
query = urllib.parse.urlencode(
|
||||
{
|
||||
"client_id": client_id,
|
||||
"redirect_uri": redirect_uri,
|
||||
"response_type": "code",
|
||||
"scope": SCOPES,
|
||||
"state": state,
|
||||
"code_challenge": challenge,
|
||||
"code_challenge_method": "S256",
|
||||
"login_hint": login_hint,
|
||||
"prompt": "select_account",
|
||||
}
|
||||
)
|
||||
return f"{AUTH_ENDPOINT}?{query}"
|
||||
|
||||
|
||||
def open_detached(
|
||||
url: str,
|
||||
*,
|
||||
popen: Callable[..., object] = subprocess.Popen,
|
||||
which: Callable[[str], str | None] = shutil.which,
|
||||
) -> bool:
|
||||
"""Open ``url`` via ``setsid xdg-open``; False if either tool is missing."""
|
||||
setsid, opener = which("setsid"), which("xdg-open")
|
||||
if not setsid or not opener:
|
||||
return False
|
||||
popen(
|
||||
[setsid, opener, url],
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
start_new_session=True,
|
||||
)
|
||||
return True
|
||||
@ -67,4 +67,8 @@ DEFAULT_APPS: Final = (
|
||||
# /kartka/source_health via credential_store_for("kartka-source-eval")
|
||||
# and falls back to the retired password only with no session here.
|
||||
"kartka-source-eval",
|
||||
# ~/src/todo/mcp: the Dart todo MCP server (systemd --user, :8735) is one
|
||||
# more sync node with its own session, read once from
|
||||
# ~/.config/todo-mcp/firebase_auth.json at start.
|
||||
"todo-mcp",
|
||||
)
|
||||
|
||||
@ -44,9 +44,17 @@ def _authorization_code(params: dict[str, str], redirect_uri: str, state: str) -
|
||||
|
||||
|
||||
def _exchange_code(
|
||||
code: str, client_id: str, client_secret: str, redirect_uri: str
|
||||
code: str,
|
||||
client_id: str,
|
||||
client_secret: str,
|
||||
redirect_uri: str,
|
||||
code_verifier: str = "",
|
||||
) -> str:
|
||||
"""Trade the authorization code for an ID token at Google's endpoint."""
|
||||
"""Trade the authorization code for an ID token at Google's endpoint.
|
||||
|
||||
``code_verifier`` is sent only when given, for PKCE flows.
|
||||
"""
|
||||
pkce = {"code_verifier": code_verifier} if code_verifier else {}
|
||||
try:
|
||||
response = requests.post(
|
||||
_TOKEN_EXCHANGE_URL,
|
||||
@ -56,6 +64,7 @@ def _exchange_code(
|
||||
"client_secret": client_secret,
|
||||
"redirect_uri": redirect_uri,
|
||||
"grant_type": "authorization_code",
|
||||
**pkce,
|
||||
},
|
||||
timeout=_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
158
crdt-sync/tool/client_probe.py
Normal file
158
crdt-sync/tool/client_probe.py
Normal file
@ -0,0 +1,158 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Click-free checks that a freshly stored Desktop OAuth client actually works.
|
||||
|
||||
Both requests need no signed-in user: Google's endpoints answer them before any
|
||||
account is involved, so they say whether the client exists, accepts a loopback
|
||||
redirect and whether the stored secret is right -- without opening a browser.
|
||||
That separates "the console step went wrong" from "Firebase rejects the
|
||||
audience" before the one human consent click is spent.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from dataclasses import dataclass
|
||||
import re
|
||||
from typing import TYPE_CHECKING
|
||||
import urllib.parse
|
||||
|
||||
import requests
|
||||
|
||||
from tool._pkce import AUTH_ENDPOINT, challenge_for
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from tool.desktop_client import DesktopClient
|
||||
|
||||
# Assembled from host and path: a literal ending in "/token" trips ruff's
|
||||
# hardcoded-password check, and suppressions are banned in this repo.
|
||||
_OAUTH_HOST = "https://oauth2.googleapis.com"
|
||||
_TOKEN_URL = f"{_OAUTH_HOST}/token"
|
||||
_PROBE_REDIRECT = "http://127.0.0.1:41234"
|
||||
_TIMEOUT_SECONDS = 20
|
||||
_ASCII_RUN = re.compile(rb"[ -~]{6,}")
|
||||
_PROBE_VERIFIER = "probe-verifier-" + "x" * 30
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProbeResult:
|
||||
"""One check's verdict and the words that explain it."""
|
||||
|
||||
ok: bool
|
||||
detail: str
|
||||
|
||||
|
||||
def classify_auth_location(location: str | None) -> ProbeResult:
|
||||
"""Judge the auth endpoint's redirect target.
|
||||
|
||||
A valid request while signed out still redirects (to the sign-in page), so
|
||||
the status code means nothing: only an ``authError`` parameter in the
|
||||
target says Google refused the client or its redirect URI.
|
||||
"""
|
||||
if not location or "authError=" not in location:
|
||||
return ProbeResult(
|
||||
ok=True, detail="auth endpoint accepts the loopback redirect"
|
||||
)
|
||||
raw = location.split("authError=", 1)[1].split("&", 1)[0]
|
||||
try:
|
||||
decoded = base64.b64decode(
|
||||
raw + "=" * (-len(raw) % 4), altchars=b"-_", validate=True
|
||||
)
|
||||
except ValueError:
|
||||
return ProbeResult(
|
||||
ok=False, detail=f"auth endpoint refused the request (authError={raw})"
|
||||
)
|
||||
text = " | ".join(run.decode("ascii") for run in _ASCII_RUN.findall(decoded))
|
||||
return ProbeResult(
|
||||
ok=False, detail=f"auth endpoint refused the request: {text[:300]}"
|
||||
)
|
||||
|
||||
|
||||
def classify_token(status: int, body: dict[str, str] | None) -> ProbeResult:
|
||||
"""Judge the token endpoint's answer to a deliberately bogus code.
|
||||
|
||||
``invalid_grant`` is the success signal: Google checked the client and
|
||||
secret first, accepted them, and only then rejected the fake code.
|
||||
"""
|
||||
if body is None:
|
||||
return ProbeResult(ok=False, detail=f"token endpoint: HTTP {status}, no JSON")
|
||||
error = body.get("error", "")
|
||||
description = body.get("error_description", "")
|
||||
if error == "invalid_grant":
|
||||
return ProbeResult(
|
||||
ok=True, detail="token endpoint accepts the client and secret"
|
||||
)
|
||||
if error == "invalid_client" and "not found" in description:
|
||||
return ProbeResult(
|
||||
ok=False,
|
||||
detail=(
|
||||
"client not found yet: new clients take a few minutes to "
|
||||
"propagate; rerun with --verify-only shortly"
|
||||
),
|
||||
)
|
||||
if error == "invalid_client":
|
||||
return ProbeResult(
|
||||
ok=False, detail=f"the stored secret is wrong ({description})"
|
||||
)
|
||||
return ProbeResult(ok=False, detail=f"token endpoint: HTTP {status} {body}")
|
||||
|
||||
|
||||
def _probe_auth(
|
||||
client: DesktopClient, get: Callable[..., requests.Response]
|
||||
) -> ProbeResult:
|
||||
query = urllib.parse.urlencode(
|
||||
{
|
||||
"client_id": client.client_id,
|
||||
"redirect_uri": _PROBE_REDIRECT,
|
||||
"response_type": "code",
|
||||
"scope": "openid email",
|
||||
"code_challenge": challenge_for(_PROBE_VERIFIER),
|
||||
"code_challenge_method": "S256",
|
||||
"hl": "en",
|
||||
}
|
||||
)
|
||||
try:
|
||||
reply = get(
|
||||
f"{AUTH_ENDPOINT}?{query}",
|
||||
allow_redirects=False,
|
||||
timeout=_TIMEOUT_SECONDS,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
return ProbeResult(ok=False, detail=f"auth endpoint unreachable: {exc}")
|
||||
return classify_auth_location(reply.headers.get("Location"))
|
||||
|
||||
|
||||
def _probe_token(
|
||||
client: DesktopClient, post: Callable[..., requests.Response]
|
||||
) -> ProbeResult:
|
||||
try:
|
||||
answer = post(
|
||||
_TOKEN_URL,
|
||||
data={
|
||||
"code": "probe-not-a-real-code",
|
||||
"client_id": client.client_id,
|
||||
"client_secret": client.client_secret,
|
||||
"redirect_uri": _PROBE_REDIRECT,
|
||||
"grant_type": "authorization_code",
|
||||
"code_verifier": _PROBE_VERIFIER,
|
||||
},
|
||||
timeout=_TIMEOUT_SECONDS,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
return ProbeResult(ok=False, detail=f"token endpoint unreachable: {exc}")
|
||||
try:
|
||||
body = answer.json()
|
||||
except ValueError:
|
||||
body = None
|
||||
return classify_token(answer.status_code, body)
|
||||
|
||||
|
||||
def probe_client(
|
||||
client: DesktopClient,
|
||||
*,
|
||||
get: Callable[..., requests.Response] = requests.get,
|
||||
post: Callable[..., requests.Response] = requests.post,
|
||||
) -> list[ProbeResult]:
|
||||
"""Run the auth-endpoint and token-endpoint checks for ``client``."""
|
||||
return [_probe_auth(client, get), _probe_token(client, post)]
|
||||
173
crdt-sync/tool/desktop_client.py
Normal file
173
crdt-sync/tool/desktop_client.py
Normal file
@ -0,0 +1,173 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Pick up, validate and store the Desktop OAuth client JSON from the console.
|
||||
|
||||
Android and iOS clients download as the same top-level ``installed`` object, so
|
||||
the type alone proves nothing: only a ``client_secret`` plus the
|
||||
``http://localhost`` redirect marks a Desktop client.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
PROJECT_ID = "kuhy-syncs"
|
||||
DEFAULT_DEST = Path.home() / ".config" / "crdt-sync" / "desktop_client.json"
|
||||
DEFAULT_DOWNLOADS = Path.home() / "Downloads"
|
||||
_POLL_SECONDS = 2.0
|
||||
_pause_between_polls = partial(time.sleep, _POLL_SECONDS)
|
||||
_CLIENT_ID_SUFFIX = ".apps.googleusercontent.com"
|
||||
_LOOPBACK_REDIRECT = "http://localhost"
|
||||
|
||||
_logger = logging.getLogger("desktop_client")
|
||||
|
||||
|
||||
class ClientFileError(Exception):
|
||||
"""The client JSON is missing, malformed or not a Desktop client."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DesktopClient:
|
||||
"""The installed-app client as downloaded from the console."""
|
||||
|
||||
client_id: str
|
||||
client_secret: str
|
||||
project_id: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Pickup:
|
||||
"""A valid client JSON found in the downloads folder."""
|
||||
|
||||
path: Path
|
||||
client: DesktopClient
|
||||
|
||||
|
||||
def parse_client_json(text: str, project_id: str = PROJECT_ID) -> DesktopClient:
|
||||
"""Return the Desktop client in ``text``, or raise saying what is wrong."""
|
||||
try:
|
||||
document = json.loads(text)
|
||||
except ValueError as exc:
|
||||
msg = f"not valid JSON: {exc}"
|
||||
raise ClientFileError(msg) from exc
|
||||
if not isinstance(document, dict):
|
||||
msg = "must contain a JSON object"
|
||||
raise ClientFileError(msg)
|
||||
installed = document.get("installed")
|
||||
if not isinstance(installed, dict):
|
||||
msg = f"no 'installed' object (found {sorted(document)}); pick 'Desktop app'"
|
||||
raise ClientFileError(msg)
|
||||
if installed.get("project_id") != project_id:
|
||||
msg = f"belongs to project {installed.get('project_id')!r}, not {project_id!r}"
|
||||
raise ClientFileError(msg)
|
||||
client_id = str(installed.get("client_id", ""))
|
||||
if not client_id.endswith(_CLIENT_ID_SUFFIX):
|
||||
msg = f"client_id {client_id!r} is not a Google OAuth client id"
|
||||
raise ClientFileError(msg)
|
||||
held = str(installed.get("client_secret", ""))
|
||||
if not held:
|
||||
msg = "no client_secret: this is an Android/iOS client, not a Desktop app"
|
||||
raise ClientFileError(msg)
|
||||
if _LOOPBACK_REDIRECT not in installed.get("redirect_uris", []):
|
||||
msg = f"redirect_uris lacks {_LOOPBACK_REDIRECT}: not a Desktop app client"
|
||||
raise ClientFileError(msg)
|
||||
return DesktopClient(client_id=client_id, client_secret=held, project_id=project_id)
|
||||
|
||||
|
||||
def _scan(
|
||||
downloads: Path, since: float, rejected: set[str], report: Callable[[str], None]
|
||||
) -> Pickup | None:
|
||||
candidates = [
|
||||
path
|
||||
for path in downloads.glob("client_secret_*.json")
|
||||
if path.stat().st_mtime >= since
|
||||
]
|
||||
for path in sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True):
|
||||
try:
|
||||
client = parse_client_json(path.read_text(encoding="utf-8"))
|
||||
except ClientFileError as exc:
|
||||
if path.name not in rejected:
|
||||
rejected.add(path.name)
|
||||
report(f"ignoring {path.name}: {exc}")
|
||||
continue
|
||||
return Pickup(path=path, client=client)
|
||||
return None
|
||||
|
||||
|
||||
def wait_for_client_json(
|
||||
downloads: Path,
|
||||
since: float,
|
||||
*,
|
||||
attempts: int,
|
||||
pause: Callable[[], None] = _pause_between_polls,
|
||||
report: Callable[[str], None] = print,
|
||||
) -> Pickup | None:
|
||||
"""Scan ``downloads`` up to ``attempts`` times for a JSON newer than ``since``."""
|
||||
rejected: set[str] = set()
|
||||
for attempt in range(attempts):
|
||||
found = _scan(downloads, since, rejected, report)
|
||||
if found:
|
||||
return found
|
||||
if attempt < attempts - 1:
|
||||
pause()
|
||||
return None
|
||||
|
||||
|
||||
def store_client(source: Path, dest: Path) -> None:
|
||||
"""Move ``source`` to ``dest`` with mode 0600, so no copy lingers in Downloads."""
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor = os.open(dest, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
||||
handle.write(source.read_text(encoding="utf-8"))
|
||||
dest.chmod(0o600)
|
||||
source.unlink()
|
||||
|
||||
|
||||
def load_client(dest: Path = DEFAULT_DEST) -> DesktopClient:
|
||||
"""Load the stored client, pointing at the register script when absent."""
|
||||
try:
|
||||
return parse_client_json(dest.read_text(encoding="utf-8"))
|
||||
except OSError as exc:
|
||||
msg = f"{dest} unreadable ({exc}); run tool/register_desktop_client.sh"
|
||||
raise ClientFileError(msg) from exc
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""Wait for the downloaded JSON and store it. Returns an exit code."""
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("action", choices=["pickup"])
|
||||
parser.add_argument("--since", type=float, required=True, help="epoch seconds")
|
||||
parser.add_argument("--timeout", type=float, default=600.0)
|
||||
parser.add_argument("--downloads", type=Path, default=DEFAULT_DOWNLOADS)
|
||||
parser.add_argument("--dest", type=Path, default=DEFAULT_DEST)
|
||||
args = parser.parse_args(argv)
|
||||
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s")
|
||||
found = wait_for_client_json(
|
||||
args.downloads,
|
||||
args.since,
|
||||
attempts=int(args.timeout / _POLL_SECONDS) + 1,
|
||||
report=_logger.info,
|
||||
)
|
||||
if found is None:
|
||||
_logger.error(" FAIL no Desktop client JSON appeared in %s", args.downloads)
|
||||
return 1
|
||||
store_client(found.path, args.dest)
|
||||
_logger.info(
|
||||
" PASS stored %s at %s (mode 600)", found.client.client_id, args.dest
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
63
crdt-sync/tool/e2e_audit.py
Normal file
63
crdt-sync/tool/e2e_audit.py
Normal file
@ -0,0 +1,63 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Read-only E2E done check: list plaintext leaves in an RTDB JSON dump.
|
||||
|
||||
Feed it ``firebase database:get /<root>`` on stdin::
|
||||
|
||||
firebase database:get /todo-sync | python3 -m tool.e2e_audit todo-sync
|
||||
|
||||
Exit 0 when every leaf is sealed (or is the clear Argon2id header or a keyed
|
||||
revision), 1 when any leaf is plaintext. A plain ``grep '"title"'`` misses
|
||||
plaintext because RTDB serves each blob as an escaped JSON string; this
|
||||
decodes the tree instead.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
from crdt_sync import is_e2e_envelope
|
||||
|
||||
_HEX64 = re.compile(r"[0-9a-f]{64}")
|
||||
|
||||
|
||||
def _decode_key(key: str) -> str:
|
||||
return re.sub(r"~([0-9A-F]{2})", lambda m: chr(int(m.group(1), 16)), key)
|
||||
|
||||
|
||||
def _is_clear_leaf(value: str) -> bool:
|
||||
# Envelopes and the Argon2id header both carry an integer "v"; keyed
|
||||
# revisions are 64 hex chars; a bare E2eCipher blob is base64, not JSON.
|
||||
if is_e2e_envelope(value) or _HEX64.fullmatch(value):
|
||||
return True
|
||||
return not value.lstrip().startswith(("{", "["))
|
||||
|
||||
|
||||
def plaintext_paths(tree: object, prefix: str = "") -> list[str]:
|
||||
"""Return the decoded paths of every plaintext leaf under ``tree``."""
|
||||
found: list[str] = []
|
||||
if isinstance(tree, dict):
|
||||
for key, value in tree.items():
|
||||
path = f"{prefix}/{_decode_key(key)}" if prefix else _decode_key(key)
|
||||
found.extend(plaintext_paths(value, path))
|
||||
elif isinstance(tree, str):
|
||||
if not _is_clear_leaf(tree):
|
||||
found.append(prefix)
|
||||
elif tree is not None:
|
||||
found.append(prefix)
|
||||
return found
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
"""Audit the dump on stdin; ``argv[0]`` names the root for the report."""
|
||||
root = argv[0] if argv else ""
|
||||
paths = plaintext_paths(json.load(sys.stdin), root)
|
||||
sys.stdout.write(f"plaintext leaves under /{root}: {len(paths)}\n")
|
||||
for path in paths:
|
||||
sys.stdout.write(f" {path}\n")
|
||||
return 1 if paths else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
58
crdt-sync/tool/e2e_cleanup.py
Normal file
58
crdt-sync/tool/e2e_cleanup.py
Normal file
@ -0,0 +1,58 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Clear an app's old plaintext RTDB prefix after the E2E copy is verified.
|
||||
|
||||
Dry run by default; ``--apply`` deletes. Run only once every device runs an
|
||||
encrypting build::
|
||||
|
||||
python3 -m tool.e2e_cleanup todo-sync/notes --app todo
|
||||
python3 -m tool.e2e_cleanup todo-sync/notes --app todo --apply
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from crdt_sync import (
|
||||
PASSWORD_FILE,
|
||||
E2eLockedError,
|
||||
FirebaseConfig,
|
||||
clear_plaintext_prefix,
|
||||
firebase_client_for,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from crdt_sync import RemoteStore
|
||||
|
||||
|
||||
def run(store: RemoteStore, old_prefix: str, passphrase: str, *, apply: bool) -> int:
|
||||
"""Clear (or list) ``old_prefix``; return a process exit code."""
|
||||
try:
|
||||
paths = clear_plaintext_prefix(
|
||||
store, old_prefix, passphrase=passphrase, apply=apply
|
||||
)
|
||||
except E2eLockedError as exc:
|
||||
sys.stdout.write(f"refused: {exc}\n")
|
||||
return 1
|
||||
verb = "deleted" if apply else "would delete"
|
||||
sys.stdout.write(f"{verb} {len(paths)} plaintext blob(s) under /{old_prefix}\n")
|
||||
for path in paths:
|
||||
sys.stdout.write(f" {path}\n")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int: # pragma: no cover - live database entry
|
||||
"""Parse args and run against the live database."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("old_prefix")
|
||||
parser.add_argument("--app", required=True, help="seeded session name")
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
client = firebase_client_for(args.app, config=FirebaseConfig.load())
|
||||
passphrase = PASSWORD_FILE.read_text(encoding="utf-8").strip()
|
||||
return run(client, args.old_prefix, passphrase, apply=args.apply)
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
64
crdt-sync/tool/e2e_vectors.py
Normal file
64
crdt-sync/tool/e2e_vectors.py
Normal file
@ -0,0 +1,64 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Write golden vectors sealed by Python for the Dart side to open.
|
||||
|
||||
Run from ``crdt-sync``::
|
||||
|
||||
python -m tool.e2e_vectors > e2e_py_vectors.json
|
||||
|
||||
then move the file to ``crdt_sync_dart/test/fixtures/``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
from crdt_sync import Argon2Params, seal_log_text, unlock_e2e
|
||||
from crdt_sync.tests.conftest import FakeStore
|
||||
|
||||
# A public test-vector passphrase, not a secret.
|
||||
PHRASE = "correct horse \u2713"
|
||||
|
||||
|
||||
def vectors() -> dict[str, object]:
|
||||
"""Return a fresh vector set (random salt and nonces)."""
|
||||
remote = FakeStore()
|
||||
key = unlock_e2e(
|
||||
remote,
|
||||
path_prefix="vec/notes-e2e",
|
||||
passphrase=PHRASE,
|
||||
params=Argon2Params(64, 2, 1),
|
||||
e2e_path="vec/e2e",
|
||||
)
|
||||
path = "vec/notes-e2e/pc/log.json"
|
||||
log = json.dumps(
|
||||
{
|
||||
"n9": {
|
||||
"id": "n9",
|
||||
"fields": {"title": ["zażółć", "2026-01-01T00:00:00.000Z-0000-pc"]},
|
||||
"deleted": False,
|
||||
"deleted_hlc": None,
|
||||
}
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
cipher = key.cipher()
|
||||
return {
|
||||
"passphrase": PHRASE,
|
||||
"header": remote.files["vec/e2e"],
|
||||
"master_hex": key.master.hex(),
|
||||
"path": path,
|
||||
"log": log,
|
||||
"envelope": seal_log_text(log, cipher, path=path),
|
||||
"whole": seal_log_text("not a log", cipher, path=path),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Print the vectors as JSON."""
|
||||
sys.stdout.write(json.dumps(vectors(), indent=2, ensure_ascii=False) + "\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
127
crdt-sync/tool/register_desktop_client.sh
Executable file
127
crdt-sync/tool/register_desktop_client.sh
Executable file
@ -0,0 +1,127 @@
|
||||
#!/bin/bash
|
||||
# Registers the Desktop OAuth client (no API exists, so one console step stays
|
||||
# manual), stores its JSON privately, then proves Firebase accepts its tokens.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
readonly SCRIPT_DIR REPO_DIR
|
||||
readonly PROJECT="kuhy-syncs"
|
||||
readonly ACCOUNT="321krzychu@gmail.com"
|
||||
readonly CONSOLE_URL="https://console.cloud.google.com/auth/clients?project=${PROJECT}&authuser=${ACCOUNT}"
|
||||
readonly APP_TYPE="Desktop app"
|
||||
readonly CLIENT_NAME="crdt-sync desktop/android PKCE"
|
||||
readonly PAUSE_SECONDS="${PAUSE_SECONDS:-12}"
|
||||
readonly WAIT_SECONDS="${WAIT_SECONDS:-900}"
|
||||
readonly PROBE_TRIES="${PROBE_TRIES:-12}"
|
||||
readonly PROBE_GAP_SECONDS="${PROBE_GAP_SECONDS:-30}"
|
||||
|
||||
log() { printf '\n\033[1m==> %s\033[0m\n' "$1"; }
|
||||
step() { printf ' %s\n' "$1"; }
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $(basename "$0") [--verify-only | --dry-run | --help]
|
||||
|
||||
(no flag) open the console, clipboard-feed the form, pick up the
|
||||
downloaded JSON, store it, then verify
|
||||
--verify-only skip registration; probe, consent flow, signInWithIdp
|
||||
--dry-run print the plan; open, copy and write nothing
|
||||
|
||||
Your part: Application type = Desktop app, paste the name, CREATE, DOWNLOAD
|
||||
JSON in the dialog; later tap Allow on the consent page.
|
||||
Stored at ~/.config/crdt-sync/desktop_client.json (mode 600).
|
||||
EOF
|
||||
}
|
||||
|
||||
# Prints the value always, so this works over ssh with no X display too.
|
||||
clip() {
|
||||
if command -v xclip >/dev/null 2>&1 && [[ -n "${DISPLAY:-}" ]]; then
|
||||
printf '%s' "$1" | xclip -selection clipboard 2>/dev/null || true
|
||||
printf ' \033[32m[copied]\033[0m %s\n' "$1"
|
||||
else
|
||||
printf ' %s\n' "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
# Without a terminal (e.g. run via `!`) read would hit EOF at once.
|
||||
pause() {
|
||||
if [[ -t 0 ]]; then
|
||||
read -rp " ...press Enter once pasted " _ || true
|
||||
else
|
||||
sleep "$PAUSE_SECONDS"
|
||||
fi
|
||||
}
|
||||
|
||||
open_console() {
|
||||
if command -v xdg-open >/dev/null 2>&1; then
|
||||
setsid xdg-open "$CONSOLE_URL" >/dev/null 2>&1 &
|
||||
else
|
||||
step "open: $CONSOLE_URL"
|
||||
fi
|
||||
}
|
||||
|
||||
run_tool() { (cd "$REPO_DIR" && python3 -m "tool.$1" "${@:2}"); }
|
||||
|
||||
wait_for_propagation() {
|
||||
local try
|
||||
for ((try = 1; try <= PROBE_TRIES; try++)); do
|
||||
if run_tool verify_desktop_client --probe-only; then
|
||||
return 0
|
||||
fi
|
||||
step "probe $try/$PROBE_TRIES failed; new clients need a few minutes"
|
||||
sleep "$PROBE_GAP_SECONDS"
|
||||
done
|
||||
echo "error: client still not accepted; rerun with --verify-only later" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
verify() {
|
||||
log "Verifying: probe, consent flow, Firebase signInWithIdp"
|
||||
step "Tap Allow on the sync account's consent page when it opens."
|
||||
run_tool verify_desktop_client
|
||||
}
|
||||
|
||||
register() {
|
||||
local started
|
||||
started="$(date +%s)"
|
||||
log "Create ONE Desktop OAuth client in project $PROJECT"
|
||||
open_console
|
||||
sleep 2
|
||||
step "Click '+ CREATE CLIENT'. Application type (paste or pick):"
|
||||
clip "$APP_TYPE"
|
||||
pause
|
||||
step "Field 'Name':"
|
||||
clip "$CLIENT_NAME"
|
||||
pause
|
||||
step "Click CREATE, then DOWNLOAD JSON in the dialog (the secret shows once)."
|
||||
step "Leave the existing Web and Android clients alone."
|
||||
log "Waiting up to ${WAIT_SECONDS}s for the downloaded client_secret_*.json"
|
||||
run_tool desktop_client pickup --since "$started" --timeout "$WAIT_SECONDS"
|
||||
wait_for_propagation
|
||||
verify
|
||||
}
|
||||
|
||||
dry_run() {
|
||||
log "Dry run: nothing is opened, copied or written"
|
||||
step "page: $CONSOLE_URL"
|
||||
step "type: $APP_TYPE"
|
||||
step "name: $CLIENT_NAME"
|
||||
step "stores at: ~/.config/crdt-sync/desktop_client.json (mode 600)"
|
||||
step "verify: python3 -m tool.verify_desktop_client (in $REPO_DIR)"
|
||||
}
|
||||
|
||||
main() {
|
||||
case "${1:-}" in
|
||||
--verify-only) verify ;;
|
||||
--dry-run) dry_run ;;
|
||||
-h | --help) usage ;;
|
||||
"") register ;;
|
||||
*)
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
165
crdt-sync/tool/verify_desktop_client.py
Normal file
165
crdt-sync/tool/verify_desktop_client.py
Normal file
@ -0,0 +1,165 @@
|
||||
# Copyright (c) 2026 Krzysztof Rudnicki
|
||||
"""Prove Firebase accepts an id_token issued to the stored Desktop OAuth client.
|
||||
|
||||
Runs the PKCE loopback flow with the installed-app client (secret included in
|
||||
the exchange), then calls Firebase Auth REST ``signInWithIdp`` with the result.
|
||||
The consent click in the browser is the only human step. No app credential
|
||||
file is written: the session lives in memory and is dropped.
|
||||
|
||||
Run from the crdt-sync directory::
|
||||
|
||||
python3 -m tool.verify_desktop_client
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
import sys
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from crdt_sync._autherrors import FirebaseAuthError
|
||||
from crdt_sync._config import ConfigError, FirebaseConfig
|
||||
from crdt_sync._credentials import MemoryCredentialStore
|
||||
from crdt_sync._firebase_auth import FirebaseTokenProvider
|
||||
from tool._oauth_callback import (
|
||||
_CONSENT_TIMEOUT_SECONDS,
|
||||
_CallbackHandler,
|
||||
_free_port,
|
||||
_start_callback_server,
|
||||
)
|
||||
from tool._pkce import build_consent_url, make_pkce, open_detached
|
||||
from tool._token_exchange import TokenError, _authorization_code, _exchange_code
|
||||
from tool.client_probe import probe_client
|
||||
from tool.desktop_client import ClientFileError, DesktopClient, load_client
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
_logger = logging.getLogger("verify_desktop_client")
|
||||
|
||||
|
||||
def audience_of(id_token: str) -> str:
|
||||
"""Return the ``aud`` claim of an unverified JWT, or ``?`` if unreadable."""
|
||||
try:
|
||||
segment = id_token.split(".")[1]
|
||||
claims = json.loads(
|
||||
base64.urlsafe_b64decode(segment + "=" * (-len(segment) % 4))
|
||||
)
|
||||
return str(claims["aud"])
|
||||
except IndexError, ValueError, KeyError, TypeError:
|
||||
return "?"
|
||||
|
||||
|
||||
def hint_for(message: str) -> str:
|
||||
"""Return what to do about a known failure message, or an empty string."""
|
||||
if "audience mismatch" in message:
|
||||
return (
|
||||
"Firebase does not trust this client yet: Firebase console > "
|
||||
"Authentication > Sign-in method > Google > Safelist client IDs "
|
||||
"from external projects, add the Desktop client id, then rerun."
|
||||
)
|
||||
if "wrong account" in message:
|
||||
return (
|
||||
"Pick the sync account in the consent picker (login_hint only "
|
||||
"preselects it). Firebase may have created a stray user for the "
|
||||
"other account: check Authentication > Users."
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
def fetch_id_token(
|
||||
client: DesktopClient,
|
||||
login_hint: str,
|
||||
*,
|
||||
open_browser: bool = True,
|
||||
opener: Callable[[str], bool] = open_detached,
|
||||
) -> str:
|
||||
"""Run the PKCE loopback flow with ``client`` and return its id_token."""
|
||||
port = _free_port()
|
||||
redirect_uri = f"http://127.0.0.1:{port}"
|
||||
state = secrets.token_urlsafe(16)
|
||||
pkce = make_pkce()
|
||||
server, thread = _start_callback_server(port, state)
|
||||
url = build_consent_url(
|
||||
client_id=client.client_id,
|
||||
redirect_uri=redirect_uri,
|
||||
challenge=pkce.challenge,
|
||||
state=state,
|
||||
login_hint=login_hint,
|
||||
)
|
||||
_logger.info(" .. consent page (opened in your browser, tap Allow):")
|
||||
_logger.info(" %s", url)
|
||||
if open_browser:
|
||||
opener(url)
|
||||
thread.join(timeout=_CONSENT_TIMEOUT_SECONDS)
|
||||
server.server_close()
|
||||
code = _authorization_code(_CallbackHandler.result, redirect_uri, state)
|
||||
return _exchange_code(
|
||||
code, client.client_id, client.client_secret, redirect_uri, pkce.verifier
|
||||
)
|
||||
|
||||
|
||||
def sign_in(config: FirebaseConfig, id_token: str) -> str:
|
||||
"""Call ``signInWithIdp`` in memory; return the account email.
|
||||
|
||||
Raises:
|
||||
FirebaseAuthError: With Firebase's exact message, or on a uid other
|
||||
than the pinned one.
|
||||
"""
|
||||
provider = FirebaseTokenProvider(config.api_key, MemoryCredentialStore())
|
||||
return provider.sign_in_with_google(id_token, expected_uid=config.uid)
|
||||
|
||||
|
||||
def _fail(message: str) -> int:
|
||||
_logger.error(" FAIL %s", message)
|
||||
hint = hint_for(message)
|
||||
if hint:
|
||||
_logger.error(" .. %s", hint)
|
||||
return 1
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""Probe the stored client, run the consent flow, sign in to Firebase."""
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--no-browser", action="store_true", help="only print the URL")
|
||||
parser.add_argument(
|
||||
"--skip-probe", action="store_true", help="skip click-free checks"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--probe-only", action="store_true", help="stop after the click-free checks"
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s")
|
||||
|
||||
try:
|
||||
config = FirebaseConfig.load()
|
||||
client = load_client()
|
||||
if not args.skip_probe:
|
||||
results = probe_client(client)
|
||||
for result in results:
|
||||
_logger.info(" %s %s", "ok" if result.ok else "!!", result.detail)
|
||||
failed = [r.detail for r in results if not r.ok]
|
||||
if failed:
|
||||
return _fail("; ".join(failed))
|
||||
if args.probe_only:
|
||||
return 0
|
||||
id_token = fetch_id_token(
|
||||
client, config.email, open_browser=not args.no_browser
|
||||
)
|
||||
_logger.info(" .. id_token audience: %s", audience_of(id_token))
|
||||
email = sign_in(config, id_token)
|
||||
except (ConfigError, ClientFileError, TokenError, FirebaseAuthError) as exc:
|
||||
return _fail(str(exc))
|
||||
|
||||
_logger.info(
|
||||
" PASS Firebase accepted the Desktop client: uid %s, %s", config.uid, email
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@ -118,6 +118,16 @@ The public/private split is not cosmetic: every repo in this fleet is public.
|
||||
APK — the security rules, not secrecy, protect the data. The account email and
|
||||
password identify *a person* and are entered once per device.
|
||||
|
||||
### Sandbox namespace for test builds
|
||||
|
||||
`--dart-define=SYNC_NAMESPACE=sandbox/slot3` moves every RTDB read and write
|
||||
under `/sandbox/slot3/...`, so a test build never touches real data. Sign-in
|
||||
is unchanged (the real account). Empty, the default, keeps the exact current
|
||||
paths. Only `^sandbox/[a-z0-9_-]+$` is accepted; anything else throws
|
||||
`ConfigException` at startup (`firebaseClientFor` / `FirebaseRestClient`), never
|
||||
falling back to production paths. The shipped rules (root-level uid pin) already
|
||||
allow these paths.
|
||||
|
||||
### `password: ''` is legal
|
||||
|
||||
The Google one-tap path and the seeded-session path both store an empty
|
||||
|
||||
@ -2,6 +2,13 @@
|
||||
library;
|
||||
|
||||
export 'src/device.dart';
|
||||
export 'src/e2e_audit.dart';
|
||||
export 'src/e2e_cipher.dart';
|
||||
export 'src/e2e_envelope.dart';
|
||||
export 'src/e2e_key.dart';
|
||||
export 'src/e2e_migration.dart';
|
||||
export 'src/e2e_open.dart';
|
||||
export 'src/encrypting_store.dart';
|
||||
export 'src/firebase_account_provisioning.dart';
|
||||
export 'src/firebase_auth_error.dart';
|
||||
export 'src/firebase_auth_rest.dart';
|
||||
@ -20,4 +27,5 @@ export 'src/remote_store.dart';
|
||||
export 'src/secure_credential_store.dart';
|
||||
export 'src/store.dart';
|
||||
export 'src/sync.dart';
|
||||
export 'src/sync_namespace.dart';
|
||||
export 'src/sync_state.dart';
|
||||
|
||||
23
crdt_sync_dart/lib/src/e2e_audit.dart
Normal file
23
crdt_sync_dart/lib/src/e2e_audit.dart
Normal file
@ -0,0 +1,23 @@
|
||||
import 'e2e_cipher.dart';
|
||||
import 'e2e_envelope.dart';
|
||||
import 'firebase_keys.dart';
|
||||
|
||||
/// Plaintext leaves in a decoded RTDB dump; empty once an app is migrated.
|
||||
/// Decodes because RTDB serves blobs as escaped strings that defeat grep.
|
||||
List<String> plaintextBlobPaths(Object? tree) {
|
||||
final found = <String>[];
|
||||
void walk(Object? node, String path) {
|
||||
if (node is String) {
|
||||
if (E2eCipher.isPlaintext(node) && !isE2eEnvelope(node)) found.add(path);
|
||||
} else if (node is Map) {
|
||||
for (final MapEntry(:key, :value) in node.entries) {
|
||||
if (key == 'e2e' && value is String) continue;
|
||||
final segment = decodeKey(key as String);
|
||||
walk(value, path.isEmpty ? segment : '$path/$segment');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
walk(tree, '');
|
||||
return found;
|
||||
}
|
||||
97
crdt_sync_dart/lib/src/e2e_cipher.dart
Normal file
97
crdt_sync_dart/lib/src/e2e_cipher.dart
Normal file
@ -0,0 +1,97 @@
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:cryptography/cryptography.dart';
|
||||
|
||||
const int e2eFormatVersion = 1;
|
||||
|
||||
const _nonceLength = 24;
|
||||
const _tagLength = 16;
|
||||
|
||||
/// A FormatException so `syncLog` skips the peer instead of aborting the tick.
|
||||
class E2eDecryptError extends FormatException {
|
||||
E2eDecryptError(super.message);
|
||||
|
||||
@override
|
||||
String toString() => 'E2eDecryptError: $message';
|
||||
}
|
||||
|
||||
/// XChaCha20-Poly1305 per blob: base64 `[version][nonce][ct][tag]`, with the
|
||||
/// version byte and remote path as AAD so blobs cannot be swapped.
|
||||
///
|
||||
/// [acceptPlaintext] reads un-migrated peers; turn it off after migration.
|
||||
class E2eCipher {
|
||||
E2eCipher(List<int> key, {this.acceptPlaintext = true})
|
||||
: _key = SecretKey(List<int>.unmodifiable(key)) {
|
||||
if (key.length != 32) {
|
||||
throw ArgumentError.value(key.length, 'key', 'must be 32 bytes');
|
||||
}
|
||||
}
|
||||
|
||||
final SecretKey _key;
|
||||
final bool acceptPlaintext;
|
||||
|
||||
static final _aead = Xchacha20.poly1305Aead();
|
||||
|
||||
static bool isPlaintext(String stored) {
|
||||
final trimmed = stored.trimLeft();
|
||||
return trimmed.startsWith('{') || trimmed.startsWith('[');
|
||||
}
|
||||
|
||||
Future<String> seal(String plaintext, {required String path}) async {
|
||||
final box = await _aead.encrypt(
|
||||
utf8.encode(plaintext),
|
||||
secretKey: _key,
|
||||
nonce: _aead.newNonce(),
|
||||
aad: _aad(e2eFormatVersion, path),
|
||||
);
|
||||
return base64.encode([
|
||||
e2eFormatVersion,
|
||||
...box.nonce,
|
||||
...box.cipherText,
|
||||
...box.mac.bytes,
|
||||
]);
|
||||
}
|
||||
|
||||
Future<String> open(String stored, {required String path}) async {
|
||||
if (isPlaintext(stored)) {
|
||||
if (acceptPlaintext) return stored;
|
||||
throw E2eDecryptError('plaintext blob at $path after migration');
|
||||
}
|
||||
final List<int> bytes;
|
||||
try {
|
||||
bytes = base64.decode(stored);
|
||||
} on FormatException {
|
||||
throw E2eDecryptError('blob at $path is neither JSON nor base64');
|
||||
}
|
||||
if (bytes.isEmpty) throw E2eDecryptError('empty blob at $path');
|
||||
final version = bytes.first;
|
||||
if (version != e2eFormatVersion) {
|
||||
throw E2eDecryptError(
|
||||
'blob at $path uses format $version; update the app to read it',
|
||||
);
|
||||
}
|
||||
if (bytes.length < 1 + _nonceLength + _tagLength) {
|
||||
throw E2eDecryptError('blob at $path is truncated');
|
||||
}
|
||||
final box = SecretBox(
|
||||
bytes.sublist(1 + _nonceLength, bytes.length - _tagLength),
|
||||
nonce: bytes.sublist(1, 1 + _nonceLength),
|
||||
mac: Mac(bytes.sublist(bytes.length - _tagLength)),
|
||||
);
|
||||
try {
|
||||
final clear = await _aead.decrypt(
|
||||
box,
|
||||
secretKey: _key,
|
||||
aad: _aad(version, path),
|
||||
);
|
||||
return utf8.decode(clear);
|
||||
} on SecretBoxAuthenticationError {
|
||||
throw E2eDecryptError('blob at $path failed authentication');
|
||||
}
|
||||
}
|
||||
|
||||
static List<int> _aad(int version, String path) => [
|
||||
version,
|
||||
...utf8.encode(path),
|
||||
];
|
||||
}
|
||||
123
crdt_sync_dart/lib/src/e2e_envelope.dart
Normal file
123
crdt_sync_dart/lib/src/e2e_envelope.dart
Normal file
@ -0,0 +1,123 @@
|
||||
/// The per-record wire envelope: record ids, the tombstone and the newest HLC
|
||||
/// stay in clear so the relay can order and inspect; every field name and
|
||||
/// value is sealed. Merge still runs on the decrypted [Log].
|
||||
library;
|
||||
|
||||
import 'dart:convert';
|
||||
|
||||
import 'e2e_cipher.dart';
|
||||
import 'hlc.dart';
|
||||
|
||||
const int _envelopeVersion = 1;
|
||||
|
||||
/// Whether [text] is an envelope (explicit `"v"` integer), not legacy plaintext.
|
||||
///
|
||||
/// A legacy log maps record ids to objects, so an id named `v` still maps to
|
||||
/// a map, never an int.
|
||||
bool isE2eEnvelope(String text) {
|
||||
try {
|
||||
final json = jsonDecode(text);
|
||||
return json is Map && json['v'] is int;
|
||||
} on FormatException {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
String _aad(
|
||||
String path,
|
||||
String id,
|
||||
bool deleted,
|
||||
String? deletedHlc,
|
||||
String? h,
|
||||
) => '$path#$id#${deleted ? 1 : 0}#${deletedHlc ?? ''}#${h ?? ''}';
|
||||
|
||||
/// Seals a log (the `logToJson` shape) into an envelope. Text that is not
|
||||
/// log-shaped is sealed whole, so nothing is ever pushed in clear.
|
||||
Future<String> sealLogText(
|
||||
String text,
|
||||
E2eCipher cipher, {
|
||||
required String path,
|
||||
}) async {
|
||||
final Map<String, dynamic> raw;
|
||||
try {
|
||||
raw = (jsonDecode(text) as Map).cast<String, dynamic>();
|
||||
for (final entry in raw.entries) {
|
||||
final rec = entry.value as Map;
|
||||
if (rec['id'] != entry.key || rec['fields'] is! Map) {
|
||||
throw const FormatException('not a log');
|
||||
}
|
||||
}
|
||||
} on Object {
|
||||
return jsonEncode({
|
||||
'v': _envelopeVersion,
|
||||
'b': await cipher.seal(text, path: path),
|
||||
});
|
||||
}
|
||||
final records = <String, Object?>{};
|
||||
for (final entry in raw.entries) {
|
||||
final rec = (entry.value as Map).cast<String, dynamic>();
|
||||
final fields = (rec['fields'] as Map).cast<String, dynamic>();
|
||||
Hlc? newest;
|
||||
for (final field in fields.values) {
|
||||
final hlc = Hlc.fromStr((field as List)[1] as String);
|
||||
if (newest == null || hlc > newest) newest = hlc;
|
||||
}
|
||||
final deleted = rec['deleted'] as bool? ?? false;
|
||||
final deletedHlc = rec['deleted_hlc'] as String?;
|
||||
final h = newest?.toStr();
|
||||
records[entry.key] = {
|
||||
'd': deleted,
|
||||
'dh': deletedHlc,
|
||||
'h': h,
|
||||
'p': await cipher.seal(
|
||||
jsonEncode(fields),
|
||||
path: _aad(path, entry.key, deleted, deletedHlc, h),
|
||||
),
|
||||
};
|
||||
}
|
||||
return jsonEncode({'v': _envelopeVersion, 'r': records});
|
||||
}
|
||||
|
||||
/// Opens an envelope back to the `logToJson` shape. Legacy plaintext is
|
||||
/// returned as-is while the cipher accepts it.
|
||||
Future<String> openLogText(
|
||||
String text,
|
||||
E2eCipher cipher, {
|
||||
required String path,
|
||||
}) async {
|
||||
if (!isE2eEnvelope(text)) {
|
||||
if (cipher.acceptPlaintext) return text;
|
||||
throw E2eDecryptError('plaintext blob at $path after migration');
|
||||
}
|
||||
final json = (jsonDecode(text) as Map).cast<String, dynamic>();
|
||||
if (json['v'] != _envelopeVersion) {
|
||||
throw E2eDecryptError(
|
||||
'blob at $path uses format ${json['v']}; update the app to read it',
|
||||
);
|
||||
}
|
||||
final whole = json['b'];
|
||||
if (whole is String) return cipher.open(whole, path: path);
|
||||
final records = json['r'];
|
||||
if (records is! Map) throw E2eDecryptError('envelope at $path is mis-shaped');
|
||||
final out = <String, Object?>{};
|
||||
for (final entry in records.entries) {
|
||||
final id = entry.key as String;
|
||||
final rec = entry.value;
|
||||
if (rec is! Map || rec['p'] is! String || rec['d'] is! bool) {
|
||||
throw E2eDecryptError('record $id at $path is mis-shaped');
|
||||
}
|
||||
final deletedHlc = rec['dh'] as String?;
|
||||
final h = rec['h'] as String?;
|
||||
final clear = await cipher.open(
|
||||
rec['p'] as String,
|
||||
path: _aad(path, id, rec['d'] as bool, deletedHlc, h),
|
||||
);
|
||||
out[id] = {
|
||||
'id': id,
|
||||
'fields': jsonDecode(clear),
|
||||
'deleted': rec['d'],
|
||||
'deleted_hlc': deletedHlc,
|
||||
};
|
||||
}
|
||||
return jsonEncode(out);
|
||||
}
|
||||
254
crdt_sync_dart/lib/src/e2e_key.dart
Normal file
254
crdt_sync_dart/lib/src/e2e_key.dart
Normal file
@ -0,0 +1,254 @@
|
||||
import 'dart:convert';
|
||||
import 'dart:math';
|
||||
|
||||
import 'package:cryptography/cryptography.dart';
|
||||
|
||||
import 'e2e_cipher.dart';
|
||||
import 'encrypting_store.dart';
|
||||
import 'remote_store.dart';
|
||||
|
||||
const _saltLength = 16;
|
||||
const _headerVersion = 1;
|
||||
|
||||
/// Argon2id cost, stored in the header so it can be raised later.
|
||||
class Argon2Params {
|
||||
const Argon2Params({
|
||||
required this.memoryKiB,
|
||||
required this.iterations,
|
||||
required this.parallelism,
|
||||
});
|
||||
|
||||
/// OWASP's floor (19 MiB, t=2); higher stalls the Flutter-web desktop build.
|
||||
static const recommended = Argon2Params(
|
||||
memoryKiB: 19456,
|
||||
iterations: 2,
|
||||
parallelism: 1,
|
||||
);
|
||||
|
||||
final int memoryKiB;
|
||||
final int iterations;
|
||||
final int parallelism;
|
||||
}
|
||||
|
||||
class E2eHeaderError extends FormatException {
|
||||
E2eHeaderError(super.message);
|
||||
|
||||
@override
|
||||
String toString() => 'E2eHeaderError: $message';
|
||||
}
|
||||
|
||||
class E2eWrongPassphraseError implements Exception {
|
||||
@override
|
||||
String toString() => 'E2eWrongPassphraseError: wrong sync passphrase';
|
||||
}
|
||||
|
||||
/// The clear header's path: a sibling of the device dirs, like `revs`.
|
||||
String defaultE2ePath(String pathPrefix) {
|
||||
final cut = pathPrefix.lastIndexOf('/');
|
||||
return cut < 0 ? '$pathPrefix/e2e' : '${pathPrefix.substring(0, cut)}/e2e';
|
||||
}
|
||||
|
||||
class _Header {
|
||||
_Header(this.salt, this.params, this.check);
|
||||
|
||||
factory _Header.parse(String text) {
|
||||
final Object? json;
|
||||
try {
|
||||
json = jsonDecode(text);
|
||||
} on FormatException {
|
||||
throw E2eHeaderError('e2e header is not JSON');
|
||||
}
|
||||
if (json is! Map<String, dynamic>) {
|
||||
throw E2eHeaderError('e2e header is not an object');
|
||||
}
|
||||
if (json['v'] != _headerVersion) {
|
||||
throw E2eHeaderError('e2e header v${json['v']}; update the app');
|
||||
}
|
||||
try {
|
||||
if (json['kdf'] != 'argon2id') throw const FormatException();
|
||||
return _Header(
|
||||
base64.decode(json['salt'] as String),
|
||||
Argon2Params(
|
||||
memoryKiB: json['m'] as int,
|
||||
iterations: json['t'] as int,
|
||||
parallelism: json['p'] as int,
|
||||
),
|
||||
json['check'] as String,
|
||||
);
|
||||
} on Object {
|
||||
throw E2eHeaderError('e2e header is malformed');
|
||||
}
|
||||
}
|
||||
|
||||
final List<int> salt;
|
||||
final Argon2Params params;
|
||||
final String check;
|
||||
|
||||
String encode() => jsonEncode({
|
||||
'v': _headerVersion,
|
||||
'kdf': 'argon2id',
|
||||
'salt': base64.encode(salt),
|
||||
'm': params.memoryKiB,
|
||||
't': params.iterations,
|
||||
'p': params.parallelism,
|
||||
'check': check,
|
||||
});
|
||||
}
|
||||
|
||||
/// A 32-byte master key plus its HKDF subkeys for sealing, revisions and the
|
||||
/// passphrase check. Cache [bytes] in the OS keystore, never in RTDB.
|
||||
class E2eKey {
|
||||
E2eKey._(this.bytes, this._enc, this._rev, this._check);
|
||||
|
||||
static Future<E2eKey> fromBytes(List<int> bytes) {
|
||||
if (bytes.length != 32) {
|
||||
throw ArgumentError.value(bytes.length, 'bytes', 'must be 32 bytes');
|
||||
}
|
||||
return _expand(List<int>.unmodifiable(bytes));
|
||||
}
|
||||
|
||||
static Future<E2eKey> _expand(List<int> master) async {
|
||||
Future<List<int>> sub(String label) async {
|
||||
final key = await Hkdf(hmac: Hmac.sha256(), outputLength: 32).deriveKey(
|
||||
secretKey: SecretKey(master),
|
||||
info: utf8.encode('crdt_sync e2e $label'),
|
||||
);
|
||||
return key.bytes;
|
||||
}
|
||||
|
||||
return E2eKey._(
|
||||
master,
|
||||
await sub('enc'),
|
||||
SecretKey(await sub('rev')),
|
||||
SecretKey(await sub('check')),
|
||||
);
|
||||
}
|
||||
|
||||
final List<int> bytes;
|
||||
final List<int> _enc;
|
||||
final SecretKey _rev;
|
||||
final SecretKey _check;
|
||||
|
||||
E2eCipher cipher({bool acceptPlaintext = true}) =>
|
||||
E2eCipher(_enc, acceptPlaintext: acceptPlaintext);
|
||||
|
||||
/// Keyed `revisionOf`: a published revision cannot confirm a guessed log.
|
||||
Future<String> revision(String text) async {
|
||||
final mac = await Hmac.sha256().calculateMac(
|
||||
utf8.encode(text),
|
||||
secretKey: _rev,
|
||||
);
|
||||
return mac.bytes.map((b) => b.toRadixString(16).padLeft(2, '0')).join();
|
||||
}
|
||||
|
||||
Future<String> _checkFor(List<int> salt) async {
|
||||
final mac = await Hmac.sha256().calculateMac(salt, secretKey: _check);
|
||||
return base64.encode(mac.bytes);
|
||||
}
|
||||
}
|
||||
|
||||
Future<E2eKey> _derive(
|
||||
String passphrase,
|
||||
List<int> salt,
|
||||
Argon2Params params,
|
||||
) async {
|
||||
final key = await Argon2id(
|
||||
parallelism: params.parallelism,
|
||||
memory: params.memoryKiB,
|
||||
iterations: params.iterations,
|
||||
hashLength: 32,
|
||||
).deriveKeyFromPassword(password: passphrase, nonce: salt);
|
||||
return E2eKey._expand(await key.extractBytes());
|
||||
}
|
||||
|
||||
Future<_Header?> _readHeader(RemoteStore store, String path) async {
|
||||
final text = await store.getFileText(path);
|
||||
return text == null ? null : _Header.parse(text);
|
||||
}
|
||||
|
||||
/// Derives this app's key from [passphrase], creating the header on first use.
|
||||
///
|
||||
/// Throws [E2eWrongPassphraseError] on a failed check and [E2eHeaderError]
|
||||
/// for an unreadable header, which is never overwritten.
|
||||
Future<E2eKey> unlockE2e({
|
||||
required RemoteStore store,
|
||||
required String pathPrefix,
|
||||
required String passphrase,
|
||||
Argon2Params params = Argon2Params.recommended,
|
||||
Random? random,
|
||||
String? e2ePath,
|
||||
}) async {
|
||||
if (passphrase.isEmpty) {
|
||||
throw ArgumentError.value('', 'passphrase', 'must not be empty');
|
||||
}
|
||||
final path = e2ePath ?? defaultE2ePath(pathPrefix);
|
||||
var header = await _readHeader(store, path);
|
||||
if (header == null) {
|
||||
final rng = random ?? Random.secure();
|
||||
final salt = List<int>.generate(_saltLength, (_) => rng.nextInt(256));
|
||||
final key = await _derive(passphrase, salt, params);
|
||||
final encoded = _Header(salt, params, await key._checkFor(salt)).encode();
|
||||
await store.putFileText(path, encoded, message: path);
|
||||
// Two devices setting up at once: the last write wins, so re-read.
|
||||
final stored = await store.getFileText(path) ?? encoded;
|
||||
if (stored == encoded) return key;
|
||||
header = _Header.parse(stored);
|
||||
}
|
||||
final key = await _derive(passphrase, header.salt, header.params);
|
||||
if (await key._checkFor(header.salt) != header.check) {
|
||||
throw E2eWrongPassphraseError();
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
sealed class E2eState {}
|
||||
|
||||
/// No header and no key: this app has never been encrypted.
|
||||
final class E2eOff extends E2eState {}
|
||||
|
||||
enum E2eLockReason { needsPassphrase, wrongKey, badHeader }
|
||||
|
||||
/// Encryption is on, but this device must not sync until it has the key.
|
||||
final class E2eLocked extends E2eState {
|
||||
E2eLocked(this.reason);
|
||||
|
||||
final E2eLockReason reason;
|
||||
}
|
||||
|
||||
final class E2eReady extends E2eState {
|
||||
E2eReady(this.store);
|
||||
|
||||
final EncryptingStore store;
|
||||
}
|
||||
|
||||
Future<E2eState> openE2e({
|
||||
required RemoteStore store,
|
||||
required String pathPrefix,
|
||||
E2eKey? key,
|
||||
String? e2ePath,
|
||||
String? revsPath,
|
||||
bool acceptPlaintext = true,
|
||||
}) async {
|
||||
final _Header? header;
|
||||
try {
|
||||
header = await _readHeader(store, e2ePath ?? defaultE2ePath(pathPrefix));
|
||||
} on E2eHeaderError {
|
||||
return E2eLocked(E2eLockReason.badHeader);
|
||||
}
|
||||
if (key == null) {
|
||||
return header == null ? E2eOff() : E2eLocked(E2eLockReason.needsPassphrase);
|
||||
}
|
||||
if (header == null || await key._checkFor(header.salt) != header.check) {
|
||||
return E2eLocked(E2eLockReason.wrongKey);
|
||||
}
|
||||
return E2eReady(
|
||||
EncryptingStore(
|
||||
store,
|
||||
key,
|
||||
pathPrefix: pathPrefix,
|
||||
revsPath: revsPath,
|
||||
e2ePath: e2ePath,
|
||||
acceptPlaintext: acceptPlaintext,
|
||||
),
|
||||
);
|
||||
}
|
||||
217
crdt_sync_dart/lib/src/e2e_migration.dart
Normal file
217
crdt_sync_dart/lib/src/e2e_migration.dart
Normal file
@ -0,0 +1,217 @@
|
||||
/// The one-time copy from a plaintext prefix to `<prefix>-e2e`.
|
||||
///
|
||||
/// Re-encrypting in place is rejected: an old build still writing the old
|
||||
/// prefix would be broken mid-migration. Instead the old prefix stays
|
||||
/// untouched until [deletePlaintextPrefix] verifies the copy.
|
||||
library;
|
||||
|
||||
import 'dart:convert';
|
||||
|
||||
import 'e2e_key.dart';
|
||||
import 'encrypting_store.dart';
|
||||
import 'log.dart';
|
||||
import 'record.dart';
|
||||
import 'remote_store.dart';
|
||||
import 'store.dart';
|
||||
import 'sync.dart';
|
||||
|
||||
/// Where an app's encrypted data lives, derived from its old plaintext prefix.
|
||||
class E2eLayout {
|
||||
const E2eLayout._(this.prefix, this.revsPath, this.e2ePath);
|
||||
|
||||
/// `<oldPrefix>-e2e`, with revisions and header apart from the old ones so
|
||||
/// old builds neither see nor clobber them.
|
||||
///
|
||||
/// A slashless prefix (`settings-sync`) keeps revisions and header inside
|
||||
/// its own node, as `defaultRevsPath` does, never at the database root.
|
||||
factory E2eLayout.of(String oldPrefix) {
|
||||
final prefix = '$oldPrefix-e2e';
|
||||
final cut = oldPrefix.lastIndexOf('/');
|
||||
if (cut < 0) {
|
||||
return E2eLayout._(prefix, '$prefix/revs', '$prefix/e2e');
|
||||
}
|
||||
final parent = oldPrefix.substring(0, cut);
|
||||
return E2eLayout._(prefix, '$parent/revs-e2e', defaultE2ePath(prefix));
|
||||
}
|
||||
|
||||
final String prefix;
|
||||
final String revsPath;
|
||||
final String e2ePath;
|
||||
}
|
||||
|
||||
class MigrationReport {
|
||||
const MigrationReport(this.copied, this.skipped);
|
||||
|
||||
/// Blobs written (new or changed) this run.
|
||||
final int copied;
|
||||
|
||||
/// Source blobs unreadable as a log, relative path.
|
||||
final List<String> skipped;
|
||||
}
|
||||
|
||||
/// Every text blob under [root], keyed by path relative to it: flat
|
||||
/// `<node>.json` leaves and nested `<device>/<file>` blobs. Real RTDB bulk
|
||||
/// reads return direct string children only, so dirs are walked.
|
||||
Future<Map<String, String>> _listBlobs(RemoteStore store, String root) async {
|
||||
final out = <String, String>{};
|
||||
if (store is BulkMapReader) {
|
||||
out.addAll(await (store as BulkMapReader).getStringMap(root));
|
||||
}
|
||||
final revs = defaultRevsPath(root);
|
||||
final header = defaultE2ePath(root);
|
||||
for (final name in await store.listDirectory(root)) {
|
||||
if (out.containsKey(name)) continue;
|
||||
final full = '$root/$name';
|
||||
if (full == header || full == revs) continue;
|
||||
final children = await _safeList(store, full);
|
||||
if (children.isEmpty) {
|
||||
final text = await _safeRead(store, full);
|
||||
if (text != null) out[name] = text;
|
||||
continue;
|
||||
}
|
||||
for (final child in children) {
|
||||
final text = await _safeRead(store, '$full/$child');
|
||||
if (text != null) out['$name/$child'] = text;
|
||||
}
|
||||
}
|
||||
out.removeWhere(
|
||||
(rel, _) => '$root/$rel' == header || '$root/$rel'.startsWith('$revs/'),
|
||||
);
|
||||
return out;
|
||||
}
|
||||
|
||||
Future<List<String>> _safeList(RemoteStore store, String path) async {
|
||||
try {
|
||||
return await store.listDirectory(path);
|
||||
} on Object {
|
||||
return const [];
|
||||
}
|
||||
}
|
||||
|
||||
Future<String?> _safeRead(RemoteStore store, String path) async {
|
||||
try {
|
||||
return await store.getFileText(path);
|
||||
} on Object {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
EncryptingStore _dest(RemoteStore store, E2eKey key, E2eLayout layout) =>
|
||||
EncryptingStore(
|
||||
store,
|
||||
key,
|
||||
pathPrefix: layout.prefix,
|
||||
revsPath: layout.revsPath,
|
||||
e2ePath: layout.e2ePath,
|
||||
);
|
||||
|
||||
/// Order-independent signature of a log, to tell "same records" cheaply.
|
||||
String _signature(Log log) {
|
||||
final ids = log.keys.toList()..sort();
|
||||
return jsonEncode([
|
||||
for (final id in ids)
|
||||
[
|
||||
id,
|
||||
log[id]!.deleted,
|
||||
log[id]!.deletedHlc?.toStr(),
|
||||
[
|
||||
for (final name in (log[id]!.fields.keys.toList()..sort()))
|
||||
[
|
||||
name,
|
||||
jsonEncode(log[id]!.fields[name]!.$1),
|
||||
log[id]!.fields[name]!.$2.toStr(),
|
||||
],
|
||||
],
|
||||
],
|
||||
]);
|
||||
}
|
||||
|
||||
Log? _parse(String text) {
|
||||
try {
|
||||
return logFromJson(text);
|
||||
} on FormatException {
|
||||
return null;
|
||||
} on TypeError {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// Copies every readable plaintext log under [fromPrefix] into the encrypted
|
||||
/// prefix, merging into what is already there. Idempotent: an unchanged
|
||||
/// destination is not rewritten. Never touches [fromPrefix].
|
||||
Future<MigrationReport> copyToEncrypted({
|
||||
required RemoteStore store,
|
||||
required E2eKey key,
|
||||
required String fromPrefix,
|
||||
required E2eLayout layout,
|
||||
}) async {
|
||||
final dest = _dest(store, key, layout);
|
||||
var copied = 0;
|
||||
final skipped = <String>[];
|
||||
final blobs = await _listBlobs(store, fromPrefix);
|
||||
for (final MapEntry(key: rel, value: text) in blobs.entries) {
|
||||
final source = _parse(text);
|
||||
if (source == null) {
|
||||
skipped.add(rel);
|
||||
continue;
|
||||
}
|
||||
if (source.isEmpty) continue;
|
||||
final path = '${layout.prefix}/$rel';
|
||||
final existing = await dest.getFileText(path);
|
||||
final current = existing == null ? null : _parse(existing);
|
||||
// Not a sync log (an app's own blob): never overwrite it.
|
||||
if (existing != null && current == null) continue;
|
||||
final merged = mergeLogs(current ?? <String, Record>{}, source);
|
||||
if (current != null && _signature(merged) == _signature(current)) continue;
|
||||
await dest.putFileText(path, logToJson(merged), message: 'e2e copy');
|
||||
copied++;
|
||||
}
|
||||
return MigrationReport(copied, skipped);
|
||||
}
|
||||
|
||||
/// True when every readable plaintext blob is contained in the encrypted copy
|
||||
/// and none is unreadable.
|
||||
Future<bool> verifyCopy({
|
||||
required RemoteStore store,
|
||||
required E2eKey key,
|
||||
required String fromPrefix,
|
||||
required E2eLayout layout,
|
||||
}) async {
|
||||
final dest = _dest(store, key, layout);
|
||||
for (final MapEntry(key: rel, value: text) in (await _listBlobs(
|
||||
store,
|
||||
fromPrefix,
|
||||
)).entries) {
|
||||
final source = _parse(text);
|
||||
if (source == null) return false;
|
||||
final existing = await dest.getFileText('${layout.prefix}/$rel');
|
||||
final current = existing == null ? null : _parse(existing);
|
||||
if (current == null) return false;
|
||||
if (_signature(mergeLogs(current, source)) != _signature(current)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Deletes the plaintext prefix, only after [verifyCopy] passes. Returns
|
||||
/// whether it deleted.
|
||||
Future<bool> deletePlaintextPrefix({
|
||||
required RemoteStore store,
|
||||
required E2eKey key,
|
||||
required String fromPrefix,
|
||||
required E2eLayout layout,
|
||||
}) async {
|
||||
if (!await verifyCopy(
|
||||
store: store,
|
||||
key: key,
|
||||
fromPrefix: fromPrefix,
|
||||
layout: layout,
|
||||
)) {
|
||||
return false;
|
||||
}
|
||||
for (final rel in (await _listBlobs(store, fromPrefix)).keys) {
|
||||
await store.deleteFile('$fromPrefix/$rel', message: 'e2e cleanup');
|
||||
}
|
||||
return true;
|
||||
}
|
||||
142
crdt_sync_dart/lib/src/e2e_open.dart
Normal file
142
crdt_sync_dart/lib/src/e2e_open.dart
Normal file
@ -0,0 +1,142 @@
|
||||
/// Opens the encrypted store for an app whose old plaintext prefix is
|
||||
/// [oldPrefix]; data lives on `E2eLayout.of(oldPrefix)`.
|
||||
///
|
||||
/// [readPassphrase] supplies the shared crdt-sync passphrase from the OS
|
||||
/// keystore, so the open is prompt-free. Fail closed: a missing or wrong
|
||||
/// passphrase, or an unreadable header, returns [E2eLocked] and the caller
|
||||
/// must not sync. On success the old plaintext prefix is copied over
|
||||
/// (idempotent; picks up writes from builds not yet migrated) and is never
|
||||
/// deleted here.
|
||||
library;
|
||||
|
||||
import 'dart:developer';
|
||||
|
||||
import 'e2e_key.dart';
|
||||
import 'e2e_migration.dart';
|
||||
import 'remote_store.dart';
|
||||
import 'sync.dart';
|
||||
|
||||
/// Keystore entry name apps store the shared passphrase under.
|
||||
const String kE2ePassphraseStorageKey = 'crdt_sync_e2e_passphrase';
|
||||
|
||||
/// Where a device keeps its derived 32-byte key, so Argon2id runs once per
|
||||
/// device rather than once per sync tick. Back it with the OS keystore.
|
||||
class E2eKeyCache {
|
||||
/// Creates a cache from [read] and [write] closures.
|
||||
const E2eKeyCache({required this.read, required this.write});
|
||||
|
||||
/// Returns the cached key bytes, or null.
|
||||
final Future<List<int>?> Function() read;
|
||||
|
||||
/// Stores freshly derived key bytes.
|
||||
final Future<void> Function(List<int> bytes) write;
|
||||
}
|
||||
|
||||
/// Old-prefix revision fingerprints already copied, per old prefix.
|
||||
final Map<String, String> _copied = {};
|
||||
|
||||
/// Forgets which old prefixes were copied; for tests.
|
||||
void resetE2eCopyMemo() => _copied.clear();
|
||||
|
||||
Future<String> _oldFingerprint(RemoteStore store, String oldPrefix) async {
|
||||
if (store is! BulkMapReader) return '';
|
||||
try {
|
||||
final revs = await (store as BulkMapReader).getStringMap(
|
||||
defaultRevsPath(oldPrefix),
|
||||
);
|
||||
final keys = revs.keys.toList()..sort();
|
||||
return [for (final k in keys) '$k=${revs[k]}'].join(';');
|
||||
} on Object {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
Future<E2eReady?> _tryCached(
|
||||
RemoteStore store,
|
||||
E2eLayout layout,
|
||||
E2eKeyCache? cache,
|
||||
) async {
|
||||
final bytes = await cache?.read();
|
||||
if (bytes == null || bytes.length != 32) return null;
|
||||
final state = await openE2e(
|
||||
store: store,
|
||||
pathPrefix: layout.prefix,
|
||||
key: await E2eKey.fromBytes(bytes),
|
||||
e2ePath: layout.e2ePath,
|
||||
revsPath: layout.revsPath,
|
||||
);
|
||||
return state is E2eReady ? state : null;
|
||||
}
|
||||
|
||||
/// Opens the encrypted store; see the library doc.
|
||||
///
|
||||
/// [keyCache] skips Argon2id when it holds the key for the current header.
|
||||
/// The one-time copy reruns only when the old prefix's published revisions
|
||||
/// change (or once per process where the backend cannot bulk-read them).
|
||||
/// [onDerive] is called each time Argon2id actually runs; for tests.
|
||||
Future<E2eState> openEncryptedStore({
|
||||
required RemoteStore store,
|
||||
required String oldPrefix,
|
||||
required Future<String?> Function() readPassphrase,
|
||||
Argon2Params params = Argon2Params.recommended,
|
||||
E2eKeyCache? keyCache,
|
||||
void Function()? onDerive,
|
||||
}) async {
|
||||
final layout = E2eLayout.of(oldPrefix);
|
||||
final E2eReady ready;
|
||||
final E2eReady? cached;
|
||||
try {
|
||||
cached = await _tryCached(store, layout, keyCache);
|
||||
} on E2eHeaderError {
|
||||
return E2eLocked(E2eLockReason.badHeader);
|
||||
}
|
||||
if (cached != null) {
|
||||
ready = cached;
|
||||
} else {
|
||||
final passphrase = await readPassphrase();
|
||||
if (passphrase == null || passphrase.isEmpty) {
|
||||
return E2eLocked(E2eLockReason.needsPassphrase);
|
||||
}
|
||||
final E2eKey key;
|
||||
try {
|
||||
onDerive?.call();
|
||||
key = await unlockE2e(
|
||||
store: store,
|
||||
pathPrefix: layout.prefix,
|
||||
passphrase: passphrase,
|
||||
params: params,
|
||||
e2ePath: layout.e2ePath,
|
||||
);
|
||||
} on E2eWrongPassphraseError {
|
||||
return E2eLocked(E2eLockReason.wrongKey);
|
||||
} on E2eHeaderError {
|
||||
return E2eLocked(E2eLockReason.badHeader);
|
||||
}
|
||||
await keyCache?.write(key.bytes);
|
||||
final state = await openE2e(
|
||||
store: store,
|
||||
pathPrefix: layout.prefix,
|
||||
key: key,
|
||||
e2ePath: layout.e2ePath,
|
||||
revsPath: layout.revsPath,
|
||||
);
|
||||
if (state is! E2eReady) return state;
|
||||
ready = state;
|
||||
}
|
||||
try {
|
||||
final print = await _oldFingerprint(store, oldPrefix);
|
||||
if (_copied[oldPrefix] != print) {
|
||||
await copyToEncrypted(
|
||||
store: store,
|
||||
key: ready.store.key,
|
||||
fromPrefix: oldPrefix,
|
||||
layout: layout,
|
||||
);
|
||||
_copied[oldPrefix] = print;
|
||||
}
|
||||
} on Object catch (error, stackTrace) {
|
||||
// The copy is retried next open; sync itself must not stop for it.
|
||||
log('e2e copy failed', level: 900, error: error, stackTrace: stackTrace);
|
||||
}
|
||||
return ready;
|
||||
}
|
||||
98
crdt_sync_dart/lib/src/encrypting_store.dart
Normal file
98
crdt_sync_dart/lib/src/encrypting_store.dart
Normal file
@ -0,0 +1,98 @@
|
||||
import 'e2e_cipher.dart';
|
||||
import 'e2e_envelope.dart';
|
||||
import 'e2e_key.dart';
|
||||
import 'remote_store.dart';
|
||||
import 'sync.dart';
|
||||
|
||||
/// A [RemoteStore] that seals every blob under `pathPrefix` (nested
|
||||
/// `<device>/<file>` or flat `<node>.json`) except the header, as a
|
||||
/// per-record envelope (ids and HLCs clear, names and values opaque), and
|
||||
/// keys every revision under `revsPath`, so the remote is a blind relay.
|
||||
///
|
||||
/// Wrap the outermost store (above a `MirrorStore`) so every backend gets
|
||||
/// ciphertext. Other paths pass through unchanged.
|
||||
class EncryptingStore implements RemoteStore, BulkMapReader {
|
||||
EncryptingStore(
|
||||
this._inner,
|
||||
this._key, {
|
||||
required String pathPrefix,
|
||||
String? revsPath,
|
||||
String? e2ePath,
|
||||
bool acceptPlaintext = true,
|
||||
}) : _cipher = _key.cipher(acceptPlaintext: acceptPlaintext),
|
||||
_prefix = pathPrefix,
|
||||
_revs = revsPath ?? defaultRevsPath(pathPrefix),
|
||||
_header = e2ePath ?? defaultE2ePath(pathPrefix);
|
||||
|
||||
final RemoteStore _inner;
|
||||
final E2eKey _key;
|
||||
|
||||
/// The key this store seals with.
|
||||
E2eKey get key => _key;
|
||||
final E2eCipher _cipher;
|
||||
final String _prefix;
|
||||
final String _revs;
|
||||
final String _header;
|
||||
|
||||
bool _isRev(String path) => path.startsWith('$_revs/');
|
||||
|
||||
bool _isLog(String path) =>
|
||||
!_isRev(path) && path != _header && path.startsWith('$_prefix/');
|
||||
|
||||
@override
|
||||
Future<String?> getFileText(String path) async {
|
||||
final text = await _inner.getFileText(path);
|
||||
if (text == null || !_isLog(path)) return text;
|
||||
return openLogText(text, _cipher, path: path);
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> putFileText(
|
||||
String path,
|
||||
String text, {
|
||||
required String message,
|
||||
}) async {
|
||||
final stored = _isLog(path)
|
||||
? await sealLogText(text, _cipher, path: path)
|
||||
: _isRev(path)
|
||||
? await _key.revision(text)
|
||||
: text;
|
||||
await _inner.putFileText(path, stored, message: message);
|
||||
}
|
||||
|
||||
@override
|
||||
Future<Map<String, String>> getStringMap(String path) async {
|
||||
if (_inner case final BulkMapReader reader) {
|
||||
final map = await reader.getStringMap(path);
|
||||
if (!_isLog('$path/x')) return map;
|
||||
final out = <String, String>{};
|
||||
for (final MapEntry(:key, :value) in map.entries) {
|
||||
final full = '$path/$key';
|
||||
if (!_isLog(full)) {
|
||||
out[key] = value;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
out[key] = await openLogText(value, _cipher, path: full);
|
||||
} on E2eDecryptError {
|
||||
// Unreadable entries are dropped, as a corrupt value would be.
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return const {};
|
||||
}
|
||||
|
||||
@override
|
||||
Future<List<String>> listDirectory(String path) => _inner.listDirectory(path);
|
||||
|
||||
@override
|
||||
Future<void> deleteFile(String path, {String message = ''}) =>
|
||||
_inner.deleteFile(path, message: message);
|
||||
|
||||
@override
|
||||
Future<bool> canAccessRemote() => _inner.canAccessRemote();
|
||||
|
||||
@override
|
||||
void close() => _inner.close();
|
||||
}
|
||||
@ -19,6 +19,7 @@ import 'firebase_auth_rest.dart';
|
||||
import 'firebase_errors.dart';
|
||||
import 'firebase_keys.dart';
|
||||
import 'remote_store.dart';
|
||||
import 'sync_namespace.dart';
|
||||
|
||||
const _httpUnauthorized = 401;
|
||||
const _httpForbidden = 403;
|
||||
@ -33,13 +34,22 @@ class FirebaseRestClient implements RemoteStore, BulkMapReader {
|
||||
required String databaseUrl,
|
||||
required this.auth,
|
||||
http.Client? httpClient,
|
||||
String namespace = syncNamespace,
|
||||
}) : _databaseUrl = databaseUrl.endsWith('/')
|
||||
? databaseUrl.substring(0, databaseUrl.length - 1)
|
||||
: databaseUrl,
|
||||
_prefix = _prefixFor(checkSyncNamespace(namespace)),
|
||||
_http = httpClient ?? http.Client();
|
||||
|
||||
final String _databaseUrl;
|
||||
|
||||
/// `<namespace>/` or empty: put in front of every RTDB path. Every request
|
||||
/// goes through [_uri], so this is the single choke point.
|
||||
final String _prefix;
|
||||
|
||||
static String _prefixFor(String namespace) =>
|
||||
namespace.isEmpty ? '' : '$namespace/';
|
||||
|
||||
/// Mints the ID token every request carries. Public so a settings screen
|
||||
/// can drive `signIn` / `signOut` through the same instance the client uses.
|
||||
final FirebaseTokenProvider auth;
|
||||
@ -47,8 +57,12 @@ class FirebaseRestClient implements RemoteStore, BulkMapReader {
|
||||
|
||||
Future<Uri> _uri(String path, {Map<String, String> query = const {}}) async {
|
||||
final encoded = encodePath(path);
|
||||
// A bare namespace (the root probe) must not leave a trailing slash.
|
||||
final full = encoded.isEmpty && _prefix.isNotEmpty
|
||||
? _prefix.substring(0, _prefix.length - 1)
|
||||
: '$_prefix$encoded';
|
||||
return Uri.parse(
|
||||
'$_databaseUrl/$encoded.json',
|
||||
'$_databaseUrl/$full.json',
|
||||
).replace(queryParameters: {'auth': await auth.idToken(), ...query});
|
||||
}
|
||||
|
||||
|
||||
@ -20,6 +20,7 @@ import 'firebase_credentials.dart';
|
||||
import 'firebase_client.dart';
|
||||
import 'mirror_store.dart';
|
||||
import 'remote_store.dart';
|
||||
import 'sync_namespace.dart';
|
||||
|
||||
/// The shared config is missing a field, or still holds a scaffold placeholder.
|
||||
class ConfigException implements Exception {
|
||||
@ -121,6 +122,8 @@ Future<FirebaseRestClient> firebaseClientFor({
|
||||
String? expectedUid,
|
||||
http.Client? httpClient,
|
||||
}) async {
|
||||
// Before any sign-in: a bad SYNC_NAMESPACE must stop the app at startup.
|
||||
checkSyncNamespace(syncNamespace);
|
||||
final auth = FirebaseTokenProvider(
|
||||
apiKey: config.apiKey,
|
||||
store: store,
|
||||
|
||||
@ -7,8 +7,11 @@ import 'store.dart';
|
||||
/// Writes atomically -- to a per-process temp file, then `rename`d over the
|
||||
/// real path -- so a concurrent reader (e.g. a background sync isolate writing
|
||||
/// while the foreground app also writes) never observes a half-written file.
|
||||
/// The pid in the temp name keeps two writers from colliding. This generalizes
|
||||
/// diet_guard's hand-rolled `LogStorageService` write scheme.
|
||||
/// The pid in the temp name keeps two processes from colliding; within one
|
||||
/// instance, writes are serialized in issue order (last issued wins), so
|
||||
/// overlapping calls (e.g. per-keystroke saves) never share the temp file
|
||||
/// mid-flight. This generalizes diet_guard's hand-rolled `LogStorageService`
|
||||
/// write scheme.
|
||||
///
|
||||
/// Lives behind the `package:crdt_sync/crdt_sync_io.dart` entrypoint, not the
|
||||
/// main barrel, because it imports `dart:io` and so is unavailable on web --
|
||||
@ -20,6 +23,10 @@ class FileLogPersistence implements LogPersistence {
|
||||
|
||||
final File _file;
|
||||
|
||||
/// Completes when the most recently issued write has settled. Never
|
||||
/// rejects, so one failed write cannot block the ones queued after it.
|
||||
Future<void> _tail = Future<void>.value();
|
||||
|
||||
@override
|
||||
Future<String?> read() async {
|
||||
if (!_file.existsSync()) return null;
|
||||
@ -31,7 +38,14 @@ class FileLogPersistence implements LogPersistence {
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> write(String text) async {
|
||||
Future<void> write(String text) {
|
||||
final result = _tail.then((_) => _writeNow(text));
|
||||
// The caller still sees its own error via [result]; the chain ignores it.
|
||||
_tail = result.then((_) {}, onError: (Object _) {});
|
||||
return result;
|
||||
}
|
||||
|
||||
Future<void> _writeNow(String text) async {
|
||||
await _file.parent.create(recursive: true);
|
||||
final tmp = File('${_file.path}.$pid.tmp');
|
||||
await tmp.writeAsString(text);
|
||||
|
||||
@ -2,6 +2,7 @@ import 'dart:convert';
|
||||
|
||||
import 'package:crypto/crypto.dart';
|
||||
|
||||
import 'e2e_cipher.dart';
|
||||
import 'log.dart';
|
||||
import 'record.dart';
|
||||
import 'remote_store.dart';
|
||||
@ -44,6 +45,9 @@ String revisionOf(String encodedLog) =>
|
||||
/// treated as this device's own rather than as a peer's, so its
|
||||
/// pre-migration log is not pulled back and re-merged every tick. Pass null
|
||||
/// once that path has been reclaimed.
|
||||
///
|
||||
/// [onPeerSkipped] reports each peer skipped as unreadable, e.g. an
|
||||
/// [E2eDecryptError] from a wrong key or an app that needs updating.
|
||||
Future<Log> syncLog({
|
||||
required RemoteStore client,
|
||||
required String deviceId,
|
||||
@ -56,6 +60,7 @@ Future<Log> syncLog({
|
||||
SyncStateStore? stateStore,
|
||||
String? revsPath,
|
||||
String? legacyDeviceId,
|
||||
void Function(String deviceId, Object error)? onPeerSkipped,
|
||||
}) async {
|
||||
final revs = revsPath ?? defaultRevsPath(pathPrefix);
|
||||
final state = await stateStore?.load() ?? const SyncState();
|
||||
@ -80,15 +85,21 @@ Future<Log> syncLog({
|
||||
seenRevs[otherDeviceId] = remoteRev;
|
||||
continue;
|
||||
}
|
||||
final text = await client.getFileText(
|
||||
'$pathPrefix/$otherDeviceId/$filename',
|
||||
);
|
||||
final String? text;
|
||||
try {
|
||||
text = await client.getFileText('$pathPrefix/$otherDeviceId/$filename');
|
||||
} on E2eDecryptError catch (error) {
|
||||
onPeerSkipped?.call(otherDeviceId, error);
|
||||
continue;
|
||||
}
|
||||
if (text == null) continue;
|
||||
try {
|
||||
mergedLog = mergeLogs(mergedLog, decode(text));
|
||||
} on FormatException {
|
||||
} on FormatException catch (error) {
|
||||
onPeerSkipped?.call(otherDeviceId, error);
|
||||
continue;
|
||||
} on TypeError {
|
||||
} on TypeError catch (error) {
|
||||
onPeerSkipped?.call(otherDeviceId, error);
|
||||
continue;
|
||||
}
|
||||
// Only remembered once the merge succeeded: a corrupt push must be
|
||||
|
||||
26
crdt_sync_dart/lib/src/sync_namespace.dart
Normal file
26
crdt_sync_dart/lib/src/sync_namespace.dart
Normal file
@ -0,0 +1,26 @@
|
||||
/// Compile-time sandbox namespace for the Realtime Database.
|
||||
///
|
||||
/// Build a test app with `--dart-define=SYNC_NAMESPACE=sandbox/slot3` and every
|
||||
/// RTDB path the library reads or writes moves under `/sandbox/slot3/`, so a
|
||||
/// test build can never touch real data. Sign-in is unchanged (real account).
|
||||
/// Empty, the default, leaves every path exactly as it was.
|
||||
library;
|
||||
|
||||
import 'firebase_config.dart';
|
||||
|
||||
/// The `SYNC_NAMESPACE` dart-define; empty means production paths.
|
||||
const String syncNamespace = String.fromEnvironment('SYNC_NAMESPACE');
|
||||
|
||||
final RegExp _allowed = RegExp(r'^sandbox/[a-z0-9_-]+$');
|
||||
|
||||
/// Returns [namespace] when it is empty or `sandbox/<slot>`, else throws.
|
||||
///
|
||||
/// Fails closed: a typo must stop the app at startup, never quietly fall back
|
||||
/// to production paths. A trailing newline is rejected too.
|
||||
String checkSyncNamespace(String namespace) {
|
||||
if (namespace.isEmpty || _allowed.hasMatch(namespace)) return namespace;
|
||||
throw ConfigException(
|
||||
'SYNC_NAMESPACE "$namespace" is invalid: it must be empty or match '
|
||||
r'^sandbox/[a-z0-9_-]+$ (e.g. sandbox/slot3)',
|
||||
);
|
||||
}
|
||||
@ -41,6 +41,14 @@ packages:
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "2.1.2"
|
||||
characters:
|
||||
dependency: transitive
|
||||
description:
|
||||
name: characters
|
||||
sha256: faf38497bda5ead2a8c7615f4f7939df04333478bf32e4173fcb06d428b5716b
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "1.4.1"
|
||||
cli_config:
|
||||
dependency: transitive
|
||||
description:
|
||||
@ -81,6 +89,22 @@ packages:
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "3.0.8"
|
||||
cryptography:
|
||||
dependency: "direct main"
|
||||
description:
|
||||
name: cryptography
|
||||
sha256: "3eda3029d34ec9095a27a198ac9785630fe525c0eb6a49f3d575272f8e792ef0"
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "2.9.0"
|
||||
ffi:
|
||||
dependency: transitive
|
||||
description:
|
||||
name: ffi
|
||||
sha256: "6d7fd89431262d8f3125e81b50d3847a091d846eafcd4fdb88dd06f36d705a45"
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "2.2.0"
|
||||
file:
|
||||
dependency: transitive
|
||||
description:
|
||||
@ -97,6 +121,14 @@ packages:
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "4.0.0"
|
||||
glados:
|
||||
dependency: "direct dev"
|
||||
description:
|
||||
name: glados
|
||||
sha256: c39446c0c4fe83a6ee6ee4fdee5a1afa27050619c7df390bf4c3b77b6f1acb33
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "1.1.7"
|
||||
glob:
|
||||
dependency: transitive
|
||||
description:
|
||||
|
||||
@ -2,7 +2,7 @@ name: crdt_sync
|
||||
description: >-
|
||||
Shared CRDT merge scheme + GitHub-Contents-API sync transport for
|
||||
cross-device apps. Pure Dart, no Flutter dependency.
|
||||
version: 0.11.3
|
||||
version: 0.14.1
|
||||
publish_to: 'none'
|
||||
|
||||
environment:
|
||||
@ -10,8 +10,10 @@ environment:
|
||||
|
||||
dependencies:
|
||||
crypto: 3.0.8
|
||||
cryptography: 2.9.0
|
||||
http: 1.6.0
|
||||
|
||||
dev_dependencies:
|
||||
glados: 1.1.7
|
||||
lints: 6.1.0
|
||||
test: 1.33.0
|
||||
|
||||
@ -40,9 +40,7 @@ void main() {
|
||||
// Nothing listens on this port, so the GET throws and the `on Exception`
|
||||
// path returns null -- while the `finally` still closes the client the
|
||||
// function created for itself.
|
||||
final account = await accountFromWrapper(
|
||||
Uri.parse('http://127.0.0.1:1'),
|
||||
);
|
||||
final account = await accountFromWrapper(Uri.parse('http://127.0.0.1:1'));
|
||||
expect(account, isNull);
|
||||
});
|
||||
|
||||
@ -50,8 +48,10 @@ void main() {
|
||||
final mock = http_testing.MockClient(
|
||||
(_) async => http.Response(jsonEncode({}), 404),
|
||||
);
|
||||
expect(await accountFromWrapper(Uri.parse('http://x'), client: mock),
|
||||
isNull);
|
||||
expect(
|
||||
await accountFromWrapper(Uri.parse('http://x'), client: mock),
|
||||
isNull,
|
||||
);
|
||||
// Still usable: the function must not close a client it did not create.
|
||||
expect((await mock.get(Uri.parse('http://x'))).statusCode, 404);
|
||||
});
|
||||
@ -106,27 +106,65 @@ void main() {
|
||||
expect(await store.read(), isNull);
|
||||
});
|
||||
|
||||
test('read returns null when the file cannot be read', () async {
|
||||
// A mode-000 file still passes existsSync, so the guard does not fire
|
||||
// and readAsString throws PathAccessException (a FileSystemException).
|
||||
// A directory would NOT work here: File.existsSync() is false for one,
|
||||
// so read() would return early down the guard path instead.
|
||||
final file = File('${dir.path}/unreadable.json')..writeAsStringSync('{}');
|
||||
final chmod = Process.runSync('chmod', ['000', file.path]);
|
||||
expect(chmod.exitCode, 0);
|
||||
addTearDown(() => Process.runSync('chmod', ['644', file.path]));
|
||||
test(
|
||||
'read returns null when the file cannot be read',
|
||||
() async {
|
||||
// A mode-000 file still passes existsSync, so the guard does not fire
|
||||
// and readAsString throws PathAccessException (a FileSystemException).
|
||||
// A directory would NOT work here: File.existsSync() is false for one,
|
||||
// so read() would return early down the guard path instead.
|
||||
final file = File('${dir.path}/unreadable.json')
|
||||
..writeAsStringSync('{}');
|
||||
final chmod = Process.runSync('chmod', ['000', file.path]);
|
||||
expect(chmod.exitCode, 0);
|
||||
addTearDown(() => Process.runSync('chmod', ['644', file.path]));
|
||||
|
||||
expect(await FileLogPersistence(file).read(), isNull);
|
||||
},
|
||||
// root ignores the permission bits, so the exception never fires.
|
||||
skip: Platform.environment['USER'] == 'root'
|
||||
? 'permission bits do not apply to root'
|
||||
: null);
|
||||
expect(await FileLogPersistence(file).read(), isNull);
|
||||
},
|
||||
// root ignores the permission bits, so the exception never fires.
|
||||
skip: Platform.environment['USER'] == 'root'
|
||||
? 'permission bits do not apply to root'
|
||||
: null,
|
||||
);
|
||||
|
||||
test('round-trips what it wrote', () async {
|
||||
final store = FileLogPersistence(File('${dir.path}/log.json'));
|
||||
await store.write('{"hello":"world"}');
|
||||
expect(await store.read(), '{"hello":"world"}');
|
||||
});
|
||||
|
||||
test(
|
||||
'50 overlapping writes all succeed and the last issued wins',
|
||||
() async {
|
||||
// Regression: every call shared one `<file>.<pid>.tmp`, so the first
|
||||
// rename moved it away and a later rename threw PathNotFoundException.
|
||||
final store = FileLogPersistence(File('${dir.path}/log.json'));
|
||||
await Future.wait([
|
||||
for (var i = 0; i < 50; i++) store.write('{"n":$i}'),
|
||||
]);
|
||||
expect(await store.read(), '{"n":49}');
|
||||
expect(dir.listSync().map((e) => e.uri.pathSegments.last), [
|
||||
'log.json',
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'a failed write rejects its caller but does not block later writes',
|
||||
() async {
|
||||
final target = File('${dir.path}/log.json');
|
||||
// A directory at the target path makes the rename fail.
|
||||
Directory(target.path).createSync();
|
||||
final store = FileLogPersistence(target);
|
||||
await expectLater(
|
||||
store.write('{"n":0}'),
|
||||
throwsA(isA<FileSystemException>()),
|
||||
);
|
||||
|
||||
Directory(target.path).deleteSync();
|
||||
await store.write('{"n":1}');
|
||||
expect(await store.read(), '{"n":1}');
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
56
crdt_sync_dart/test/e2e_audit_test.dart
Normal file
56
crdt_sync_dart/test/e2e_audit_test.dart
Normal file
@ -0,0 +1,56 @@
|
||||
/// The done check: find plaintext blobs in a raw RTDB dump.
|
||||
///
|
||||
/// RTDB returns each blob as a JSON string, so its quotes arrive escaped and
|
||||
/// `grep -c '"title"'` prints 0 even on plaintext. The audit decodes instead.
|
||||
library;
|
||||
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:crdt_sync/crdt_sync.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
// Shape copied from a live `GET /<root>.json` (scratch data).
|
||||
const _livePlaintextDump =
|
||||
r'{"notes":{"pc":{"log~2Ejson":"{\"n2\":{\"id\":\"n2\",\"fields\":'
|
||||
r'{\"title\":[\"call mum\",\"1970-01-01T00:00:01.001Z-0000-pc\"]},'
|
||||
r'\"deleted\":false,\"deleted_hlc\":null}}"}},"revs":{"pc":"ab12"}}';
|
||||
|
||||
void main() {
|
||||
test('the grep done check misses live plaintext; the audit does not', () {
|
||||
expect('"title"'.allMatches(_livePlaintextDump), isEmpty);
|
||||
expect(plaintextBlobPaths(jsonDecode(_livePlaintextDump)), [
|
||||
'notes/pc/log.json',
|
||||
]);
|
||||
});
|
||||
|
||||
test('sealed blobs, revs and the clear e2e header pass', () async {
|
||||
final sealed = await E2eCipher(
|
||||
List<int>.filled(32, 1),
|
||||
).seal('{"title":"x"}', path: 'notes/pc/log.json');
|
||||
final dump = {
|
||||
'e2e': '{"v":1,"kdf":"argon2id"}',
|
||||
'notes': {
|
||||
'pc': {'log~2Ejson': sealed},
|
||||
},
|
||||
'revs': {'pc': 'ab12'},
|
||||
};
|
||||
expect(plaintextBlobPaths(dump), isEmpty);
|
||||
});
|
||||
|
||||
test('every plaintext leaf is reported, with keys unescaped', () {
|
||||
final dump = {
|
||||
'a': {
|
||||
'x~2Ey': '[1]',
|
||||
'b': {'c': ' {"k":1}'},
|
||||
},
|
||||
'n': 3,
|
||||
'nil': null,
|
||||
};
|
||||
expect(plaintextBlobPaths(dump), unorderedEquals(['a/x.y', 'a/b/c']));
|
||||
});
|
||||
|
||||
test('a root that is itself plaintext, or empty, is handled', () {
|
||||
expect(plaintextBlobPaths('{"a":1}'), ['']);
|
||||
expect(plaintextBlobPaths(null), isEmpty);
|
||||
});
|
||||
}
|
||||
130
crdt_sync_dart/test/e2e_cipher_test.dart
Normal file
130
crdt_sync_dart/test/e2e_cipher_test.dart
Normal file
@ -0,0 +1,130 @@
|
||||
/// The record-blob AEAD: what makes the remote a blind relay.
|
||||
library;
|
||||
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:crdt_sync/crdt_sync.dart';
|
||||
import 'package:glados/glados.dart' hide expect, group, test, setUp;
|
||||
import 'package:test/test.dart';
|
||||
|
||||
const _path = 'todo-sync/notes/phone/log.json';
|
||||
|
||||
E2eCipher _cipher([int fill = 7, bool acceptPlaintext = true]) =>
|
||||
E2eCipher(List<int>.filled(32, fill), acceptPlaintext: acceptPlaintext);
|
||||
|
||||
void main() {
|
||||
Glados(any.letterOrDigits, ExploreConfig(numRuns: 60)).test(
|
||||
'open(seal(x)) == x for any text',
|
||||
(text) async {
|
||||
final cipher = _cipher();
|
||||
final sealed = await cipher.seal('{"t":"$text"}', path: _path);
|
||||
expect(await cipher.open(sealed, path: _path), '{"t":"$text"}');
|
||||
},
|
||||
);
|
||||
|
||||
test('sealed text contains no plaintext and is not JSON', () async {
|
||||
final sealed = await _cipher().seal('{"title":"secret"}', path: _path);
|
||||
expect(sealed, isNot(contains('title')));
|
||||
expect(sealed, isNot(contains('secret')));
|
||||
expect(() => jsonDecode(sealed), throwsFormatException);
|
||||
expect(E2eCipher.isPlaintext(sealed), isFalse);
|
||||
});
|
||||
|
||||
test('two seals of the same text differ (random nonce)', () async {
|
||||
final cipher = _cipher();
|
||||
expect(
|
||||
await cipher.seal('{}', path: _path),
|
||||
isNot(await cipher.seal('{}', path: _path)),
|
||||
);
|
||||
});
|
||||
|
||||
test('wire format starts with the version byte', () async {
|
||||
final bytes = base64.decode(await _cipher().seal('{}', path: _path));
|
||||
expect(bytes.first, e2eFormatVersion);
|
||||
expect(bytes.length, 1 + 24 + 2 + 16);
|
||||
});
|
||||
|
||||
test('plaintext JSON passes through during migration', () async {
|
||||
expect(await _cipher().open(' {"a":1}', path: _path), ' {"a":1}');
|
||||
expect(await _cipher().open('[1]', path: _path), '[1]');
|
||||
expect(E2eCipher.isPlaintext('\n{'), isTrue);
|
||||
});
|
||||
|
||||
test('plaintext is refused once migration is over', () async {
|
||||
expect(
|
||||
() => _cipher(7, false).open('{"a":1}', path: _path),
|
||||
throwsA(isA<E2eDecryptError>()),
|
||||
);
|
||||
});
|
||||
|
||||
group('rejects, as a FormatException so syncLog skips the peer', () {
|
||||
test('a wrong key', () async {
|
||||
final sealed = await _cipher(1).seal('{}', path: _path);
|
||||
expect(
|
||||
() => _cipher(2).open(sealed, path: _path),
|
||||
throwsA(isA<FormatException>()),
|
||||
);
|
||||
});
|
||||
|
||||
test('a blob moved to another path', () async {
|
||||
final sealed = await _cipher().seal('{}', path: _path);
|
||||
expect(
|
||||
() => _cipher().open(sealed, path: 'todo-sync/notes/pc/log.json'),
|
||||
throwsA(isA<E2eDecryptError>()),
|
||||
);
|
||||
});
|
||||
|
||||
test('a flipped ciphertext byte', () async {
|
||||
final bytes = base64.decode(await _cipher().seal('{}', path: _path));
|
||||
bytes[30] ^= 1;
|
||||
expect(
|
||||
() => _cipher().open(base64.encode(bytes), path: _path),
|
||||
throwsA(isA<E2eDecryptError>()),
|
||||
);
|
||||
});
|
||||
|
||||
test('an unknown version byte', () async {
|
||||
final bytes = base64.decode(await _cipher().seal('{}', path: _path));
|
||||
bytes[0] = 9;
|
||||
expect(
|
||||
() => _cipher().open(base64.encode(bytes), path: _path),
|
||||
throwsA(
|
||||
isA<E2eDecryptError>().having(
|
||||
(e) => e.message,
|
||||
'message',
|
||||
contains('update the app'),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('text that is not base64', () {
|
||||
expect(
|
||||
() => _cipher().open('not base64 !!', path: _path),
|
||||
throwsA(isA<E2eDecryptError>()),
|
||||
);
|
||||
});
|
||||
|
||||
test('a blob too short to hold nonce and tag', () {
|
||||
expect(
|
||||
() => _cipher().open(base64.encode([1, 2, 3]), path: _path),
|
||||
throwsA(isA<E2eDecryptError>()),
|
||||
);
|
||||
});
|
||||
|
||||
test('an empty blob', () {
|
||||
expect(
|
||||
() => _cipher().open('', path: _path),
|
||||
throwsA(isA<E2eDecryptError>()),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('a key that is not 32 bytes is a programming error', () {
|
||||
expect(() => E2eCipher(const [1, 2]), throwsArgumentError);
|
||||
});
|
||||
|
||||
test('E2eDecryptError reads as itself', () {
|
||||
expect(E2eDecryptError('x').toString(), 'E2eDecryptError: x');
|
||||
});
|
||||
}
|
||||
156
crdt_sync_dart/test/e2e_envelope_test.dart
Normal file
156
crdt_sync_dart/test/e2e_envelope_test.dart
Normal file
@ -0,0 +1,156 @@
|
||||
/// Per-record envelope: ids and HLCs stay clear, field names and values are
|
||||
/// opaque, and merge results are identical with and without encryption.
|
||||
library;
|
||||
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:crdt_sync/crdt_sync.dart';
|
||||
import 'package:glados/glados.dart' hide expect, group, test, setUp;
|
||||
import 'package:test/test.dart';
|
||||
|
||||
import 'e2e_fakes.dart';
|
||||
|
||||
const _path = 'app/notes-e2e/phone/log.json';
|
||||
|
||||
E2eCipher _cipher([int fill = 7, bool acceptPlaintext = true]) =>
|
||||
E2eCipher(List<int>.filled(32, fill), acceptPlaintext: acceptPlaintext);
|
||||
|
||||
Hlc _hlc(int ms, int counter, String node) =>
|
||||
Hlc(wallTimeMs: ms, counter: counter, nodeId: node);
|
||||
|
||||
Log _logFrom(List<(int, int, String)> rows) {
|
||||
final log = <String, Record>{};
|
||||
for (final (id, field, value) in rows) {
|
||||
final key = 'r$id';
|
||||
final old = log[key];
|
||||
log[key] = Record(
|
||||
id: key,
|
||||
fields: {
|
||||
...?old?.fields,
|
||||
'f$field': (value, _hlc(1000 + id * 7 + field, field, 'n$id')),
|
||||
},
|
||||
deleted: id % 4 == 0,
|
||||
deletedHlc: id % 4 == 0 ? _hlc(5000, 0, 'n$id') : null,
|
||||
);
|
||||
}
|
||||
return log;
|
||||
}
|
||||
|
||||
final _rows = any.list(
|
||||
any.combine3(
|
||||
any.intInRange(0, 6),
|
||||
any.intInRange(0, 4),
|
||||
any.letterOrDigits,
|
||||
(a, b, c) => (a, b, c),
|
||||
),
|
||||
);
|
||||
|
||||
void main() {
|
||||
Glados(_rows, ExploreConfig(numRuns: 40)).test(
|
||||
'decode(open(seal(encode(log)))) == log',
|
||||
(rows) async {
|
||||
final log = _logFrom(rows);
|
||||
final cipher = _cipher();
|
||||
final sealed = await sealLogText(encodeLog(log), cipher, path: _path);
|
||||
final opened = await openLogText(sealed, cipher, path: _path);
|
||||
expect(encodeLog(decodeLog(opened)), encodeLog(log));
|
||||
},
|
||||
);
|
||||
|
||||
Glados2(_rows, _rows, ExploreConfig(numRuns: 40)).test(
|
||||
'merge through the envelope == merge without it',
|
||||
(a, b) async {
|
||||
final cipher = _cipher();
|
||||
Future<Log> viaCipher(Log log) async => decodeLog(
|
||||
await openLogText(
|
||||
await sealLogText(encodeLog(log), cipher, path: _path),
|
||||
cipher,
|
||||
path: _path,
|
||||
),
|
||||
);
|
||||
final la = _logFrom(a);
|
||||
final lb = _logFrom(b);
|
||||
final plain = mergeLogs(la, lb);
|
||||
final sealed = mergeLogs(await viaCipher(la), await viaCipher(lb));
|
||||
expect(encodeLog(sealed), encodeLog(plain));
|
||||
},
|
||||
);
|
||||
|
||||
test('ids and HLCs are clear, names and values are not', () async {
|
||||
final log = logOf({'n1': 'buy milk'});
|
||||
final sealed = await sealLogText(encodeLog(log), _cipher(), path: _path);
|
||||
final json = jsonDecode(sealed) as Map<String, dynamic>;
|
||||
expect(json['v'], 1);
|
||||
final rec = (json['r'] as Map<String, dynamic>)['n1'] as Map;
|
||||
expect(rec['h'], log['n1']!.fields['title']!.$2.toStr());
|
||||
expect(rec['d'], false);
|
||||
expect(sealed, isNot(contains('title')));
|
||||
expect(sealed, isNot(contains('buy milk')));
|
||||
expect(isE2eEnvelope(sealed), isTrue);
|
||||
expect(isE2eEnvelope(encodeLog(log)), isFalse);
|
||||
});
|
||||
|
||||
test('legacy plaintext passes only while acceptPlaintext', () async {
|
||||
final text = encodeLog(logOf({'n1': 'x'}));
|
||||
expect(await openLogText(text, _cipher(), path: _path), text);
|
||||
expect(
|
||||
() => openLogText(text, _cipher(7, false), path: _path),
|
||||
throwsA(isA<E2eDecryptError>()),
|
||||
);
|
||||
});
|
||||
|
||||
test('a record id named v is still legacy plaintext', () async {
|
||||
final text = encodeLog(logOf({'v': 'x'}));
|
||||
expect(isE2eEnvelope(text), isFalse);
|
||||
});
|
||||
|
||||
group('tamper and wrong key are E2eDecryptError', () {
|
||||
Future<String> sealed() async =>
|
||||
sealLogText(encodeLog(logOf({'n1': 'x'})), _cipher(), path: _path);
|
||||
|
||||
test('wrong key', () async {
|
||||
final text = await sealed();
|
||||
expect(
|
||||
() => openLogText(text, _cipher(8), path: _path),
|
||||
throwsA(isA<E2eDecryptError>()),
|
||||
);
|
||||
});
|
||||
test('moved path', () async {
|
||||
final text = await sealed();
|
||||
expect(
|
||||
() => openLogText(text, _cipher(), path: '$_path-x'),
|
||||
throwsA(isA<E2eDecryptError>()),
|
||||
);
|
||||
});
|
||||
test('flipped clear deleted flag', () async {
|
||||
final json = jsonDecode(await sealed()) as Map<String, dynamic>;
|
||||
((json['r'] as Map)['n1'] as Map)['d'] = true;
|
||||
expect(
|
||||
() => openLogText(jsonEncode(json), _cipher(), path: _path),
|
||||
throwsA(isA<E2eDecryptError>()),
|
||||
);
|
||||
});
|
||||
test('record swapped to another id', () async {
|
||||
final json = jsonDecode(await sealed()) as Map<String, dynamic>;
|
||||
final r = json['r'] as Map;
|
||||
r['n2'] = r.remove('n1');
|
||||
expect(
|
||||
() => openLogText(jsonEncode(json), _cipher(), path: _path),
|
||||
throwsA(isA<E2eDecryptError>()),
|
||||
);
|
||||
});
|
||||
test('mis-shaped envelope', () async {
|
||||
expect(
|
||||
() => openLogText('{"v":1,"r":5}', _cipher(), path: _path),
|
||||
throwsA(isA<E2eDecryptError>()),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('text that is not a log is sealed whole and opens back', () async {
|
||||
final cipher = _cipher();
|
||||
final sealed = await sealLogText('{"n":{"id":1}}', cipher, path: _path);
|
||||
expect(sealed, isNot(contains('"id"')));
|
||||
expect(await openLogText(sealed, cipher, path: _path), '{"n":{"id":1}}');
|
||||
});
|
||||
}
|
||||
111
crdt_sync_dart/test/e2e_fakes.dart
Normal file
111
crdt_sync_dart/test/e2e_fakes.dart
Normal file
@ -0,0 +1,111 @@
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:crdt_sync/crdt_sync.dart';
|
||||
|
||||
const cheapArgon = Argon2Params(memoryKiB: 8, iterations: 1, parallelism: 1);
|
||||
|
||||
Log logOf(Map<String, String> titles, {String nodeId = 'node-a'}) => {
|
||||
for (final entry in titles.entries)
|
||||
entry.key: Record(
|
||||
id: entry.key,
|
||||
fields: {
|
||||
'title': (
|
||||
entry.value,
|
||||
Hlc(wallTimeMs: 1000, counter: 0, nodeId: nodeId),
|
||||
),
|
||||
},
|
||||
),
|
||||
};
|
||||
|
||||
String encodeLog(Log log) =>
|
||||
jsonEncode(log.map((id, record) => MapEntry(id, record.toJson())));
|
||||
|
||||
Log decodeLog(String text) => (jsonDecode(text) as Map<String, dynamic>).map(
|
||||
(id, data) => MapEntry(id, Record.fromJson(data as Map<String, dynamic>)),
|
||||
);
|
||||
|
||||
/// In-memory remote; [bulk] toggles the [BulkMapReader] capability.
|
||||
class MemoryRemote implements RemoteStore, BulkMapReader {
|
||||
MemoryRemote([Map<String, String>? files]) : files = files ?? {};
|
||||
|
||||
final Map<String, String> files;
|
||||
final List<String> writes = [];
|
||||
final List<String> reads = [];
|
||||
bool closed = false;
|
||||
|
||||
/// Overrides the next reads of a path, oldest first, to fake a race.
|
||||
final Map<String, List<String?>> scriptedReads = {};
|
||||
|
||||
@override
|
||||
Future<List<String>> listDirectory(String path) async => files.keys
|
||||
.where((key) => key.startsWith('$path/'))
|
||||
.map((key) => key.substring(path.length + 1).split('/').first)
|
||||
.toSet()
|
||||
.toList();
|
||||
|
||||
@override
|
||||
Future<String?> getFileText(String path) async {
|
||||
reads.add(path);
|
||||
final script = scriptedReads[path];
|
||||
if (script != null && script.isNotEmpty) return script.removeAt(0);
|
||||
return files[path];
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> putFileText(
|
||||
String path,
|
||||
String text, {
|
||||
required String message,
|
||||
}) async {
|
||||
writes.add(path);
|
||||
files[path] = text;
|
||||
}
|
||||
|
||||
@override
|
||||
Future<Map<String, String>> getStringMap(String path) async => {
|
||||
for (final entry in files.entries)
|
||||
if (entry.key.startsWith('$path/') &&
|
||||
!entry.key.substring(path.length + 1).contains('/'))
|
||||
entry.key.substring(path.length + 1): entry.value,
|
||||
};
|
||||
|
||||
@override
|
||||
Future<void> deleteFile(String path, {String message = ''}) async =>
|
||||
files.remove(path);
|
||||
|
||||
@override
|
||||
Future<bool> canAccessRemote() async => true;
|
||||
|
||||
@override
|
||||
void close() => closed = true;
|
||||
}
|
||||
|
||||
/// Same storage without the bulk capability, like `GitHubClient`.
|
||||
class PlainRemote implements RemoteStore {
|
||||
PlainRemote(this.inner);
|
||||
|
||||
final MemoryRemote inner;
|
||||
|
||||
@override
|
||||
Future<List<String>> listDirectory(String path) => inner.listDirectory(path);
|
||||
|
||||
@override
|
||||
Future<String?> getFileText(String path) => inner.getFileText(path);
|
||||
|
||||
@override
|
||||
Future<void> putFileText(
|
||||
String path,
|
||||
String text, {
|
||||
required String message,
|
||||
}) => inner.putFileText(path, text, message: message);
|
||||
|
||||
@override
|
||||
Future<void> deleteFile(String path, {String message = ''}) =>
|
||||
inner.deleteFile(path);
|
||||
|
||||
@override
|
||||
Future<bool> canAccessRemote() => inner.canAccessRemote();
|
||||
|
||||
@override
|
||||
void close() => inner.close();
|
||||
}
|
||||
45
crdt_sync_dart/test/e2e_interop_test.dart
Normal file
45
crdt_sync_dart/test/e2e_interop_test.dart
Normal file
@ -0,0 +1,45 @@
|
||||
/// Cross-language golden vectors: Dart opens what the Python port sealed.
|
||||
library;
|
||||
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:crdt_sync/crdt_sync.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
import 'e2e_fakes.dart';
|
||||
|
||||
void main() {
|
||||
final vec =
|
||||
jsonDecode(File('test/fixtures/e2e_py_vectors.json').readAsStringSync())
|
||||
as Map<String, dynamic>;
|
||||
|
||||
Future<E2eKey> key() => unlockE2e(
|
||||
store: MemoryRemote({'vec/e2e': vec['header'] as String}),
|
||||
pathPrefix: 'vec/notes-e2e',
|
||||
passphrase: vec['passphrase'] as String,
|
||||
e2ePath: 'vec/e2e',
|
||||
);
|
||||
|
||||
test('Argon2id and HKDF match Python', () async {
|
||||
final hex = (await key()).bytes
|
||||
.map((b) => b.toRadixString(16).padLeft(2, '0'))
|
||||
.join();
|
||||
expect(hex, vec['master_hex']);
|
||||
});
|
||||
|
||||
test('opens a Python envelope and whole blob', () async {
|
||||
final cipher = (await key()).cipher();
|
||||
final path = vec['path'] as String;
|
||||
final opened = await openLogText(
|
||||
vec['envelope'] as String,
|
||||
cipher,
|
||||
path: path,
|
||||
);
|
||||
expect(jsonDecode(opened), jsonDecode(vec['log'] as String));
|
||||
expect(
|
||||
await openLogText(vec['whole'] as String, cipher, path: path),
|
||||
'not a log',
|
||||
);
|
||||
});
|
||||
}
|
||||
181
crdt_sync_dart/test/e2e_key_test.dart
Normal file
181
crdt_sync_dart/test/e2e_key_test.dart
Normal file
@ -0,0 +1,181 @@
|
||||
/// Passphrase -> key, the clear header beside the data, and the open gate.
|
||||
library;
|
||||
|
||||
import 'dart:convert';
|
||||
import 'dart:math';
|
||||
|
||||
import 'package:crdt_sync/crdt_sync.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
import 'e2e_fakes.dart';
|
||||
|
||||
const _prefix = 'todo-sync/notes';
|
||||
const _headerPath = 'todo-sync/e2e';
|
||||
|
||||
Future<E2eKey> _unlock(MemoryRemote remote, String passphrase) => unlockE2e(
|
||||
store: remote,
|
||||
pathPrefix: _prefix,
|
||||
passphrase: passphrase,
|
||||
params: cheapArgon,
|
||||
random: Random(1),
|
||||
);
|
||||
|
||||
void main() {
|
||||
test('the header sits beside the device dirs, not among them', () {
|
||||
expect(defaultE2ePath(_prefix), _headerPath);
|
||||
expect(defaultE2ePath('wake-alarm'), 'wake-alarm/e2e');
|
||||
});
|
||||
|
||||
test(
|
||||
'first unlock writes a clear header with salt, params and check',
|
||||
() async {
|
||||
final remote = MemoryRemote();
|
||||
await _unlock(remote, 'correct horse');
|
||||
final header = jsonDecode(remote.files[_headerPath]!) as Map;
|
||||
expect(header['v'], 1);
|
||||
expect(header['kdf'], 'argon2id');
|
||||
expect(base64.decode(header['salt'] as String), hasLength(16));
|
||||
expect(header['m'], 8);
|
||||
expect(header['t'], 1);
|
||||
expect(header['p'], 1);
|
||||
expect(header['check'], isA<String>());
|
||||
expect(remote.files[_headerPath], isNot(contains('correct horse')));
|
||||
},
|
||||
);
|
||||
|
||||
test('the same passphrase on another device yields the same key', () async {
|
||||
final remote = MemoryRemote();
|
||||
final a = await _unlock(remote, 'pw');
|
||||
final b = await unlockE2e(
|
||||
store: remote,
|
||||
pathPrefix: _prefix,
|
||||
passphrase: 'pw',
|
||||
random: Random(99),
|
||||
);
|
||||
expect(b.bytes, a.bytes);
|
||||
expect(remote.writes, [_headerPath]);
|
||||
});
|
||||
|
||||
test('a wrong passphrase is refused before anything is written', () async {
|
||||
final remote = MemoryRemote();
|
||||
await _unlock(remote, 'pw');
|
||||
remote.writes.clear();
|
||||
await expectLater(
|
||||
_unlock(remote, 'not pw'),
|
||||
throwsA(isA<E2eWrongPassphraseError>()),
|
||||
);
|
||||
expect(remote.writes, isEmpty);
|
||||
});
|
||||
|
||||
test('a setup race derives from the header that won', () async {
|
||||
final winner = MemoryRemote();
|
||||
await unlockE2e(
|
||||
store: winner,
|
||||
pathPrefix: _prefix,
|
||||
passphrase: 'pw',
|
||||
params: cheapArgon,
|
||||
random: Random(5),
|
||||
);
|
||||
final remote = MemoryRemote()
|
||||
..scriptedReads[_headerPath] = [null, winner.files[_headerPath]];
|
||||
final key = await _unlock(remote, 'pw');
|
||||
final expected = await _unlock(winner, 'pw');
|
||||
expect(key.bytes, expected.bytes);
|
||||
});
|
||||
|
||||
test('an empty passphrase is refused', () {
|
||||
expect(() => _unlock(MemoryRemote(), ''), throwsArgumentError);
|
||||
});
|
||||
|
||||
test('a corrupt or newer header is refused, never overwritten', () async {
|
||||
for (final bad in ['nope', '[]', '{"v":2}', '{"v":1,"kdf":"scrypt"}']) {
|
||||
final remote = MemoryRemote({_headerPath: bad});
|
||||
await expectLater(
|
||||
_unlock(remote, 'pw'),
|
||||
throwsA(isA<E2eHeaderError>()),
|
||||
reason: bad,
|
||||
);
|
||||
expect(remote.files[_headerPath], bad);
|
||||
}
|
||||
});
|
||||
|
||||
test('E2eKey.fromBytes round-trips a cached key', () async {
|
||||
final key = await _unlock(MemoryRemote(), 'pw');
|
||||
final cached = await E2eKey.fromBytes(
|
||||
base64.decode(base64.encode(key.bytes)),
|
||||
);
|
||||
expect(cached.bytes, key.bytes);
|
||||
expect(await cached.revision('x'), await key.revision('x'));
|
||||
expect(() => E2eKey.fromBytes(const [1]), throwsArgumentError);
|
||||
});
|
||||
|
||||
test('revision is keyed: not sha256, and differs across keys', () async {
|
||||
final a = await _unlock(MemoryRemote(), 'a');
|
||||
final b = await _unlock(MemoryRemote(), 'b');
|
||||
expect(await a.revision('{}'), isNot(revisionOf('{}')));
|
||||
expect(await a.revision('{}'), isNot(await b.revision('{}')));
|
||||
expect(await a.revision('{}'), await a.revision('{}'));
|
||||
});
|
||||
|
||||
test('errors read as themselves', () {
|
||||
expect(E2eWrongPassphraseError().toString(), contains('passphrase'));
|
||||
expect(E2eHeaderError('x').toString(), 'E2eHeaderError: x');
|
||||
});
|
||||
|
||||
group('openE2e', () {
|
||||
test('no header and no key: off, plaintext sync allowed', () async {
|
||||
final state = await openE2e(store: MemoryRemote(), pathPrefix: _prefix);
|
||||
expect(state, isA<E2eOff>());
|
||||
});
|
||||
|
||||
test('header but no key: locked, needs the passphrase', () async {
|
||||
final remote = MemoryRemote();
|
||||
await _unlock(remote, 'pw');
|
||||
final state = await openE2e(store: remote, pathPrefix: _prefix);
|
||||
expect(state, isA<E2eLocked>());
|
||||
expect((state as E2eLocked).reason, E2eLockReason.needsPassphrase);
|
||||
});
|
||||
|
||||
test('a cached key the header rejects: locked, wrong key', () async {
|
||||
final remote = MemoryRemote();
|
||||
await _unlock(remote, 'pw');
|
||||
final stale = await _unlock(MemoryRemote(), 'old');
|
||||
final state = await openE2e(
|
||||
store: remote,
|
||||
pathPrefix: _prefix,
|
||||
key: stale,
|
||||
);
|
||||
expect((state as E2eLocked).reason, E2eLockReason.wrongKey);
|
||||
});
|
||||
|
||||
test('a cached key with the header gone: locked, wrong key', () async {
|
||||
final key = await _unlock(MemoryRemote(), 'pw');
|
||||
final state = await openE2e(
|
||||
store: MemoryRemote(),
|
||||
pathPrefix: _prefix,
|
||||
key: key,
|
||||
);
|
||||
expect((state as E2eLocked).reason, E2eLockReason.wrongKey);
|
||||
});
|
||||
|
||||
test('a corrupt header: locked, unreadable header', () async {
|
||||
final state = await openE2e(
|
||||
store: MemoryRemote({_headerPath: 'garbage'}),
|
||||
pathPrefix: _prefix,
|
||||
);
|
||||
expect((state as E2eLocked).reason, E2eLockReason.badHeader);
|
||||
});
|
||||
|
||||
test('a matching key: ready with an encrypting store', () async {
|
||||
final remote = MemoryRemote();
|
||||
final key = await _unlock(remote, 'pw');
|
||||
final state = await openE2e(store: remote, pathPrefix: _prefix, key: key);
|
||||
expect((state as E2eReady).store, isA<EncryptingStore>());
|
||||
});
|
||||
});
|
||||
|
||||
test('recommended params meet the OWASP Argon2id floor', () {
|
||||
expect(Argon2Params.recommended.memoryKiB, greaterThanOrEqualTo(19456));
|
||||
expect(Argon2Params.recommended.iterations, greaterThanOrEqualTo(2));
|
||||
});
|
||||
}
|
||||
221
crdt_sync_dart/test/e2e_migration_test.dart
Normal file
221
crdt_sync_dart/test/e2e_migration_test.dart
Normal file
@ -0,0 +1,221 @@
|
||||
/// One-time copy from the plaintext prefix to `<prefix>-e2e`.
|
||||
library;
|
||||
|
||||
import 'package:crdt_sync/crdt_sync.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
import 'e2e_fakes.dart';
|
||||
|
||||
const _old = 'app/notes';
|
||||
|
||||
Future<E2eKey> _key(MemoryRemote remote, E2eLayout layout) => unlockE2e(
|
||||
store: remote,
|
||||
pathPrefix: layout.prefix,
|
||||
passphrase: 'pw',
|
||||
params: cheapArgon,
|
||||
e2ePath: layout.e2ePath,
|
||||
);
|
||||
|
||||
MemoryRemote _plainRemote() => MemoryRemote({
|
||||
'$_old/phone/log.json': encodeLog(logOf({'a': 'milk', 'b': 'eggs'})),
|
||||
'$_old/pc/log.json': encodeLog(logOf({'c': 'tea'}, nodeId: 'node-b')),
|
||||
});
|
||||
|
||||
void main() {
|
||||
final layout = E2eLayout.of(_old);
|
||||
|
||||
test('layout puts ciphertext on a new prefix, revs apart from old', () {
|
||||
expect(layout.prefix, 'app/notes-e2e');
|
||||
expect(layout.revsPath, isNot(defaultRevsPath(_old)));
|
||||
expect(layout.e2ePath, startsWith('app/'));
|
||||
});
|
||||
|
||||
test('copy seals every device log under the new prefix', () async {
|
||||
final remote = _plainRemote();
|
||||
final key = await _key(remote, layout);
|
||||
final report = await copyToEncrypted(
|
||||
store: remote,
|
||||
key: key,
|
||||
fromPrefix: _old,
|
||||
layout: layout,
|
||||
);
|
||||
expect(report.copied, 2);
|
||||
final blob = remote.files['app/notes-e2e/phone/log.json']!;
|
||||
expect(blob, isNot(contains('milk')));
|
||||
expect(blob, isNot(contains('"title"')));
|
||||
expect(remote.files['$_old/phone/log.json'], contains('milk'));
|
||||
final enc = EncryptingStore(
|
||||
remote,
|
||||
key,
|
||||
pathPrefix: layout.prefix,
|
||||
revsPath: layout.revsPath,
|
||||
e2ePath: layout.e2ePath,
|
||||
);
|
||||
final merged = await syncLog(
|
||||
client: enc,
|
||||
deviceId: 'tablet',
|
||||
pathPrefix: layout.prefix,
|
||||
localLog: {},
|
||||
encode: encodeLog,
|
||||
decode: decodeLog,
|
||||
);
|
||||
expect(merged.keys, unorderedEquals(['a', 'b', 'c']));
|
||||
});
|
||||
|
||||
test('copying twice adds no duplicate and loses nothing', () async {
|
||||
final remote = _plainRemote();
|
||||
final key = await _key(remote, layout);
|
||||
Future<MigrationReport> run() => copyToEncrypted(
|
||||
store: remote,
|
||||
key: key,
|
||||
fromPrefix: _old,
|
||||
layout: layout,
|
||||
);
|
||||
await run();
|
||||
final first = Map.of(remote.files);
|
||||
remote.writes.clear();
|
||||
final second = await run();
|
||||
expect(second.copied, 0);
|
||||
expect(remote.writes, isEmpty);
|
||||
expect(remote.files, first);
|
||||
});
|
||||
|
||||
test('a later plaintext write by an old build is copied again', () async {
|
||||
final remote = _plainRemote();
|
||||
final key = await _key(remote, layout);
|
||||
await copyToEncrypted(
|
||||
store: remote,
|
||||
key: key,
|
||||
fromPrefix: _old,
|
||||
layout: layout,
|
||||
);
|
||||
remote.files['$_old/pc/log.json'] = encodeLog(
|
||||
logOf({'c': 'tea', 'd': 'jam'}, nodeId: 'node-b'),
|
||||
);
|
||||
final report = await copyToEncrypted(
|
||||
store: remote,
|
||||
key: key,
|
||||
fromPrefix: _old,
|
||||
layout: layout,
|
||||
);
|
||||
expect(report.copied, 1);
|
||||
expect(
|
||||
await verifyCopy(
|
||||
store: remote,
|
||||
key: key,
|
||||
fromPrefix: _old,
|
||||
layout: layout,
|
||||
),
|
||||
isTrue,
|
||||
);
|
||||
});
|
||||
|
||||
test('cleanup refuses before the copy is verified, then deletes', () async {
|
||||
final remote = _plainRemote();
|
||||
final key = await _key(remote, layout);
|
||||
expect(
|
||||
await deletePlaintextPrefix(
|
||||
store: remote,
|
||||
key: key,
|
||||
fromPrefix: _old,
|
||||
layout: layout,
|
||||
),
|
||||
isFalse,
|
||||
);
|
||||
expect(remote.files['$_old/phone/log.json'], isNotNull);
|
||||
await copyToEncrypted(
|
||||
store: remote,
|
||||
key: key,
|
||||
fromPrefix: _old,
|
||||
layout: layout,
|
||||
);
|
||||
expect(
|
||||
await deletePlaintextPrefix(
|
||||
store: remote,
|
||||
key: key,
|
||||
fromPrefix: _old,
|
||||
layout: layout,
|
||||
),
|
||||
isTrue,
|
||||
);
|
||||
expect(remote.files.keys.where((k) => k.startsWith('$_old/')), isEmpty);
|
||||
expect(remote.files['app/notes-e2e/phone/log.json'], isNotNull);
|
||||
});
|
||||
|
||||
test('a corrupt source blob is skipped and blocks cleanup', () async {
|
||||
final remote = _plainRemote();
|
||||
remote.files['$_old/bad/log.json'] = '{oops';
|
||||
final key = await _key(remote, layout);
|
||||
final report = await copyToEncrypted(
|
||||
store: remote,
|
||||
key: key,
|
||||
fromPrefix: _old,
|
||||
layout: layout,
|
||||
);
|
||||
expect(report.copied, 2);
|
||||
expect(report.skipped, ['bad/log.json']);
|
||||
expect(
|
||||
await deletePlaintextPrefix(
|
||||
store: remote,
|
||||
key: key,
|
||||
fromPrefix: _old,
|
||||
layout: layout,
|
||||
),
|
||||
isFalse,
|
||||
);
|
||||
});
|
||||
|
||||
group('slashless prefix (settings-sync)', () {
|
||||
final flat = E2eLayout.of('settings-sync');
|
||||
|
||||
test('revs and header stay inside the app node', () {
|
||||
expect(flat.prefix, 'settings-sync-e2e');
|
||||
expect(flat.revsPath, 'settings-sync-e2e/revs');
|
||||
expect(flat.e2ePath, 'settings-sync-e2e/e2e');
|
||||
});
|
||||
|
||||
test('old revs and header are not copied as logs', () async {
|
||||
final remote = MemoryRemote({
|
||||
'settings-sync/phone/log.json': encodeLog(logOf({'a': 'x'})),
|
||||
'settings-sync/revs/phone': 'abc',
|
||||
'settings-sync/e2e': '{"v":1}',
|
||||
});
|
||||
final key = await _key(remote, flat);
|
||||
final report = await copyToEncrypted(
|
||||
store: remote,
|
||||
key: key,
|
||||
fromPrefix: 'settings-sync',
|
||||
layout: flat,
|
||||
);
|
||||
expect(report.copied, 1);
|
||||
expect(report.skipped, isEmpty);
|
||||
expect(
|
||||
await verifyCopy(
|
||||
store: remote,
|
||||
key: key,
|
||||
fromPrefix: 'settings-sync',
|
||||
layout: flat,
|
||||
),
|
||||
isTrue,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('never overwrites a non-log blob and skips empty sources', () async {
|
||||
final remote = MemoryRemote({
|
||||
'app/notes/pc/current.json': '{}',
|
||||
'app/notes-e2e/pc/current.json': '{"photos":["a"]}',
|
||||
'app/notes/tab/current.json': '{}',
|
||||
});
|
||||
final key = await _key(remote, layout);
|
||||
final report = await copyToEncrypted(
|
||||
store: remote,
|
||||
key: key,
|
||||
fromPrefix: _old,
|
||||
layout: layout,
|
||||
);
|
||||
expect(report.copied, 0);
|
||||
expect(remote.files['app/notes-e2e/pc/current.json'], '{"photos":["a"]}');
|
||||
expect(remote.files.containsKey('app/notes-e2e/tab/current.json'), isFalse);
|
||||
});
|
||||
}
|
||||
98
crdt_sync_dart/test/e2e_open_cache_test.dart
Normal file
98
crdt_sync_dart/test/e2e_open_cache_test.dart
Normal file
@ -0,0 +1,98 @@
|
||||
/// openEncryptedStore must not pay Argon2id or a full copy on every tick.
|
||||
library;
|
||||
|
||||
import 'package:crdt_sync/crdt_sync.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
import 'e2e_fakes.dart';
|
||||
|
||||
const _old = 'app/notes';
|
||||
|
||||
void main() {
|
||||
setUp(resetE2eCopyMemo);
|
||||
|
||||
test('a cached key skips Argon2 and still unlocks', () async {
|
||||
final remote = MemoryRemote();
|
||||
List<int>? cached;
|
||||
var derived = 0;
|
||||
Future<E2eState> open() => openEncryptedStore(
|
||||
store: remote,
|
||||
oldPrefix: _old,
|
||||
readPassphrase: () async => 'pw',
|
||||
params: cheapArgon,
|
||||
keyCache: E2eKeyCache(
|
||||
read: () async => cached,
|
||||
write: (bytes) async => cached = bytes,
|
||||
),
|
||||
onDerive: () => derived++,
|
||||
);
|
||||
expect(await open(), isA<E2eReady>());
|
||||
expect(cached, hasLength(32));
|
||||
expect(await open(), isA<E2eReady>());
|
||||
expect(derived, 1);
|
||||
});
|
||||
|
||||
test('a stale cached key falls back to the passphrase', () async {
|
||||
final remote = MemoryRemote();
|
||||
List<int>? cached = List<int>.filled(32, 9);
|
||||
final state = await openEncryptedStore(
|
||||
store: remote,
|
||||
oldPrefix: _old,
|
||||
readPassphrase: () async => 'pw',
|
||||
params: cheapArgon,
|
||||
keyCache: E2eKeyCache(
|
||||
read: () async => cached,
|
||||
write: (bytes) async => cached = bytes,
|
||||
),
|
||||
);
|
||||
expect(state, isA<E2eReady>());
|
||||
expect(cached, isNot(List<int>.filled(32, 9)));
|
||||
});
|
||||
|
||||
test('the copy reruns only when old revisions change', () async {
|
||||
final remote = MemoryRemote({
|
||||
'$_old/pc/log.json': encodeLog(logOf({'a': 'x'})),
|
||||
'app/revs/pc': 'r1',
|
||||
});
|
||||
Future<void> open() => openEncryptedStore(
|
||||
store: remote,
|
||||
oldPrefix: _old,
|
||||
readPassphrase: () async => 'pw',
|
||||
params: cheapArgon,
|
||||
);
|
||||
await open();
|
||||
remote.reads.clear();
|
||||
await open();
|
||||
expect(remote.reads.where((p) => p.startsWith('app/notes-e2e/')), isEmpty);
|
||||
remote.files['$_old/pc/log.json'] = encodeLog(logOf({'a': 'x', 'b': 'y'}));
|
||||
remote.files['app/revs/pc'] = 'r2';
|
||||
await open();
|
||||
final state =
|
||||
await openEncryptedStore(
|
||||
store: remote,
|
||||
oldPrefix: _old,
|
||||
readPassphrase: () async => 'pw',
|
||||
params: cheapArgon,
|
||||
)
|
||||
as E2eReady;
|
||||
final text = await state.store.getFileText('app/notes-e2e/pc/log.json');
|
||||
expect(decodeLog(text!).keys, unorderedEquals(['a', 'b']));
|
||||
});
|
||||
|
||||
test('bulk reads of a data dir come back decrypted', () async {
|
||||
final remote = MemoryRemote();
|
||||
final state =
|
||||
await openEncryptedStore(
|
||||
store: remote,
|
||||
oldPrefix: _old,
|
||||
readPassphrase: () async => 'pw',
|
||||
params: cheapArgon,
|
||||
)
|
||||
as E2eReady;
|
||||
final log = encodeLog(logOf({'k': 'secret'}));
|
||||
await state.store.putFileText('app/notes-e2e/k1', log, message: 'm');
|
||||
expect(remote.files['app/notes-e2e/k1'], isNot(contains('secret')));
|
||||
final map = await state.store.getStringMap('app/notes-e2e');
|
||||
expect(map['k1'], log);
|
||||
});
|
||||
}
|
||||
63
crdt_sync_dart/test/e2e_open_test.dart
Normal file
63
crdt_sync_dart/test/e2e_open_test.dart
Normal file
@ -0,0 +1,63 @@
|
||||
import 'package:crdt_sync/crdt_sync.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
import 'e2e_fakes.dart';
|
||||
|
||||
const _old = 'app/notes';
|
||||
|
||||
Future<E2eState> _open(MemoryRemote r, String? pass) => openEncryptedStore(
|
||||
store: r,
|
||||
oldPrefix: _old,
|
||||
readPassphrase: () async => pass,
|
||||
params: cheapArgon,
|
||||
);
|
||||
|
||||
void main() {
|
||||
test('no passphrase is locked', () async {
|
||||
for (final p in [null, '']) {
|
||||
final s = await _open(MemoryRemote(), p);
|
||||
expect((s as E2eLocked).reason, E2eLockReason.needsPassphrase);
|
||||
}
|
||||
});
|
||||
|
||||
test('right passphrase copies and is ready; wrong one is locked', () async {
|
||||
final r = MemoryRemote({
|
||||
'$_old/phone/log.json': encodeLog(logOf({'a': 'milk'})),
|
||||
});
|
||||
expect(await _open(r, 'pw'), isA<E2eReady>());
|
||||
expect(r.files['app/notes-e2e/phone/log.json'], isNot(contains('milk')));
|
||||
expect(await _open(r, 'pw'), isA<E2eReady>());
|
||||
expect(
|
||||
((await _open(r, 'bad')) as E2eLocked).reason,
|
||||
E2eLockReason.wrongKey,
|
||||
);
|
||||
});
|
||||
|
||||
test('a corrupt header locks as badHeader', () async {
|
||||
final r = MemoryRemote({'app/e2e': 'garbage'});
|
||||
expect(
|
||||
((await _open(r, 'pw')) as E2eLocked).reason,
|
||||
E2eLockReason.badHeader,
|
||||
);
|
||||
});
|
||||
|
||||
test('a failing copy does not stop the open', () async {
|
||||
final r = MemoryRemote();
|
||||
r.files['$_old/phone/log.json'] = encodeLog(logOf({'a': 'x'}));
|
||||
final failing = _FailingList(r);
|
||||
final s = await openEncryptedStore(
|
||||
store: failing,
|
||||
oldPrefix: _old,
|
||||
readPassphrase: () async => 'pw',
|
||||
params: cheapArgon,
|
||||
);
|
||||
expect(s, isA<E2eReady>());
|
||||
});
|
||||
}
|
||||
|
||||
class _FailingList extends PlainRemote {
|
||||
_FailingList(super.inner);
|
||||
@override
|
||||
Future<List<String>> listDirectory(String path) =>
|
||||
throw StateError('list down');
|
||||
}
|
||||
290
crdt_sync_dart/test/encrypting_store_test.dart
Normal file
290
crdt_sync_dart/test/encrypting_store_test.dart
Normal file
@ -0,0 +1,290 @@
|
||||
/// syncLog over an EncryptingStore: the relay sees ciphertext, merge is
|
||||
/// unchanged.
|
||||
library;
|
||||
|
||||
import 'package:crdt_sync/crdt_sync.dart';
|
||||
import 'package:glados/glados.dart' hide expect, group, test, setUp;
|
||||
import 'package:test/test.dart';
|
||||
|
||||
import 'e2e_fakes.dart';
|
||||
|
||||
const _prefix = 'todo-sync/notes';
|
||||
const _revs = 'todo-sync/revs';
|
||||
|
||||
Future<E2eKey> _key(MemoryRemote remote, [String passphrase = 'pw']) =>
|
||||
unlockE2e(
|
||||
store: remote,
|
||||
pathPrefix: _prefix,
|
||||
passphrase: passphrase,
|
||||
params: cheapArgon,
|
||||
random: Random(1),
|
||||
);
|
||||
|
||||
Future<Log> _tick(
|
||||
RemoteStore client,
|
||||
String deviceId,
|
||||
Log local, {
|
||||
SyncStateStore? state,
|
||||
List<String>? skipped,
|
||||
}) => syncLog(
|
||||
client: client,
|
||||
deviceId: deviceId,
|
||||
pathPrefix: _prefix,
|
||||
localLog: local,
|
||||
encode: encodeLog,
|
||||
decode: decodeLog,
|
||||
stateStore: state,
|
||||
onPeerSkipped: (id, _) => skipped?.add(id),
|
||||
);
|
||||
|
||||
void main() {
|
||||
test('pushed blob is ciphertext and a peer merges it', () async {
|
||||
final remote = MemoryRemote();
|
||||
final key = await _key(remote);
|
||||
final store = EncryptingStore(remote, key, pathPrefix: _prefix);
|
||||
await _tick(store, 'phone', logOf({'n1': 'buy milk'}));
|
||||
|
||||
final blob = remote.files['$_prefix/phone/log.json']!;
|
||||
expect(blob, isNot(contains('title')));
|
||||
expect(blob, isNot(contains('buy milk')));
|
||||
|
||||
final merged = await _tick(store, 'pc', logOf({'n2': 'call mum'}));
|
||||
expect(merged.keys, unorderedEquals(['n1', 'n2']));
|
||||
expect(remote.files.values.where((v) => v.contains('"title"')), isEmpty);
|
||||
});
|
||||
|
||||
test('published revisions are keyed, not sha256 of the plaintext', () async {
|
||||
final remote = MemoryRemote();
|
||||
final store = EncryptingStore(
|
||||
remote,
|
||||
await _key(remote),
|
||||
pathPrefix: _prefix,
|
||||
);
|
||||
final local = logOf({'n1': 'x'});
|
||||
await _tick(store, 'phone', local, state: InMemorySyncStateStore());
|
||||
final rev = remote.files['$_revs/phone']!;
|
||||
expect(rev, isNot(revisionOf(encodeLog(local))));
|
||||
expect(rev, hasLength(64));
|
||||
});
|
||||
|
||||
test('a peer that has not upgraded is still read', () async {
|
||||
final remote = MemoryRemote();
|
||||
await _tick(remote, 'pc', logOf({'old': 'plain'}));
|
||||
final store = EncryptingStore(
|
||||
remote,
|
||||
await _key(remote),
|
||||
pathPrefix: _prefix,
|
||||
);
|
||||
final merged = await _tick(store, 'phone', logOf({'n1': 'x'}));
|
||||
expect(merged.keys, unorderedEquals(['old', 'n1']));
|
||||
});
|
||||
|
||||
test('a tampered peer is skipped and the others still merge', () async {
|
||||
final remote = MemoryRemote();
|
||||
final store = EncryptingStore(
|
||||
remote,
|
||||
await _key(remote),
|
||||
pathPrefix: _prefix,
|
||||
);
|
||||
await _tick(store, 'b', logOf({'b1': 'y'}));
|
||||
await _tick(store, 'a', logOf({'a1': 'x'}));
|
||||
final blob = remote.files['$_prefix/a/log.json']!;
|
||||
remote.files['$_prefix/a/log.json'] = blob.replaceRange(
|
||||
40,
|
||||
41,
|
||||
blob[40] == 'A' ? 'B' : 'A',
|
||||
);
|
||||
final skipped = <String>[];
|
||||
final merged = await _tick(store, 'c', {}, skipped: skipped);
|
||||
expect(merged.keys, ['b1']);
|
||||
expect(skipped, ['a']);
|
||||
});
|
||||
|
||||
test('unparsable and mis-shaped peers are reported as skipped', () async {
|
||||
final remote = MemoryRemote({
|
||||
'$_prefix/bad/log.json': '{oops',
|
||||
'$_prefix/shape/log.json': '{"n":{"id":1}}',
|
||||
});
|
||||
final skipped = <String>[];
|
||||
await _tick(remote, 'me', {}, skipped: skipped);
|
||||
expect(skipped, unorderedEquals(['bad', 'shape']));
|
||||
});
|
||||
|
||||
test('a peer under another passphrase is skipped, not fatal', () async {
|
||||
final remote = MemoryRemote();
|
||||
final other = MemoryRemote();
|
||||
final mine = EncryptingStore(
|
||||
remote,
|
||||
await _key(remote),
|
||||
pathPrefix: _prefix,
|
||||
);
|
||||
final theirs = EncryptingStore(
|
||||
remote,
|
||||
await _key(other, 'other'),
|
||||
pathPrefix: _prefix,
|
||||
);
|
||||
await _tick(theirs, 'x', logOf({'x1': 'x'}));
|
||||
expect(await _tick(mine, 'me', {}), isEmpty);
|
||||
});
|
||||
|
||||
test('a prefix without a slash keeps revs out of the sealed set', () async {
|
||||
final remote = MemoryRemote();
|
||||
final key = await unlockE2e(
|
||||
store: remote,
|
||||
pathPrefix: 'wake-alarm',
|
||||
passphrase: 'pw',
|
||||
params: cheapArgon,
|
||||
);
|
||||
final store = EncryptingStore(remote, key, pathPrefix: 'wake-alarm');
|
||||
await syncLog(
|
||||
client: store,
|
||||
deviceId: 'phone',
|
||||
pathPrefix: 'wake-alarm',
|
||||
localLog: logOf({'n': 'x'}),
|
||||
encode: encodeLog,
|
||||
decode: decodeLog,
|
||||
stateStore: InMemorySyncStateStore(),
|
||||
);
|
||||
expect(remote.files['wake-alarm/revs/phone'], hasLength(64));
|
||||
expect(remote.files['wake-alarm/phone/log.json'], isNot(contains('"x"')));
|
||||
expect(remote.files['wake-alarm/e2e'], contains('argon2id'));
|
||||
});
|
||||
|
||||
test('paths outside the prefix pass through untouched', () async {
|
||||
final remote = MemoryRemote();
|
||||
final store = EncryptingStore(
|
||||
remote,
|
||||
await _key(remote),
|
||||
pathPrefix: _prefix,
|
||||
);
|
||||
await store.putFileText('other/x', '{"a":1}', message: 'm');
|
||||
expect(remote.files['other/x'], '{"a":1}');
|
||||
expect(await store.getFileText('other/x'), '{"a":1}');
|
||||
expect(await store.getFileText('$_prefix/none/log.json'), isNull);
|
||||
});
|
||||
|
||||
test('a flat <prefix>/<node>.json layout (todo) is sealed too', () async {
|
||||
final remote = MemoryRemote();
|
||||
final store = EncryptingStore(
|
||||
remote,
|
||||
await _key(remote),
|
||||
pathPrefix: _prefix,
|
||||
);
|
||||
final log = encodeLog(logOf({'n1': 'buy milk'}));
|
||||
await store.putFileText('$_prefix/node-a.json', log, message: 'm');
|
||||
expect(remote.files['$_prefix/node-a.json'], isNot(contains('milk')));
|
||||
expect(isE2eEnvelope(remote.files['$_prefix/node-a.json']!), isTrue);
|
||||
expect(await store.getFileText('$_prefix/node-a.json'), log);
|
||||
});
|
||||
|
||||
test('a slashless prefix keeps the header and revs out of sealing', () async {
|
||||
final remote = MemoryRemote();
|
||||
final key = await unlockE2e(
|
||||
store: remote,
|
||||
pathPrefix: 'wake-alarm',
|
||||
passphrase: 'pw',
|
||||
params: cheapArgon,
|
||||
);
|
||||
final store = EncryptingStore(remote, key, pathPrefix: 'wake-alarm');
|
||||
final header = remote.files['wake-alarm/e2e']!;
|
||||
expect(await store.getFileText('wake-alarm/e2e'), header);
|
||||
await store.putFileText('wake-alarm/e2e', header, message: 'm');
|
||||
expect(remote.files['wake-alarm/e2e'], header);
|
||||
});
|
||||
|
||||
test('plaintext is refused when acceptPlaintext is off', () async {
|
||||
final remote = MemoryRemote({
|
||||
'$_prefix/pc/log.json': encodeLog(logOf({'a': 'b'})),
|
||||
});
|
||||
final store = EncryptingStore(
|
||||
remote,
|
||||
await _key(remote),
|
||||
pathPrefix: _prefix,
|
||||
acceptPlaintext: false,
|
||||
);
|
||||
expect(await _tick(store, 'phone', {}), isEmpty);
|
||||
});
|
||||
|
||||
group('delegation', () {
|
||||
test('bulk reads forward when the inner store has them', () async {
|
||||
final remote = MemoryRemote({'$_revs/a': 'r'});
|
||||
final store = EncryptingStore(
|
||||
remote,
|
||||
await _key(remote),
|
||||
pathPrefix: _prefix,
|
||||
);
|
||||
expect(await store.getStringMap(_revs), {'a': 'r'});
|
||||
});
|
||||
|
||||
test('bulk reads degrade to "fetch everything" without them', () async {
|
||||
final remote = MemoryRemote({'$_revs/a': 'r'});
|
||||
final store = EncryptingStore(
|
||||
PlainRemote(remote),
|
||||
await _key(remote),
|
||||
pathPrefix: _prefix,
|
||||
);
|
||||
expect(await store.getStringMap(_revs), isEmpty);
|
||||
});
|
||||
|
||||
test('list, delete, access and close reach the inner store', () async {
|
||||
final remote = MemoryRemote({'$_prefix/a/log.json': '{}'});
|
||||
final store = EncryptingStore(
|
||||
remote,
|
||||
await _key(remote),
|
||||
pathPrefix: _prefix,
|
||||
);
|
||||
expect(await store.listDirectory(_prefix), ['a']);
|
||||
await store.deleteFile('$_prefix/a/log.json', message: 'm');
|
||||
expect(remote.files.containsKey('$_prefix/a/log.json'), isFalse);
|
||||
expect(await store.canAccessRemote(), isTrue);
|
||||
store.close();
|
||||
expect(remote.closed, isTrue);
|
||||
});
|
||||
});
|
||||
|
||||
// Differential: the same sync rounds over a plain and an encrypting relay
|
||||
// must converge to the same logs.
|
||||
Glados(
|
||||
any.list(
|
||||
any.combine3(
|
||||
any.intInRange(0, 3),
|
||||
any.intInRange(0, 6),
|
||||
any.letterOrDigits,
|
||||
(int d, int n, String t) => (d, n, t),
|
||||
),
|
||||
),
|
||||
ExploreConfig(numRuns: 25),
|
||||
).test('merge is identical with and without encryption', (edits) async {
|
||||
const devices = ['a', 'b', 'c'];
|
||||
Log localFor(int d) => {
|
||||
for (final (i, (dev, note, title)) in edits.indexed)
|
||||
if (dev == d)
|
||||
'n$note': Record(
|
||||
id: 'n$note',
|
||||
fields: {
|
||||
'title': (
|
||||
title,
|
||||
Hlc(wallTimeMs: 1000 + i, counter: 0, nodeId: devices[d]),
|
||||
),
|
||||
},
|
||||
),
|
||||
};
|
||||
final plain = MemoryRemote();
|
||||
final sealedRemote = MemoryRemote();
|
||||
final sealed = EncryptingStore(
|
||||
sealedRemote,
|
||||
await E2eKey.fromBytes(List<int>.filled(32, 3)),
|
||||
pathPrefix: _prefix,
|
||||
);
|
||||
final plainLogs = [for (var d = 0; d < 3; d++) localFor(d)];
|
||||
final sealedLogs = [for (var d = 0; d < 3; d++) localFor(d)];
|
||||
for (var round = 0; round < 2; round++) {
|
||||
for (var d = 0; d < 3; d++) {
|
||||
plainLogs[d] = await _tick(plain, devices[d], plainLogs[d]);
|
||||
sealedLogs[d] = await _tick(sealed, devices[d], sealedLogs[d]);
|
||||
}
|
||||
}
|
||||
expect(sealedLogs, plainLogs);
|
||||
expect(sealedLogs[0], sealedLogs[2]);
|
||||
});
|
||||
}
|
||||
10
crdt_sync_dart/test/fixtures/e2e_dart_vectors.json
vendored
Normal file
10
crdt_sync_dart/test/fixtures/e2e_dart_vectors.json
vendored
Normal file
@ -0,0 +1,10 @@
|
||||
{
|
||||
"passphrase": "correct horse ✓",
|
||||
"header": "{\"v\":1,\"kdf\":\"argon2id\",\"salt\":\"HGW+FZJe6hd71He22BrlUA==\",\"m\":64,\"t\":2,\"p\":1,\"check\":\"okKnATkNoZKvjSGfrgQGOjdRw5NORh2lC32zuBq7V6I=\"}",
|
||||
"master_hex": "cad5afe8ddc6e21d2f39eb196de454db6b6d1f60629619ce8539f03a4bb0b086",
|
||||
"revision_of_abc": "e7f845e1b0a7ea2b571fd3e3bdfb60641e650d63171bf73b6356c7909d2748c8",
|
||||
"path": "vec/notes-e2e/phone/log.json",
|
||||
"log": "{\"n1\":{\"id\":\"n1\",\"fields\":{\"title\":[\"buy milk\",\"1970-01-01T00:00:01.000Z-0000-node-a\"]},\"deleted\":false,\"deleted_hlc\":null},\"n2\":{\"id\":\"n2\",\"fields\":{\"title\":[\"zażółć\",\"1970-01-01T00:00:01.000Z-0000-node-a\"]},\"deleted\":false,\"deleted_hlc\":null}}",
|
||||
"envelope": "{\"v\":1,\"r\":{\"n1\":{\"d\":false,\"dh\":null,\"h\":\"1970-01-01T00:00:01.000Z-0000-node-a\",\"p\":\"AR1zODx4TD67X3TGa0N7EV/V4FcZAaqc41jxZVtskFu3Pqb/pNlqm6db66rQUaMyNLtTPa4HMOJzZomBRyaSHjaPb4oNxgVPkMeIF5IbuI/GA30gR0sz/MSvYmfZ5wSnlBo1+fpH\"},\"n2\":{\"d\":false,\"dh\":null,\"h\":\"1970-01-01T00:00:01.000Z-0000-node-a\",\"p\":\"Aa0omlBGnmlsItzq7A21G0A1tqT8xMOJE0wgxcxJ1c3hVpRmNt4kq2IPNNGuW8v4A7fvs2FrnGX4A+NVtXPpu41ckKSShTqk8Ml6zTuxyrT85g50sgzbM7PKFczeEaS+ITWDj0N04u0=\"}}}",
|
||||
"whole": "{\"v\":1,\"b\":\"AYdAwPehG84oHBRhRr5dP5VmleBGzQC1ryV5QHTCqBZE9ZsOZMMkJiP40+KzpnfIc2g=\"}"
|
||||
}
|
||||
9
crdt_sync_dart/test/fixtures/e2e_py_vectors.json
vendored
Normal file
9
crdt_sync_dart/test/fixtures/e2e_py_vectors.json
vendored
Normal file
@ -0,0 +1,9 @@
|
||||
{
|
||||
"passphrase": "correct horse ✓",
|
||||
"header": "{\"v\":1,\"kdf\":\"argon2id\",\"salt\":\"fGy9SnWSTwxKbS/sAluVjA==\",\"m\":64,\"t\":2,\"p\":1,\"check\":\"ekpcuZxZl+Ei6Y5yQp+cwdi4yR6ItJiYR7S3cd214ds=\"}",
|
||||
"master_hex": "834f54fef4eb086e4201b918c938be179f91cf72e157e28d56dfe72bc46cbefd",
|
||||
"path": "vec/notes-e2e/pc/log.json",
|
||||
"log": "{\"n9\": {\"id\": \"n9\", \"fields\": {\"title\": [\"zażółć\", \"2026-01-01T00:00:00.000Z-0000-pc\"]}, \"deleted\": false, \"deleted_hlc\": null}}",
|
||||
"envelope": "{\"v\":1,\"r\":{\"n9\":{\"d\":false,\"dh\":null,\"h\":\"2026-01-01T00:00:00.000Z-0000-pc\",\"p\":\"ATAIR4jQmlBjDF+/4+AYp3ATcGgjZQ60tQ05uvF1howYly4W4h3lLI04BssCRASargNV/+hDhqTl7n2cz4hh7Z5lmZYgElso/4k85hwO1SRuDPOYMKsLCpYgTy8zYzEiiRlv+g==\"}}}",
|
||||
"whole": "{\"v\":1,\"b\":\"ARvUAj5nIfmJdsxdaiAl5uYkgry9VG5VQLQBi6+BNbyJ0t//UGjc1Ji2hmbBJl3v9kw=\"}"
|
||||
}
|
||||
170
crdt_sync_dart/test/sync_namespace_test.dart
Normal file
170
crdt_sync_dart/test/sync_namespace_test.dart
Normal file
@ -0,0 +1,170 @@
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:crdt_sync/crdt_sync.dart';
|
||||
import 'package:http/http.dart' as http;
|
||||
import 'package:http/testing.dart' as http_testing;
|
||||
import 'package:test/test.dart';
|
||||
|
||||
const _databaseUrl = 'https://example-default-rtdb.firebasedatabase.app';
|
||||
|
||||
FirebaseTokenProvider _auth() => FirebaseTokenProvider(
|
||||
apiKey: 'fake-api-key',
|
||||
store: InMemoryCredentialStore(
|
||||
FirebaseCredentials(
|
||||
idToken: 'id-token',
|
||||
refreshToken: 'refresh-token',
|
||||
expiresAt: DateTime.utc(2099),
|
||||
),
|
||||
),
|
||||
httpClient: http_testing.MockClient(
|
||||
(_) async => throw StateError('auth must not be contacted'),
|
||||
),
|
||||
);
|
||||
|
||||
/// Client over an in-memory transport that records every request path.
|
||||
(FirebaseRestClient, List<http.Request>) _client(String? namespace) {
|
||||
final requests = <http.Request>[];
|
||||
final mock = http_testing.MockClient((request) async {
|
||||
requests.add(request);
|
||||
return http.Response(jsonEncode(request.method == 'GET' ? null : 'x'), 200);
|
||||
});
|
||||
final client = namespace == null
|
||||
? FirebaseRestClient(
|
||||
databaseUrl: _databaseUrl,
|
||||
auth: _auth(),
|
||||
httpClient: mock,
|
||||
)
|
||||
: FirebaseRestClient(
|
||||
databaseUrl: _databaseUrl,
|
||||
auth: _auth(),
|
||||
httpClient: mock,
|
||||
namespace: namespace,
|
||||
);
|
||||
return (client, requests);
|
||||
}
|
||||
|
||||
void main() {
|
||||
group('compile-time default', () {
|
||||
test('is empty when SYNC_NAMESPACE is not defined', () {
|
||||
expect(syncNamespace, isEmpty);
|
||||
});
|
||||
|
||||
test('empty namespace leaves paths byte-for-byte unchanged', () async {
|
||||
final (client, requests) = _client(null);
|
||||
await client.getFileText('devices/pc/log.json');
|
||||
expect(requests.single.url.path, '/devices/pc/log~2Ejson.json');
|
||||
});
|
||||
});
|
||||
|
||||
group('prefixing', () {
|
||||
test('every operation lands under the namespace', () async {
|
||||
final (client, requests) = _client('sandbox/slot3');
|
||||
await client.getFileText('devices/pc/log.json');
|
||||
await client.putFileText('devices/pc/log.json', 't', message: 'm');
|
||||
await client.listDirectory('devices');
|
||||
await client.getStringMap('revs');
|
||||
await client.patchValues('revs', {'a': 'b'});
|
||||
await client.deleteFile('devices/pc/log.json');
|
||||
await client.canAccessRemote();
|
||||
expect(requests.map((r) => r.url.path), [
|
||||
'/sandbox/slot3/devices/pc/log~2Ejson.json',
|
||||
'/sandbox/slot3/devices/pc/log~2Ejson.json',
|
||||
'/sandbox/slot3/devices.json',
|
||||
'/sandbox/slot3/revs.json',
|
||||
'/sandbox/slot3/revs.json',
|
||||
'/sandbox/slot3/devices/pc/log~2Ejson.json',
|
||||
'/sandbox/slot3.json',
|
||||
]);
|
||||
});
|
||||
|
||||
test('an explicitly empty namespace is the unprefixed path', () async {
|
||||
final (client, requests) = _client('');
|
||||
await client.getStringMap('revs');
|
||||
expect(requests.single.url.path, '/revs.json');
|
||||
});
|
||||
});
|
||||
|
||||
group('validation (fail closed)', () {
|
||||
const bad = [
|
||||
'sandbox',
|
||||
'sandbox/',
|
||||
'sandbox/Slot3',
|
||||
'sandbox/slot 3',
|
||||
'sandbox/slot3/',
|
||||
'sandbox/a/b',
|
||||
'/sandbox/slot3',
|
||||
'prod/slot3',
|
||||
'sandbox/slot3\n',
|
||||
'Sandbox/slot3',
|
||||
'sandbox/slot.3',
|
||||
'sandbox/..',
|
||||
];
|
||||
|
||||
for (final value in bad) {
|
||||
test('rejects ${jsonEncode(value)}', () {
|
||||
expect(
|
||||
() => checkSyncNamespace(value),
|
||||
throwsA(isA<ConfigException>()),
|
||||
);
|
||||
expect(
|
||||
() => FirebaseRestClient(
|
||||
databaseUrl: _databaseUrl,
|
||||
auth: _auth(),
|
||||
namespace: value,
|
||||
),
|
||||
throwsA(isA<ConfigException>()),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
test('the message names the define and the required shape', () {
|
||||
expect(
|
||||
() => checkSyncNamespace('typo'),
|
||||
throwsA(
|
||||
isA<ConfigException>().having(
|
||||
(e) => e.message,
|
||||
'message',
|
||||
allOf(
|
||||
contains('SYNC_NAMESPACE'),
|
||||
contains(r'^sandbox/[a-z0-9_-]+$'),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts the allowed alphabet', () {
|
||||
expect(() => checkSyncNamespace('sandbox/slot3'), returnsNormally);
|
||||
expect(() => checkSyncNamespace('sandbox/a_b-c9'), returnsNormally);
|
||||
expect(() => checkSyncNamespace(''), returnsNormally);
|
||||
});
|
||||
|
||||
test(
|
||||
'a bad compile-time define stops firebaseClientFor at startup',
|
||||
() async {
|
||||
// The define is compile-time, so only a fresh process can set it.
|
||||
final result = await Process.run(Platform.resolvedExecutable, [
|
||||
'-DSYNC_NAMESPACE=sandbox/Typo',
|
||||
'run',
|
||||
'tool/namespace_probe.dart',
|
||||
]);
|
||||
expect(result.exitCode, isNot(0));
|
||||
expect(
|
||||
result.stderr,
|
||||
contains('SYNC_NAMESPACE "sandbox/Typo" is invalid'),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test('a good compile-time define reaches the request path', () async {
|
||||
final result = await Process.run(Platform.resolvedExecutable, [
|
||||
'-DSYNC_NAMESPACE=sandbox/slot3',
|
||||
'run',
|
||||
'tool/namespace_probe.dart',
|
||||
]);
|
||||
expect(result.stderr, isEmpty);
|
||||
expect(result.stdout, '/sandbox/slot3/devices.json\n');
|
||||
});
|
||||
});
|
||||
}
|
||||
47
crdt_sync_dart/tool/e2e_audit.dart
Normal file
47
crdt_sync_dart/tool/e2e_audit.dart
Normal file
@ -0,0 +1,47 @@
|
||||
/// Read-only done check: lists plaintext blobs under an RTDB root.
|
||||
///
|
||||
/// Usage: `dart run tool/e2e_audit.dart <root> <firebase_auth.json>`
|
||||
/// Exit 0 when every blob is sealed, 1 when any is plaintext, 2 on misuse.
|
||||
library;
|
||||
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:crdt_sync/crdt_sync.dart';
|
||||
import 'package:http/http.dart' as http;
|
||||
|
||||
Future<void> main(List<String> args) async {
|
||||
if (args.length != 2) {
|
||||
stderr.writeln('usage: e2e_audit.dart <root> <firebase_auth.json>');
|
||||
exit(2);
|
||||
}
|
||||
final [root, sessionFile] = args;
|
||||
final home = Platform.environment['HOME'];
|
||||
final cfg =
|
||||
jsonDecode(
|
||||
File('$home/.config/crdt-sync/firebase.json').readAsStringSync(),
|
||||
)
|
||||
as Map<String, dynamic>;
|
||||
final session = FirebaseCredentials.fromJson(
|
||||
jsonDecode(File(sessionFile).readAsStringSync()) as Map<String, dynamic>,
|
||||
);
|
||||
final auth = FirebaseTokenProvider(
|
||||
apiKey: cfg['apiKey'] as String,
|
||||
store: InMemoryCredentialStore(session),
|
||||
);
|
||||
final url = Uri.parse(
|
||||
'${cfg['databaseUrl']}/${encodePath(root)}.json'
|
||||
'?auth=${await auth.idToken()}',
|
||||
);
|
||||
final response = await http.get(url);
|
||||
if (response.statusCode != 200) {
|
||||
stderr.writeln('GET $root: HTTP ${response.statusCode}');
|
||||
exit(2);
|
||||
}
|
||||
final plaintext = plaintextBlobPaths(jsonDecode(response.body));
|
||||
stdout.writeln('plaintext blobs under $root: ${plaintext.length}');
|
||||
for (final path in plaintext) {
|
||||
stdout.writeln(' $root/$path');
|
||||
}
|
||||
exit(plaintext.isEmpty ? 0 : 1);
|
||||
}
|
||||
41
crdt_sync_dart/tool/e2e_vectors.dart
Normal file
41
crdt_sync_dart/tool/e2e_vectors.dart
Normal file
@ -0,0 +1,41 @@
|
||||
/// Writes cross-language golden vectors for the Python port to open.
|
||||
///
|
||||
/// Run: `dart run tool/e2e_vectors.dart > test/fixtures/e2e_dart_vectors.json`
|
||||
library;
|
||||
|
||||
import 'dart:convert';
|
||||
import 'dart:math';
|
||||
|
||||
import 'package:crdt_sync/crdt_sync.dart';
|
||||
|
||||
import '../test/e2e_fakes.dart';
|
||||
|
||||
Future<void> main() async {
|
||||
const params = Argon2Params(memoryKiB: 64, iterations: 2, parallelism: 1);
|
||||
final remote = MemoryRemote();
|
||||
final key = await unlockE2e(
|
||||
store: remote,
|
||||
pathPrefix: 'vec/notes-e2e',
|
||||
passphrase: 'correct horse ✓',
|
||||
params: params,
|
||||
random: Random(7),
|
||||
e2ePath: 'vec/e2e',
|
||||
);
|
||||
final cipher = key.cipher();
|
||||
const path = 'vec/notes-e2e/phone/log.json';
|
||||
final log = encodeLog(logOf({'n1': 'buy milk', 'n2': 'zażółć'}));
|
||||
print(
|
||||
const JsonEncoder.withIndent(' ').convert({
|
||||
'passphrase': 'correct horse ✓',
|
||||
'header': remote.files['vec/e2e'],
|
||||
'master_hex': key.bytes
|
||||
.map((b) => b.toRadixString(16).padLeft(2, '0'))
|
||||
.join(),
|
||||
'revision_of_abc': await key.revision('abc'),
|
||||
'path': path,
|
||||
'log': log,
|
||||
'envelope': await sealLogText(log, cipher, path: path),
|
||||
'whole': await sealLogText('not a log', cipher, path: path),
|
||||
}),
|
||||
);
|
||||
}
|
||||
@ -12,15 +12,15 @@ import 'package:crdt_sync/crdt_sync.dart';
|
||||
/// mode 0600 -- never in source, because this repository is public.
|
||||
final _configDir = '${Platform.environment['HOME']}/.config/crdt-sync';
|
||||
|
||||
Log _decode(String t) => (jsonDecode(t) as Map<String, dynamic>)
|
||||
.map((k, v) => MapEntry(k, Record.fromJson(v as Map<String, dynamic>)));
|
||||
String _encode(Log l) =>
|
||||
jsonEncode(l.map((k, v) => MapEntry(k, v.toJson())));
|
||||
Log _decode(String t) => (jsonDecode(t) as Map<String, dynamic>).map(
|
||||
(k, v) => MapEntry(k, Record.fromJson(v as Map<String, dynamic>)),
|
||||
);
|
||||
String _encode(Log l) => jsonEncode(l.map((k, v) => MapEntry(k, v.toJson())));
|
||||
|
||||
Future<void> main(List<String> args) async {
|
||||
final cfg = jsonDecode(
|
||||
File('$_configDir/firebase.json').readAsStringSync(),
|
||||
) as Map<String, dynamic>;
|
||||
final cfg =
|
||||
jsonDecode(File('$_configDir/firebase.json').readAsStringSync())
|
||||
as Map<String, dynamic>;
|
||||
final auth = FirebaseTokenProvider(
|
||||
apiKey: cfg['apiKey'] as String,
|
||||
store: InMemoryCredentialStore(),
|
||||
|
||||
@ -23,7 +23,9 @@ const _prefix = '_smoketest/devices';
|
||||
var _failures = 0;
|
||||
|
||||
void _check(String what, bool ok, [String detail = '']) {
|
||||
stdout.writeln('${ok ? " ok " : " FAIL"} $what${detail.isEmpty ? '' : ' ($detail)'}');
|
||||
stdout.writeln(
|
||||
'${ok ? " ok " : " FAIL"} $what${detail.isEmpty ? '' : ' ($detail)'}',
|
||||
);
|
||||
if (!ok) _failures++;
|
||||
}
|
||||
|
||||
@ -68,16 +70,24 @@ Future<void> main(List<String> args) async {
|
||||
_check('canAccessRemote is false before sign-in', !await auth.hasSession());
|
||||
await auth.signIn(email: email, password: password);
|
||||
_check('signIn stored a session', await auth.hasSession());
|
||||
_check('canAccessRemote is true after sign-in', await client.canAccessRemote());
|
||||
_check(
|
||||
'canAccessRemote is true after sign-in',
|
||||
await client.canAccessRemote(),
|
||||
);
|
||||
|
||||
stdout.writeln('\n== round-trip and key escaping ==');
|
||||
const path = '$_prefix/pc/log.json';
|
||||
await client.putFileText(path, '{"hello":"world"}', message: 'smoke');
|
||||
final read = await client.getFileText(path);
|
||||
_check('a .json filename round-trips byte-for-byte',
|
||||
read == '{"hello":"world"}', read ?? 'null');
|
||||
_check('missing path reads as null',
|
||||
await client.getFileText('$_prefix/nope/log.json') == null);
|
||||
_check(
|
||||
'a .json filename round-trips byte-for-byte',
|
||||
read == '{"hello":"world"}',
|
||||
read ?? 'null',
|
||||
);
|
||||
_check(
|
||||
'missing path reads as null',
|
||||
await client.getFileText('$_prefix/nope/log.json') == null,
|
||||
);
|
||||
final listed = await client.listDirectory(_prefix);
|
||||
_check('listDirectory sees the device', listed.contains('pc'), '$listed');
|
||||
|
||||
@ -96,8 +106,11 @@ Future<void> main(List<String> args) async {
|
||||
final pushed = await client.getFileText(path);
|
||||
_check('the log landed in the database', pushed != null);
|
||||
final revs = await client.getStringMap('_smoketest/revs');
|
||||
_check('a revision was published for this device',
|
||||
revs['pc'] == revisionOf(_encode(pcLog)), '${revs.keys}');
|
||||
_check(
|
||||
'a revision was published for this device',
|
||||
revs['pc'] == revisionOf(_encode(pcLog)),
|
||||
'${revs.keys}',
|
||||
);
|
||||
|
||||
stdout.writeln('\n== sync: an unchanged tick writes nothing ==');
|
||||
final before = await client.getStringMap('_smoketest/revs');
|
||||
@ -110,8 +123,10 @@ Future<void> main(List<String> args) async {
|
||||
decode: _decode,
|
||||
stateStore: storePc,
|
||||
);
|
||||
_check('revision unchanged, so nothing was re-uploaded',
|
||||
(await client.getStringMap('_smoketest/revs'))['pc'] == before['pc']);
|
||||
_check(
|
||||
'revision unchanged, so nothing was re-uploaded',
|
||||
(await client.getStringMap('_smoketest/revs'))['pc'] == before['pc'],
|
||||
);
|
||||
|
||||
stdout.writeln('\n== sync: two devices converge ==');
|
||||
final storePhone = InMemorySyncStateStore();
|
||||
@ -135,8 +150,10 @@ Future<void> main(List<String> args) async {
|
||||
stateStore: storePc,
|
||||
);
|
||||
_check('pc merged the phone record', pcAfter.containsKey('b'));
|
||||
_check('each device wrote only its own revision key',
|
||||
(await client.getStringMap('_smoketest/revs')).length == 2);
|
||||
_check(
|
||||
'each device wrote only its own revision key',
|
||||
(await client.getStringMap('_smoketest/revs')).length == 2,
|
||||
);
|
||||
|
||||
stdout.writeln('\n== fail-closed ==');
|
||||
await auth.signOut();
|
||||
@ -160,7 +177,9 @@ Future<void> main(List<String> args) async {
|
||||
client.close();
|
||||
auth.close();
|
||||
stdout.writeln(
|
||||
_failures == 0 ? '\nALL LIVE CHECKS PASSED' : '\n$_failures LIVE CHECK(S) FAILED',
|
||||
_failures == 0
|
||||
? '\nALL LIVE CHECKS PASSED'
|
||||
: '\n$_failures LIVE CHECK(S) FAILED',
|
||||
);
|
||||
exit(_failures == 0 ? 0 : 1);
|
||||
}
|
||||
|
||||
31
crdt_sync_dart/tool/namespace_probe.dart
Normal file
31
crdt_sync_dart/tool/namespace_probe.dart
Normal file
@ -0,0 +1,31 @@
|
||||
// Helper for test/sync_namespace_test.dart: the SYNC_NAMESPACE define is
|
||||
// compile-time, so the test runs this in a fresh process with -D set. It signs
|
||||
// in from an in-memory session (no network) and prints the first request path.
|
||||
import 'package:crdt_sync/crdt_sync.dart';
|
||||
import 'package:http/http.dart' as http;
|
||||
import 'package:http/testing.dart' as http_testing;
|
||||
|
||||
Future<void> main() async {
|
||||
final mock = http_testing.MockClient((request) async {
|
||||
print(request.url.path);
|
||||
return http.Response('null', 200);
|
||||
});
|
||||
final client = await firebaseClientFor(
|
||||
config: const FirebaseConfig(
|
||||
apiKey: 'k',
|
||||
databaseUrl: 'https://example.firebasedatabase.app',
|
||||
projectId: 'p',
|
||||
uid: 'u',
|
||||
email: 'e@x.y',
|
||||
),
|
||||
store: InMemoryCredentialStore(
|
||||
FirebaseCredentials(
|
||||
idToken: 'i',
|
||||
refreshToken: 'r',
|
||||
expiresAt: DateTime.utc(2099),
|
||||
),
|
||||
),
|
||||
httpClient: mock,
|
||||
);
|
||||
await client.listDirectory('devices');
|
||||
}
|
||||
@ -8,7 +8,7 @@ bootstrap, so a new app reaches "syncing" without writing any of it.
|
||||
|
||||
`crdt_sync` is pure Dart on purpose — it runs from command-line tools and from
|
||||
`wake_alarm`'s headless systemd job, so it cannot depend on
|
||||
`flutter_secure_storage` or `google_sign_in`. It takes closures instead.
|
||||
`flutter_secure_storage` or the Android Google plugin. It takes closures instead.
|
||||
|
||||
Every Flutter app then wrote the same ~390 lines to supply those closures
|
||||
(`firebase_backend.dart` + `google_sign_in_backend.dart` + `google_platform*`),
|
||||
@ -110,6 +110,11 @@ const kServerClientId = '845446124781-...apps.googleusercontent.com';
|
||||
/// to loopback.
|
||||
const kDesktopClientId = '845446124781-...apps.googleusercontent.com';
|
||||
|
||||
/// That Desktop client's secret. Google rejects the token exchange without
|
||||
/// it, but treats it as non-confidential for installed apps: inject it at
|
||||
/// build time (`--dart-define`), never commit it. '' sends none.
|
||||
const kDesktopClientSecret = String.fromEnvironment('DESKTOP_CLIENT_SECRET');
|
||||
|
||||
SyncSettingsScreen(
|
||||
accountLoader: () => loadAccount(),
|
||||
accountSaver: saveAccount,
|
||||
@ -121,6 +126,7 @@ SyncSettingsScreen(
|
||||
tokenFetcher: () => googleAnyIdToken(
|
||||
serverClientId: kServerClientId,
|
||||
desktopClientId: kDesktopClientId,
|
||||
desktopClientSecret: kDesktopClientSecret,
|
||||
),
|
||||
),
|
||||
googleAvailable: googleAnySignInSupported(
|
||||
@ -132,13 +138,13 @@ SyncSettingsScreen(
|
||||
|
||||
### One tap on every platform
|
||||
|
||||
`google_sign_in` ships **android, ios and web** implementations only — there
|
||||
is no `google_sign_in_linux` — so a real GTK desktop build cannot use the
|
||||
plugin at all. This package covers both halves and picks between them:
|
||||
[`google_account_token`](../google_account_token) wraps Android's AccountManager
|
||||
(no Google client library in the APK), so a real GTK desktop build cannot use
|
||||
it. This package covers both halves and picks between them:
|
||||
|
||||
| Platform | Flow | Entry point |
|
||||
|---|---|---|
|
||||
| Android / iOS | `google_sign_in` plugin, OS account picker | `googleIdToken` |
|
||||
| Android | `google_account_token` plugin, OS account chooser | `googleIdToken` |
|
||||
| Linux / Windows / macOS | OAuth **loopback** + PKCE, system browser | `googleDesktopIdToken` |
|
||||
| Web | none (GIS renders its own button) | falls back to password |
|
||||
|
||||
|
||||
@ -14,6 +14,8 @@ library;
|
||||
export 'src/account_store.dart';
|
||||
export 'src/bootstrap.dart';
|
||||
export 'src/device_id.dart';
|
||||
export 'src/e2e_bootstrap.dart';
|
||||
export 'src/e2e_passphrase_button.dart';
|
||||
export 'src/google_any_sign_in.dart';
|
||||
export 'src/google_desktop_sign_in.dart';
|
||||
export 'src/google_pkce.dart';
|
||||
|
||||
@ -108,9 +108,7 @@ Future<FirebaseAccount?> _adoptSeededSession(
|
||||
}) async {
|
||||
final httpClient = client ?? http.Client();
|
||||
try {
|
||||
final response = await httpClient.get(
|
||||
base.resolve(routes.credentialsPath),
|
||||
);
|
||||
final response = await httpClient.get(base.resolve(routes.credentialsPath));
|
||||
if (response.statusCode != 200) return null;
|
||||
final body = jsonDecode(utf8.decode(response.bodyBytes));
|
||||
if (body is! Map<String, dynamic>) return null;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user