mirror of
https://github.com/kuhyx/testsAndMisc.git
synced 2026-11-10 19:27:03 +01:00
feat(endurain): self-hosted run viewer fed by RunnerUp WebDAV exports
Stands up Endurain behind the existing Caddy edge and imports RunnerUp activities into it. Endurain is a VIEWER: screen-locker's workout gate still reads the phone over adb and is untouched here, because a WebDAV drop folder is spoofable by anything that can write to ~/cloud and must not become a screen-unlock authority. Primary path is phone -> WebDAV (existing dufs) -> ~/cloud/RunnerUp/ -> importer -> Endurain, with an adb pull as fallback when a run has not been uploaded yet. A systemd user timer runs the importer every 15 min. Findings that shaped the implementation, each verified against the running system rather than assumed: - Endurain moved to Codeberg; the GitHub repo is a read-only mirror stale at v0.17.7 against v0.19.0. The image is pulled from Codeberg. - postgres:18 moved PGDATA to /var/lib/postgresql/18/docker. Upstream's compose example still mounts the pre-18 path, which fails as an unexplained "container is unhealthy"; the real cause is only visible in docker logs. - Endurain seeds a hardcoded admin/admin via an Alembic migration with no env override, and endurain.kuhy.duckdns.org already resolved publicly. Publishing is therefore a separate script that refuses to install the Caddy site while those credentials still work. - Endurain performs NO duplicate detection, so the importer owns dedupe on two axes: a content hash, and an activity key that collapses the .gpx/.tcx pair RunnerUp writes for a single run (different bytes, same run). - Ambiguous uploads (5xx/timeout) are never re-queued, since the activity may already have been committed; only definite 4xx rejections are staged for retry. The importer exits non-zero so the timer surfaces the problem. - RunnerUp has no background sync at all, so every upload is a manual tap. - Its WebDAV synchronizer needs HTTPS with a real cert (Android blocks cleartext at targetSdk 36; self-signed is broken upstream) and a URL with no trailing slash, and cannot MKCOL against dufs - so the inbox must pre-exist. The phone gets a dufs credential scoped to /RunnerUp, verified unable to read Keepass or gdrive_backups. Verified: 52 tests at 100% branch coverage; the timer imports 4 real runs and re-runs create zero duplicates; https://endurain.kuhy.duckdns.org serves 200 with a valid cert and the other edge sites are unaffected; screen-locker's suite still passes 1090 tests at 100% coverage. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GbYEnfGnf1988cdFdGebot
This commit is contained in:
parent
ad55d1b6a1
commit
558883f281
6
.gitignore
vendored
6
.gitignore
vendored
@ -412,3 +412,9 @@ billsplit/linux/
|
|||||||
*.tsbuildinfo
|
*.tsbuildinfo
|
||||||
# CocoIndex Code (ccc)
|
# CocoIndex Code (ccc)
|
||||||
/.cocoindex_code/
|
/.cocoindex_code/
|
||||||
|
|
||||||
|
# Endurain: generated secrets (SECRET_KEY, FERNET_KEY, DB password) and the
|
||||||
|
# importer's API key + dedupe ledger. Never commit.
|
||||||
|
endurain/.env
|
||||||
|
endurain/.api_key
|
||||||
|
endurain/state/
|
||||||
|
|||||||
@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"title": "Endurain viewer fed by RunnerUp WebDAV exports",
|
||||||
|
"objective": "Stand up self-hosted Endurain behind the existing Caddy edge and import RunnerUp activities into it, primarily over WebDAV from the phone with an adb pull as fallback. Endurain is a VIEWER only: screen-locker's workout gate keeps reading the phone over adb and is not modified, because a WebDAV drop folder is spoofable by anything that can write to ~/cloud and must never become a screen-unlock authority. Success means real activities from the phone are visible in Endurain, importing twice never creates a duplicate, and screen-locker still passes its full suite.",
|
||||||
|
"acceptance_criteria": [
|
||||||
|
"Endurain reachable at https://endurain.kuhy.duckdns.org with a valid Let's Encrypt certificate",
|
||||||
|
"At least 4 real RunnerUp activities imported and visible, with correct distance and duration",
|
||||||
|
"Re-running the importer over already-imported files creates ZERO new activities",
|
||||||
|
"The .gpx and .tcx of a single run import as ONE activity, not two",
|
||||||
|
"The seeded admin/admin login is rejected before the service is ever published",
|
||||||
|
"The phone-facing WebDAV credential cannot read ~/cloud/Keepass or gdrive_backups",
|
||||||
|
"screen-locker's full test suite still passes at 100% coverage"
|
||||||
|
],
|
||||||
|
"out_of_scope": [
|
||||||
|
"Any edit to screen-locker's _runnerup_verification.py, _runnerup_tcx.py or _runnerup_db.py",
|
||||||
|
"Full history backfill out of RunnerUp's SQLite DB (the phone holds only 5 files)",
|
||||||
|
"Strava and Garmin Connect integration",
|
||||||
|
"Patching RunnerUp to add background auto-upload (it has none by design)"
|
||||||
|
],
|
||||||
|
"verifier": "pre-commit run --files <changed>; pytest python_pkg/endurain_import/tests/ (52 tests, 100% branch coverage); systemctl --user start endurain-import.service; psql count against endurain-postgres; screen-locker pytest suite (1090 tests)"
|
||||||
|
}
|
||||||
@ -0,0 +1,64 @@
|
|||||||
|
{
|
||||||
|
"intent": "Give RunnerUp runs a self-hosted dashboard (Endurain) fed over WebDAV from the phone, without touching the adb-based workout gate that unlocks the PC.",
|
||||||
|
"scope": [
|
||||||
|
"endurain/ - compose, setup and publish scripts, systemd units, README",
|
||||||
|
"python_pkg/endurain_import/ - importer package and its tests",
|
||||||
|
"~/.config/dufs/dufs.yaml - one added scoped auth entry (not in this repo)",
|
||||||
|
"~/gitea/sites/endurain.caddy - new edge site (not in this repo)",
|
||||||
|
"NON-GOAL: screen-locker remains byte-for-byte unmodified"
|
||||||
|
],
|
||||||
|
"changes": [
|
||||||
|
"docker-compose.yml pinned to codeberg.org (the GitHub repo is a stale read-only mirror at v0.17.7 vs v0.19.0), bound to 127.0.0.1:8085 so the LAN cannot reach it unfronted, redis renamed endurain-redis to avoid a container name collision",
|
||||||
|
"postgres volume mounted at /var/lib/postgresql/18/docker instead of upstream's pre-18 /var/lib/postgresql/data; upstream's example fails as an unexplained 'container is unhealthy' because mounting over that path leaves it root-owned",
|
||||||
|
"setup_endurain.sh generates secrets, chowns each bind mount to the uid its own image runs as (1000 app, 999 postgres, 999 redis), and gates on the seeded admin/admin login",
|
||||||
|
"publish_endurain.sh installs the Caddy site only after that gate passes, and validates the config before reloading so a bad snippet cannot take down gitea/cloud/website",
|
||||||
|
"importer dedupes on TWO axes: a SHA-256 content hash, and an activity key that collapses the .gpx/.tcx pair RunnerUp writes for one run (different bytes, same run) and strips the device model RunnerUp's WebDAV names embed",
|
||||||
|
"ambiguous upload outcomes (5xx/timeout) are never re-queued to bulk_import, because Endurain has no duplicate detection and the activity may already be committed; only definite 4xx rejections are staged",
|
||||||
|
"second dufs auth entry scoped to /RunnerUp so the phone credential cannot read Keepass or gdrive_backups; setup_dufs_cloud.sh patched to preserve it across re-runs"
|
||||||
|
],
|
||||||
|
"verification": [
|
||||||
|
{
|
||||||
|
"command": "pre-commit run --files <staged>",
|
||||||
|
"result": "pass",
|
||||||
|
"evidence": "ruff, ruff format, mypy, pylint, bandit, pytest-with-coverage, shellcheck, leaked-secrets and python_pkg-location hooks all Passed"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"command": "pytest python_pkg/endurain_import/tests/",
|
||||||
|
"result": "pass",
|
||||||
|
"evidence": "52 passed; 100% branch coverage on all 5 modules"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"command": "systemctl --user start endurain-import.service (real deploy path)",
|
||||||
|
"result": "pass",
|
||||||
|
"evidence": "first run: 4 imported, 1 already known (the .gpx twin correctly skipped); re-run: 0 imported, 5 already known; DB count stayed 4 with no duplicate start_times"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"command": "curl https://endurain.kuhy.duckdns.org/",
|
||||||
|
"result": "pass",
|
||||||
|
"evidence": "HTTP 200, ssl_verify_result=0, issuer Let's Encrypt; gitea/cloud/dice on the same edge still 200/401/200"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"command": "curl -u runnerup:*** -X PROPFIND https://kuhy-cloud.duckdns.org/{RunnerUp,Keepass}",
|
||||||
|
"result": "pass",
|
||||||
|
"evidence": "207 on /RunnerUp, PUT 201, 403 on /Keepass and on a direct Keepass file GET; root PROPFIND lists only / and /RunnerUp"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"command": "pytest screen_locker/tests (regression guard)",
|
||||||
|
"result": "pass",
|
||||||
|
"evidence": "1090 passed, 100% coverage; _runnerup_*.py unmodified per git status; all 5 phone files intact after the adb pulls"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"risks": [
|
||||||
|
"RunnerUp has NO background sync (no WorkManager/JobScheduler/AlarmManager anywhere in its source), so every upload needs a manual tap after a run. Not fixable without patching the app.",
|
||||||
|
"The ledger JSON is the only thing preventing duplicates; deleting it and re-importing WILL duplicate every activity, since Endurain does not deduplicate.",
|
||||||
|
"Endurain's OpenAPI schema mislabels the API-key header as X-Client-Type; the working header is X-API-Key, established empirically. A future version could change this.",
|
||||||
|
"The service is now internet-reachable. Its admin password is the only thing protecting it, and Endurain seeds admin/admin with no env override on a fresh database."
|
||||||
|
],
|
||||||
|
"rollback": [
|
||||||
|
"systemctl --user disable --now endurain-import.timer",
|
||||||
|
"rm ~/gitea/sites/endurain.caddy && docker exec gitea-caddy caddy reload --config /etc/caddy/Caddyfile # unpublishes immediately",
|
||||||
|
"docker compose -f endurain/docker-compose.yml down; sudo rm -rf /var/opt/endurain # destroys all imported activities",
|
||||||
|
"Remove the runnerup line from ~/.config/dufs/dufs.yaml and restart dufs",
|
||||||
|
"After rollback, confirm screen-locker still verifies a workout: pytest screen_locker/tests -k runnerup"
|
||||||
|
]
|
||||||
|
}
|
||||||
68
endurain/README.md
Normal file
68
endurain/README.md
Normal file
@ -0,0 +1,68 @@
|
|||||||
|
# Endurain + RunnerUp WebDAV sync
|
||||||
|
|
||||||
|
Self-hosted fitness tracker ([Endurain](https://codeberg.org/endurain-project/endurain))
|
||||||
|
fed by RunnerUp exports from the phone.
|
||||||
|
|
||||||
|
**Endurain is a viewer.** `screen-locker`'s workout gate still reads the phone
|
||||||
|
over adb and is untouched by anything here — a WebDAV drop folder is trivially
|
||||||
|
spoofable by anything that can write to `~/cloud`, so it must never become an
|
||||||
|
unlock authority.
|
||||||
|
|
||||||
|
## Paths
|
||||||
|
|
||||||
|
| What | Where |
|
||||||
|
|---|---|
|
||||||
|
| Public URL | `https://endurain.kuhy.duckdns.org` |
|
||||||
|
| Local URL | `http://127.0.0.1:8085` (loopback only) |
|
||||||
|
| WebDAV inbox | `https://kuhy-cloud.duckdns.org/RunnerUp` → `~/cloud/RunnerUp/` |
|
||||||
|
| Runtime data | `/var/opt/endurain/` |
|
||||||
|
| Ledger | `~/.local/state/endurain-import/ledger.json` |
|
||||||
|
| Importer code | `../python_pkg/endurain_import/` (repo requires Python under `python_pkg/`) |
|
||||||
|
| Secrets | `.env`, `.api_key` (both gitignored, mode 600) |
|
||||||
|
|
||||||
|
## Flow
|
||||||
|
|
||||||
|
```
|
||||||
|
phone --(WebDAV, manual tap)--> dufs ~/cloud/RunnerUp/ --\
|
||||||
|
>-- importer --> Endurain
|
||||||
|
phone --(adb pull, fallback)--> ~/cloud/RunnerUp/ -------/
|
||||||
|
```
|
||||||
|
|
||||||
|
A systemd user timer (`endurain-import.timer`) runs every 15 min.
|
||||||
|
|
||||||
|
## Commands
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/setup_endurain.sh # provision / restart the stack
|
||||||
|
./scripts/publish_endurain.sh # install the Caddy site (gated)
|
||||||
|
systemctl --user start endurain-import.service # import now
|
||||||
|
journalctl --user -u endurain-import.service -n 30
|
||||||
|
cd .. && .venv/bin/python -m pytest python_pkg/endurain_import/tests/ -q # 52 tests
|
||||||
|
```
|
||||||
|
|
||||||
|
## Things that will bite you
|
||||||
|
|
||||||
|
- **RunnerUp has no background sync.** No `WorkManager`, `JobScheduler` or
|
||||||
|
`AlarmManager` anywhere in its source. Every upload is a manual tap on the
|
||||||
|
save/upload button after a run. This is not configurable.
|
||||||
|
- **The WebDAV URL must not end in a slash.** RunnerUp builds the target as
|
||||||
|
`url + fileBase + fileExt` where `fileBase` already starts with `/`.
|
||||||
|
- **`MKCOL` is not supported by dufs.** RunnerUp `PROPFIND`s the directory and
|
||||||
|
only issues `MKCOL` on 404, so `~/cloud/RunnerUp/` must exist on disk or
|
||||||
|
adding the account fails with an opaque generic error (upstream #1147/#1172).
|
||||||
|
- **HTTPS is mandatory.** RunnerUp targets `targetSdk 36` with no
|
||||||
|
`usesCleartextTraffic`, so Android blocks plain HTTP. Self-signed certs are
|
||||||
|
broken upstream (stock `OkHttpClient`, system trust store only) — this is why
|
||||||
|
it rides on the existing Let's Encrypt cert.
|
||||||
|
- **Endurain does not deduplicate.** Re-uploading a file creates a second
|
||||||
|
activity. All dedupe is the importer's job, on two axes: the content hash,
|
||||||
|
*and* an activity key that collapses the `.gpx`/`.tcx` pair RunnerUp writes
|
||||||
|
for a single run (different bytes, same run).
|
||||||
|
- **`X-API-Key`, not `X-Client-Type`.** Endurain's own OpenAPI schema declares
|
||||||
|
the `APIKeyHeader` scheme with the wrong name; verified against the server.
|
||||||
|
- **postgres:18 moved `PGDATA`** to `/var/lib/postgresql/18/docker`. Upstream's
|
||||||
|
compose example still mounts the pre-18 path, which fails as an unexplained
|
||||||
|
"container is unhealthy".
|
||||||
|
- **The admin account is seeded `admin`/`admin`** by an Alembic migration with
|
||||||
|
no env override. `publish_endurain.sh` refuses to expose the service until
|
||||||
|
that password is changed.
|
||||||
69
endurain/docker-compose.yml
Normal file
69
endurain/docker-compose.yml
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
# Endurain self-hosted fitness tracker.
|
||||||
|
#
|
||||||
|
# Based on the upstream docker-compose.yml.example from Codeberg (the GitHub
|
||||||
|
# repo is a read-only mirror and is stale). Deviations from upstream, all
|
||||||
|
# deliberate:
|
||||||
|
#
|
||||||
|
# 1. The app port is bound to 127.0.0.1 explicitly. Upstream's "8080:8080"
|
||||||
|
# binds every interface; this host runs Docker alongside a Caddy edge on
|
||||||
|
# the host network, so an unqualified mapping would expose Endurain
|
||||||
|
# unfronted to the LAN. Public access is via gitea-caddy ONLY
|
||||||
|
# (sites/endurain.caddy -> endurain.kuhy.duckdns.org).
|
||||||
|
# 2. The redis container is named endurain-redis, not upstream's bare
|
||||||
|
# "redis", which would collide with any other redis on this host.
|
||||||
|
# 3. user: "1000:1000" is set explicitly. v0.19.0 removed the UID/GID env
|
||||||
|
# vars, so the host bind mounts must be chown'd to match.
|
||||||
|
services:
|
||||||
|
endurain:
|
||||||
|
container_name: endurain-app
|
||||||
|
image: codeberg.org/endurain-project/endurain:latest
|
||||||
|
user: "1000:1000"
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
volumes:
|
||||||
|
- ${LOCAL_PATH:-/var/opt/endurain}/backend/data:/app/backend/data:Z
|
||||||
|
- ${LOCAL_PATH:-/var/opt/endurain}/backend/logs:/app/backend/logs:Z
|
||||||
|
ports:
|
||||||
|
# Loopback only — see note 1 above. Do NOT change to "8085:8080".
|
||||||
|
- "127.0.0.1:8085:8080"
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
redis:
|
||||||
|
condition: service_healthy
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
postgres:
|
||||||
|
image: docker.io/postgres:18@sha256:8ff36f3c66371cba71d20ceedccfc3de9669a68737607888c4ef0af93abe8e39
|
||||||
|
container_name: endurain-postgres
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -U endurain"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 5
|
||||||
|
volumes:
|
||||||
|
# postgres:18 moved its default PGDATA to /var/lib/postgresql/18/docker
|
||||||
|
# and ships /var/lib/postgresql as mode-1777 owned by uid 999. Upstream's
|
||||||
|
# example still mounts at the pre-18 /var/lib/postgresql/data and sets
|
||||||
|
# PGDATA=.../data/pgdata; mounting over that path leaves it root-owned, so
|
||||||
|
# the entrypoint dies with "mkdir: cannot create directory
|
||||||
|
# '/var/lib/postgresql/data': Permission denied" and the only symptom
|
||||||
|
# compose reports is "container endurain-postgres is unhealthy".
|
||||||
|
# Mount at the image's own path and let PGDATA keep its default.
|
||||||
|
- ${LOCAL_PATH:-/var/opt/endurain}/postgres:/var/lib/postgresql/18/docker:Z
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: redis:8-alpine@sha256:09160599abd229764c0fb44cb6be640294e1d360a54b19985ab4843dcf2d90f1
|
||||||
|
container_name: endurain-redis
|
||||||
|
command: ["redis-server", "--appendonly", "yes"]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "redis-cli", "ping"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 5
|
||||||
|
volumes:
|
||||||
|
- ${LOCAL_PATH:-/var/opt/endurain}/redis:/data:Z
|
||||||
|
restart: unless-stopped
|
||||||
104
endurain/scripts/publish_endurain.sh
Executable file
104
endurain/scripts/publish_endurain.sh
Executable file
@ -0,0 +1,104 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# Publish Endurain on the shared Caddy edge (endurain.kuhy.duckdns.org).
|
||||||
|
#
|
||||||
|
# Split out of setup_endurain.sh deliberately. endurain.kuhy.duckdns.org
|
||||||
|
# already resolves to this host via the duckdns wildcard on "kuhy", so
|
||||||
|
# installing the site snippet makes the service world-reachable the moment
|
||||||
|
# Caddy reloads. Endurain ships a hardcoded admin/admin account seeded by an
|
||||||
|
# Alembic migration with no environment override, so publishing before that
|
||||||
|
# password is changed hands the internet an admin login.
|
||||||
|
#
|
||||||
|
# This script therefore re-runs the same gate and REFUSES to publish while the
|
||||||
|
# seeded credentials still work.
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_NAME="$(basename "$0")"
|
||||||
|
readonly SCRIPT_NAME
|
||||||
|
readonly APP_URL="http://127.0.0.1:8085"
|
||||||
|
readonly PUBLIC_HOST="endurain.kuhy.duckdns.org"
|
||||||
|
readonly EDGE_DIR="${HOME}/gitea"
|
||||||
|
readonly SITE_FILE="${EDGE_DIR}/sites/endurain.caddy"
|
||||||
|
readonly EDGE_CONTAINER="gitea-caddy"
|
||||||
|
|
||||||
|
log() { printf '%s\n' "$*"; }
|
||||||
|
log_ok() { printf ' ok: %s\n' "$*"; }
|
||||||
|
die() {
|
||||||
|
printf 'Error: %s\n' "$*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo "Usage: ${SCRIPT_NAME}"
|
||||||
|
echo " Installs the Caddy site for ${PUBLIC_HOST} and reloads the edge."
|
||||||
|
exit 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# Fail closed. Only an explicit rejection counts as "safe"; any unexpected
|
||||||
|
# status is treated as still-vulnerable.
|
||||||
|
assert_admin_password_changed() {
|
||||||
|
local code
|
||||||
|
code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 \
|
||||||
|
-X POST "${APP_URL}/api/v1/auth/login" \
|
||||||
|
-H 'Content-Type: application/x-www-form-urlencoded' \
|
||||||
|
-H 'X-Client-Type: mobile' \
|
||||||
|
--data 'username=admin&password=admin' 2>/dev/null || echo 000)"
|
||||||
|
case "${code}" in
|
||||||
|
401 | 403) log_ok "seeded admin/admin is rejected" ;;
|
||||||
|
200) die "REFUSING TO PUBLISH: Endurain still accepts admin/admin.
|
||||||
|
Change it at ${APP_URL} first." ;;
|
||||||
|
*) die "REFUSING TO PUBLISH: unexpected status ${code} from the login
|
||||||
|
probe; cannot prove the seeded password was changed." ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_requirements() {
|
||||||
|
command -v docker >/dev/null || die "docker not installed"
|
||||||
|
[[ -d "${EDGE_DIR}/sites" ]] || die "Caddy edge not found at ${EDGE_DIR}/sites"
|
||||||
|
docker ps --format '{{.Names}}' | grep -qx "${EDGE_CONTAINER}" ||
|
||||||
|
die "${EDGE_CONTAINER} is not running"
|
||||||
|
curl -fsS --max-time 5 "${APP_URL}/api/v1/about" >/dev/null ||
|
||||||
|
die "Endurain is not responding on ${APP_URL}"
|
||||||
|
}
|
||||||
|
|
||||||
|
install_site() {
|
||||||
|
cat >"${SITE_FILE}" <<EOF
|
||||||
|
# Managed by ${SCRIPT_NAME} — do not edit by hand.
|
||||||
|
${PUBLIC_HOST} {
|
||||||
|
reverse_proxy 127.0.0.1:8085
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
log_ok "Wrote ${SITE_FILE}"
|
||||||
|
}
|
||||||
|
|
||||||
|
reload_edge() {
|
||||||
|
# Validate before reloading so a broken snippet cannot take the whole edge
|
||||||
|
# (gitea, the cloud, the website) down with it.
|
||||||
|
docker exec "${EDGE_CONTAINER}" caddy validate --config /etc/caddy/Caddyfile \
|
||||||
|
>/dev/null 2>&1 || die "Caddy config failed validation; edge NOT reloaded"
|
||||||
|
docker exec "${EDGE_CONTAINER}" caddy reload --config /etc/caddy/Caddyfile \
|
||||||
|
>/dev/null || die "caddy reload failed"
|
||||||
|
log_ok "Edge reloaded"
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
validate_requirements
|
||||||
|
assert_admin_password_changed
|
||||||
|
install_site
|
||||||
|
reload_edge
|
||||||
|
log ""
|
||||||
|
log "Published: https://${PUBLIC_HOST}"
|
||||||
|
log "Caddy obtains the certificate on the first request; allow a few seconds."
|
||||||
|
}
|
||||||
|
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case $1 in
|
||||||
|
-h | --help) usage ;;
|
||||||
|
*) die "Unknown option: $1" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
main "$@"
|
||||||
193
endurain/scripts/setup_endurain.sh
Executable file
193
endurain/scripts/setup_endurain.sh
Executable file
@ -0,0 +1,193 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# Provision the Endurain self-hosted fitness tracker.
|
||||||
|
#
|
||||||
|
# Idempotent: re-running keeps the existing .env (and therefore the existing
|
||||||
|
# database credentials — regenerating them would orphan the Postgres volume).
|
||||||
|
#
|
||||||
|
# ORDERING IS LOAD-BEARING. Endurain seeds a hardcoded admin/admin account via
|
||||||
|
# an Alembic migration with no env override, and endurain.kuhy.duckdns.org
|
||||||
|
# already resolves publicly. So this script brings the stack up on loopback
|
||||||
|
# ONLY and refuses to install the Caddy site; publishing is a separate,
|
||||||
|
# deliberate step taken after the admin password has been changed.
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_NAME="$(basename "$0")"
|
||||||
|
REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
readonly SCRIPT_NAME REPO_DIR
|
||||||
|
readonly ENV_FILE="${REPO_DIR}/.env"
|
||||||
|
readonly LOCAL_PATH="${ENDURAIN_LOCAL_PATH:-/var/opt/endurain}"
|
||||||
|
readonly APP_URL="http://127.0.0.1:8085"
|
||||||
|
readonly PUBLIC_HOST="endurain.kuhy.duckdns.org"
|
||||||
|
|
||||||
|
log() { printf '%s\n' "$*"; }
|
||||||
|
log_ok() { printf ' ok: %s\n' "$*"; }
|
||||||
|
die() {
|
||||||
|
printf 'Error: %s\n' "$*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo "Usage: ${SCRIPT_NAME} [--recreate-env]"
|
||||||
|
echo " --recreate-env Regenerate secrets. DESTRUCTIVE: orphans the existing DB."
|
||||||
|
exit 0
|
||||||
|
}
|
||||||
|
|
||||||
|
RECREATE_ENV=0
|
||||||
|
|
||||||
|
validate_requirements() {
|
||||||
|
command -v docker >/dev/null || die "docker not installed"
|
||||||
|
docker compose version >/dev/null 2>&1 || die "docker compose plugin missing"
|
||||||
|
command -v openssl >/dev/null || die "openssl not installed"
|
||||||
|
# Fail early rather than after the volumes are created.
|
||||||
|
docker info >/dev/null 2>&1 || die "cannot talk to the docker daemon"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Secrets are generated once and never printed to a file outside .env.
|
||||||
|
write_env() {
|
||||||
|
if [[ -f "${ENV_FILE}" && "${RECREATE_ENV}" -eq 0 ]]; then
|
||||||
|
log_ok ".env already exists — keeping existing credentials"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
local db_password secret_key fernet_key
|
||||||
|
db_password="$(openssl rand -hex 32)"
|
||||||
|
secret_key="$(openssl rand -hex 32)"
|
||||||
|
fernet_key="$(openssl rand -base64 32)"
|
||||||
|
|
||||||
|
umask 077
|
||||||
|
cat >"${ENV_FILE}" <<EOF
|
||||||
|
# Generated by ${SCRIPT_NAME} — gitignored, never commit.
|
||||||
|
DB_PASSWORD=${db_password}
|
||||||
|
POSTGRES_PASSWORD=${db_password}
|
||||||
|
SECRET_KEY=${secret_key}
|
||||||
|
FERNET_KEY=${fernet_key}
|
||||||
|
|
||||||
|
TZ=Europe/Warsaw
|
||||||
|
ENDURAIN_HOST=https://${PUBLIC_HOST}
|
||||||
|
ENVIRONMENT=production
|
||||||
|
BEHIND_PROXY=true
|
||||||
|
# Upstream defaults to "*"; only the local Caddy edge fronts this service.
|
||||||
|
TRUSTED_PROXIES=127.0.0.1
|
||||||
|
|
||||||
|
RATE_LIMIT_STORAGE_URI=redis://redis:6379/0
|
||||||
|
AUTH_SECURITY_STORAGE_URI=redis://redis:6379/0
|
||||||
|
POSTGRES_DB=endurain
|
||||||
|
POSTGRES_USER=endurain
|
||||||
|
# PGDATA intentionally NOT set: postgres:18 defaults to
|
||||||
|
# /var/lib/postgresql/18/docker, which is where the volume is mounted.
|
||||||
|
LOCAL_PATH=${LOCAL_PATH}
|
||||||
|
EOF
|
||||||
|
chmod 600 "${ENV_FILE}"
|
||||||
|
log_ok "Wrote ${ENV_FILE} (secrets generated, mode 600)"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Each image runs as a different non-root uid, so the bind mounts must be
|
||||||
|
# chown'd per-service or the container fails to create its data directory:
|
||||||
|
# endurain -> 1000 (v0.19.0 removed the UID/GID env vars)
|
||||||
|
# postgres -> 999 (image drops privileges internally after entrypoint)
|
||||||
|
# redis -> 999
|
||||||
|
# Getting this wrong surfaces as "dependency failed to start: container
|
||||||
|
# endurain-postgres is unhealthy", with the real cause (mkdir: Permission
|
||||||
|
# denied) visible only in `docker logs endurain-postgres`.
|
||||||
|
readonly POSTGRES_UID=999
|
||||||
|
readonly REDIS_UID=999
|
||||||
|
readonly ENDURAIN_UID=1000
|
||||||
|
|
||||||
|
prepare_data_dirs() {
|
||||||
|
local d
|
||||||
|
for d in "${LOCAL_PATH}/backend/data" "${LOCAL_PATH}/backend/logs" \
|
||||||
|
"${LOCAL_PATH}/postgres" "${LOCAL_PATH}/redis"; do
|
||||||
|
sudo mkdir -p "${d}"
|
||||||
|
done
|
||||||
|
sudo chown -R "${ENDURAIN_UID}:${ENDURAIN_UID}" "${LOCAL_PATH}/backend"
|
||||||
|
sudo chown -R "${POSTGRES_UID}:${POSTGRES_UID}" "${LOCAL_PATH}/postgres"
|
||||||
|
sudo chown -R "${REDIS_UID}:${REDIS_UID}" "${LOCAL_PATH}/redis"
|
||||||
|
log_ok "Data directories ready under ${LOCAL_PATH}"
|
||||||
|
}
|
||||||
|
|
||||||
|
start_stack() {
|
||||||
|
docker compose -f "${REPO_DIR}/docker-compose.yml" up -d
|
||||||
|
log_ok "Stack started"
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_for_health() {
|
||||||
|
for _ in $(seq 1 60); do
|
||||||
|
if curl -fsS --max-time 5 "${APP_URL}/api/v1/about" >/dev/null 2>&1; then
|
||||||
|
log_ok "Endurain healthy at ${APP_URL}"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
die "Endurain did not become healthy within 120s. Check: docker logs endurain-app"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Fail closed: if the seeded admin/admin still works, the service must not be
|
||||||
|
# published. This is a gate, not a warning.
|
||||||
|
#
|
||||||
|
# The endpoint is /api/v1/auth/login with X-Client-Type: mobile. Two traps here,
|
||||||
|
# both of which produce a FALSE PASS (gate says "safe" while admin/admin works):
|
||||||
|
# - /api/v1/token does not exist; it falls through to the SPA index.
|
||||||
|
# - Without X-Client-Type: mobile the API answers 403 "CSRF token required"
|
||||||
|
# before ever checking the credentials.
|
||||||
|
# So only an explicit 200 means "still vulnerable", and any unexpected status is
|
||||||
|
# treated as vulnerable rather than assumed safe.
|
||||||
|
check_admin_password_changed() {
|
||||||
|
local code
|
||||||
|
code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 \
|
||||||
|
-X POST "${APP_URL}/api/v1/auth/login" \
|
||||||
|
-H 'Content-Type: application/x-www-form-urlencoded' \
|
||||||
|
-H 'X-Client-Type: mobile' \
|
||||||
|
--data 'username=admin&password=admin' 2>/dev/null || echo 000)"
|
||||||
|
case "${code}" in
|
||||||
|
401 | 403) return 0 ;; # credentials rejected — password was changed
|
||||||
|
200) return 1 ;; # seeded admin/admin still works
|
||||||
|
*)
|
||||||
|
log " (unexpected status ${code} from the login probe)"
|
||||||
|
return 1
|
||||||
|
;; # fail closed on anything unexpected
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
validate_requirements
|
||||||
|
log "=== Endurain setup ==="
|
||||||
|
write_env
|
||||||
|
prepare_data_dirs
|
||||||
|
start_stack
|
||||||
|
wait_for_health
|
||||||
|
|
||||||
|
log ""
|
||||||
|
if check_admin_password_changed; then
|
||||||
|
log_ok "Seeded admin/admin no longer accepted"
|
||||||
|
log ""
|
||||||
|
log "Next: install the Caddy site to publish it, then configure the phone."
|
||||||
|
log " ${REPO_DIR}/scripts/publish_endurain.sh"
|
||||||
|
else
|
||||||
|
log "!! ACTION REQUIRED — the service is NOT safe to publish yet."
|
||||||
|
log " Endurain still accepts the seeded admin/admin login."
|
||||||
|
log " 1. Open ${APP_URL} and log in as admin/admin"
|
||||||
|
log " 2. Change the admin password immediately (store it in Keepass)"
|
||||||
|
log " 3. Settings -> Security -> API Keys: create a key scoped"
|
||||||
|
log " 'activities:upload' and save it for the importer"
|
||||||
|
log " 4. Re-run this script to confirm, then publish."
|
||||||
|
log ""
|
||||||
|
log " The Caddy site is deliberately NOT installed while this is true:"
|
||||||
|
log " https://${PUBLIC_HOST} already resolves to this host."
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case $1 in
|
||||||
|
--recreate-env)
|
||||||
|
RECREATE_ENV=1
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
-h | --help) usage ;;
|
||||||
|
*) die "Unknown option: $1" ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
main "$@"
|
||||||
21
endurain/systemd/endurain-import.service
Normal file
21
endurain/systemd/endurain-import.service
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Import RunnerUp activities into Endurain
|
||||||
|
Documentation=file:///home/kuhy/testsAndMisc/endurain/README.md
|
||||||
|
# Endurain runs in Docker; without this the timer fires during boot before the
|
||||||
|
# stack is listening and logs a spurious "not reachable" failure.
|
||||||
|
After=network-online.target docker.service
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/home/kuhy/testsAndMisc
|
||||||
|
Environment=ENDURAIN_URL=http://127.0.0.1:8085
|
||||||
|
Environment=ENDURAIN_INBOX=/home/kuhy/cloud/RunnerUp
|
||||||
|
Environment=ENDURAIN_STATE=/home/kuhy/.local/state/endurain-import
|
||||||
|
Environment=ENDURAIN_BULK_DIR=/var/opt/endurain/backend/data/activity_files/bulk_import
|
||||||
|
# The API key is read from a mode-600 file rather than embedded here: systemd
|
||||||
|
# unit files are world-readable.
|
||||||
|
ExecStart=/usr/bin/bash -c 'ENDURAIN_API_KEY="$(cat /home/kuhy/testsAndMisc/endurain/.api_key)" /home/kuhy/testsAndMisc/.venv/bin/python -m python_pkg.endurain_import'
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=default.target
|
||||||
13
endurain/systemd/endurain-import.timer
Normal file
13
endurain/systemd/endurain-import.timer
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Import RunnerUp activities into Endurain every 15 minutes
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=5min
|
||||||
|
OnUnitActiveSec=15min
|
||||||
|
# The phone uploads on a manual tap, so the exact minute does not matter;
|
||||||
|
# jitter avoids lining up with every other timer on this box.
|
||||||
|
RandomizedDelaySec=60
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@ -64,6 +64,10 @@ unfixable = []
|
|||||||
"S101", # assert IS what tests do — every Python test suite suppresses this.
|
"S101", # assert IS what tests do — every Python test suite suppresses this.
|
||||||
"SLF001", # Unit tests must exercise private internals (_method, _attr) to reach
|
"SLF001", # Unit tests must exercise private internals (_method, _attr) to reach
|
||||||
# 100% branch coverage; only integration tests can avoid this.
|
# 100% branch coverage; only integration tests can avoid this.
|
||||||
|
"TC002", # pytest resolves fixture parameter annotations at RUNTIME, so moving
|
||||||
|
"TC003", # pytest/Path into a TYPE_CHECKING block breaks collection. Verified
|
||||||
|
# empirically: guarding them failed 11 tests with UnboundLocalError
|
||||||
|
# and NameError, not a hypothetical concern.
|
||||||
]
|
]
|
||||||
"**/test_*.py" = [
|
"**/test_*.py" = [
|
||||||
"ARG",
|
"ARG",
|
||||||
|
|||||||
11
python_pkg/endurain_import/__init__.py
Normal file
11
python_pkg/endurain_import/__init__.py
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
"""RunnerUp -> Endurain importer.
|
||||||
|
|
||||||
|
Moves RunnerUp activity exports into Endurain exactly once, from two sources:
|
||||||
|
|
||||||
|
1. the WebDAV inbox the phone uploads to (primary), and
|
||||||
|
2. ``adb pull`` from the phone's RunnerUp export dir (fallback).
|
||||||
|
|
||||||
|
Endurain performs no duplicate detection of its own -- re-importing a file
|
||||||
|
creates a second activity -- so deduplication is entirely this package's
|
||||||
|
responsibility. See :mod:`ledger`.
|
||||||
|
"""
|
||||||
170
python_pkg/endurain_import/__main__.py
Normal file
170
python_pkg/endurain_import/__main__.py
Normal file
@ -0,0 +1,170 @@
|
|||||||
|
"""Import RunnerUp activity files into Endurain, exactly once each.
|
||||||
|
|
||||||
|
Run from a systemd user timer. Configuration comes from the environment:
|
||||||
|
|
||||||
|
ENDURAIN_URL base URL (default http://127.0.0.1:8085)
|
||||||
|
ENDURAIN_API_KEY API key with the activities:upload scope (required)
|
||||||
|
ENDURAIN_INBOX WebDAV inbox (default ~/cloud/RunnerUp)
|
||||||
|
ENDURAIN_STATE ledger directory (default ~/.local/state/endurain-import)
|
||||||
|
ENDURAIN_BULK_DIR bulk_import dir for files the server rejected
|
||||||
|
ENDURAIN_NO_ADB set to 1 to disable the phone fallback
|
||||||
|
|
||||||
|
Exit status is 0 only when every file was accounted for; anything left in an
|
||||||
|
ambiguous state exits non-zero so the timer surfaces it rather than logging
|
||||||
|
quietly and moving on.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from python_pkg.endurain_import.ledger import (
|
||||||
|
Entry,
|
||||||
|
Ledger,
|
||||||
|
activity_key,
|
||||||
|
file_digest,
|
||||||
|
now_iso,
|
||||||
|
)
|
||||||
|
from python_pkg.endurain_import.sources import inbox_files, pull_from_phone
|
||||||
|
from python_pkg.endurain_import.upload import EndurainClient, Outcome, SupportsUpload
|
||||||
|
|
||||||
|
_logger = logging.getLogger("endurain_import")
|
||||||
|
|
||||||
|
_DEFAULT_URL = "http://127.0.0.1:8085"
|
||||||
|
_DEFAULT_INBOX = Path.home() / "cloud" / "RunnerUp"
|
||||||
|
_DEFAULT_STATE = Path.home() / ".local" / "state" / "endurain-import"
|
||||||
|
|
||||||
|
|
||||||
|
def _configure_logging() -> None:
|
||||||
|
logging.basicConfig(
|
||||||
|
level=logging.INFO,
|
||||||
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _require_api_key() -> str:
|
||||||
|
key = os.environ.get("ENDURAIN_API_KEY", "").strip()
|
||||||
|
if not key:
|
||||||
|
_logger.error(
|
||||||
|
"ENDURAIN_API_KEY is not set. Create one in Endurain under "
|
||||||
|
"Settings -> Security -> API Keys with the activities:upload scope."
|
||||||
|
)
|
||||||
|
raise SystemExit(2)
|
||||||
|
return key
|
||||||
|
|
||||||
|
|
||||||
|
def _route_rejected(path: Path, bulk_dir: Path | None) -> None:
|
||||||
|
"""Copy a server-rejected file into bulk_import for manual recovery.
|
||||||
|
|
||||||
|
Only ever called for a definite 4xx. Ambiguous failures must not land here:
|
||||||
|
Endurain would import them a second time if the first attempt had in fact
|
||||||
|
been committed.
|
||||||
|
"""
|
||||||
|
if bulk_dir is None:
|
||||||
|
_logger.warning("no bulk_import dir configured; leaving %s", path.name)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
bulk_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.copy2(path, bulk_dir / path.name)
|
||||||
|
_logger.warning("copied %s to bulk_import for manual retry", path.name)
|
||||||
|
except OSError:
|
||||||
|
_logger.exception("could not stage %s for bulk import", path.name)
|
||||||
|
|
||||||
|
|
||||||
|
def _process(
|
||||||
|
path: Path,
|
||||||
|
client: SupportsUpload,
|
||||||
|
ledger: Ledger,
|
||||||
|
processed_dir: Path,
|
||||||
|
bulk_dir: Path | None,
|
||||||
|
) -> str:
|
||||||
|
"""Import one file. Returns one of: skipped, imported, rejected, ambiguous."""
|
||||||
|
digest = file_digest(path)
|
||||||
|
key = activity_key(path)
|
||||||
|
# Two independent dedupe checks. The digest catches the identical file
|
||||||
|
# arriving twice; the activity key catches the SAME RUN arriving as both
|
||||||
|
# .gpx and .tcx, whose bytes differ and which Endurain would otherwise
|
||||||
|
# import as two separate activities.
|
||||||
|
if digest in ledger or ledger.has_activity(key):
|
||||||
|
processed_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.move(str(path), str(processed_dir / path.name))
|
||||||
|
_logger.info("%s already imported; moved to processed/", path.name)
|
||||||
|
return "skipped"
|
||||||
|
|
||||||
|
result = client.upload(path)
|
||||||
|
if result.outcome is Outcome.OK:
|
||||||
|
ledger.record(Entry(digest, key, path.name, result.activity_id, now_iso()))
|
||||||
|
processed_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.move(str(path), str(processed_dir / path.name))
|
||||||
|
_logger.info("imported %s (activity_id=%s)", path.name, result.activity_id)
|
||||||
|
return "imported"
|
||||||
|
|
||||||
|
if result.outcome is Outcome.REJECTED:
|
||||||
|
_logger.error("rejected %s -- %s", path.name, result.detail)
|
||||||
|
_route_rejected(path, bulk_dir)
|
||||||
|
return "rejected"
|
||||||
|
|
||||||
|
# Ambiguous: leave the file exactly where it is and do NOT record it.
|
||||||
|
_logger.error(
|
||||||
|
"ambiguous outcome for %s -- %s. Left in the inbox; verify in Endurain "
|
||||||
|
"before the next run to avoid a duplicate.",
|
||||||
|
path.name,
|
||||||
|
result.detail,
|
||||||
|
)
|
||||||
|
return "ambiguous"
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
"""Run one import pass. Returns the process exit status."""
|
||||||
|
_configure_logging()
|
||||||
|
api_key = _require_api_key()
|
||||||
|
base_url = os.environ.get("ENDURAIN_URL", _DEFAULT_URL)
|
||||||
|
inbox = Path(os.environ.get("ENDURAIN_INBOX", _DEFAULT_INBOX))
|
||||||
|
state = Path(os.environ.get("ENDURAIN_STATE", _DEFAULT_STATE))
|
||||||
|
bulk = os.environ.get("ENDURAIN_BULK_DIR")
|
||||||
|
bulk_dir = Path(bulk) if bulk else None
|
||||||
|
|
||||||
|
client = EndurainClient(base_url, api_key)
|
||||||
|
try:
|
||||||
|
client.about()
|
||||||
|
# requests.RequestException covers the transport failures; OSError/ValueError
|
||||||
|
# cover a malformed response body. ConnectionError subclasses OSError, so a
|
||||||
|
# separate requests handler would be unreachable.
|
||||||
|
except (OSError, ValueError, RuntimeError, requests.RequestException):
|
||||||
|
_logger.exception("Endurain is not reachable at %s", base_url)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
ledger = Ledger(state / "ledger.json")
|
||||||
|
processed_dir = inbox / "processed"
|
||||||
|
|
||||||
|
if os.environ.get("ENDURAIN_NO_ADB") != "1":
|
||||||
|
pull_from_phone(inbox)
|
||||||
|
|
||||||
|
# TCX carries more than GPX (laps, cadence, HR), so when a run is present
|
||||||
|
# in both formats the TCX must be the one that wins the dedupe race.
|
||||||
|
pending = sorted(inbox_files(inbox), key=lambda p: p.suffix.lower() != ".tcx")
|
||||||
|
if not pending:
|
||||||
|
_logger.info("nothing to import (ledger holds %d file(s))", len(ledger))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
counts = {"skipped": 0, "imported": 0, "rejected": 0, "ambiguous": 0}
|
||||||
|
for path in pending:
|
||||||
|
counts[_process(path, client, ledger, processed_dir, bulk_dir)] += 1
|
||||||
|
|
||||||
|
_logger.info(
|
||||||
|
"done: %(imported)d imported, %(skipped)d already known, "
|
||||||
|
"%(rejected)d rejected, %(ambiguous)d ambiguous",
|
||||||
|
counts,
|
||||||
|
)
|
||||||
|
# Fail closed so the timer reports a problem instead of hiding it.
|
||||||
|
return 1 if counts["ambiguous"] or counts["rejected"] else 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
134
python_pkg/endurain_import/ledger.py
Normal file
134
python_pkg/endurain_import/ledger.py
Normal file
@ -0,0 +1,134 @@
|
|||||||
|
"""Content-addressed record of what has already been imported.
|
||||||
|
|
||||||
|
Endurain has no duplicate detection: uploading the same file twice creates two
|
||||||
|
activities. The ledger is therefore the only thing standing between a re-run
|
||||||
|
and a duplicated activity history, and it keys on the file's SHA-256 rather
|
||||||
|
than its name so that the same run imports once even when it arrives by both
|
||||||
|
the WebDAV and adb routes under different filenames.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_CHUNK = 1 << 20
|
||||||
|
# "RunnerUp", <timestamp>, <sport> -- the minimum for a parseable export name.
|
||||||
|
_MIN_NAME_PARTS = 3
|
||||||
|
|
||||||
|
|
||||||
|
class LedgerCorruptError(RuntimeError):
|
||||||
|
"""Raised when the on-disk ledger cannot be parsed."""
|
||||||
|
|
||||||
|
|
||||||
|
def activity_key(path: Path) -> str:
|
||||||
|
"""Return a stable identity for the *run*, independent of export format.
|
||||||
|
|
||||||
|
RunnerUp writes the same activity as both .gpx and .tcx
|
||||||
|
(RunnerUp_2026-08-22-23-51-04_Running.{gpx,tcx}). Those files have
|
||||||
|
different bytes, so a content hash sees two distinct files and Endurain --
|
||||||
|
which does no duplicate detection -- happily creates two activities for one
|
||||||
|
run. Keying on the stem collapses them.
|
||||||
|
|
||||||
|
The device model that RunnerUp's WebDAV synchronizer injects
|
||||||
|
(RunnerUp_Pixel_6a_<ts>_Running) is stripped too, so the same run arriving
|
||||||
|
over WebDAV and over adb resolves to one key.
|
||||||
|
"""
|
||||||
|
stem = path.stem
|
||||||
|
parts = stem.split("_")
|
||||||
|
if len(parts) >= _MIN_NAME_PARTS and parts[0] == "RunnerUp":
|
||||||
|
# Keep the trailing <timestamp>_<sport>; drop any model in between.
|
||||||
|
return f"RunnerUp_{parts[-2]}_{parts[-1]}"
|
||||||
|
return stem
|
||||||
|
|
||||||
|
|
||||||
|
def file_digest(path: Path) -> str:
|
||||||
|
"""Return the SHA-256 of ``path``, read incrementally."""
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
while chunk := handle.read(_CHUNK):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Entry:
|
||||||
|
"""One successfully imported file."""
|
||||||
|
|
||||||
|
sha256: str
|
||||||
|
activity_key: str
|
||||||
|
filename: str
|
||||||
|
activity_id: int | None
|
||||||
|
uploaded_at: str
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, Any]:
|
||||||
|
"""Return the JSON-serialisable form stored in the ledger."""
|
||||||
|
return {
|
||||||
|
"activity_key": self.activity_key,
|
||||||
|
"filename": self.filename,
|
||||||
|
"activity_id": self.activity_id,
|
||||||
|
"uploaded_at": self.uploaded_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class Ledger:
|
||||||
|
"""A JSON-backed set of imported content hashes.
|
||||||
|
|
||||||
|
Writes are atomic (temp file + replace) because the importer runs from a
|
||||||
|
systemd timer: a crash mid-write must not leave a truncated ledger, which
|
||||||
|
would look like "nothing has been imported" and duplicate the entire
|
||||||
|
history on the next run.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, path: Path) -> None:
|
||||||
|
"""Open (or create) the ledger stored at ``path``."""
|
||||||
|
self._path = path
|
||||||
|
self._entries: dict[str, dict[str, Any]] = {}
|
||||||
|
self._load()
|
||||||
|
|
||||||
|
def _load(self) -> None:
|
||||||
|
if not self._path.exists():
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
raw = json.loads(self._path.read_text())
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
# Fail closed: a corrupt ledger must stop the run, not silently
|
||||||
|
# re-import everything.
|
||||||
|
message = f"corrupt ledger at {self._path}: {exc}"
|
||||||
|
raise LedgerCorruptError(message) from exc
|
||||||
|
if isinstance(raw, dict):
|
||||||
|
self._entries = raw
|
||||||
|
|
||||||
|
def __contains__(self, sha256: str) -> bool:
|
||||||
|
"""True when a file with this digest has already been imported."""
|
||||||
|
return sha256 in self._entries
|
||||||
|
|
||||||
|
def has_activity(self, key: str) -> bool:
|
||||||
|
"""True when this run was imported already, in any export format."""
|
||||||
|
return any(entry.get("activity_key") == key for entry in self._entries.values())
|
||||||
|
|
||||||
|
def __len__(self) -> int:
|
||||||
|
"""Number of files recorded as imported."""
|
||||||
|
return len(self._entries)
|
||||||
|
|
||||||
|
def record(self, entry: Entry) -> None:
|
||||||
|
"""Add ``entry`` and flush to disk immediately."""
|
||||||
|
self._entries[entry.sha256] = entry.as_dict()
|
||||||
|
self._flush()
|
||||||
|
|
||||||
|
def _flush(self) -> None:
|
||||||
|
self._path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
tmp = self._path.with_suffix(".tmp")
|
||||||
|
tmp.write_text(json.dumps(self._entries, indent=2, sort_keys=True))
|
||||||
|
tmp.replace(self._path)
|
||||||
|
|
||||||
|
|
||||||
|
def now_iso() -> str:
|
||||||
|
"""UTC timestamp for ledger entries."""
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
117
python_pkg/endurain_import/sources.py
Normal file
117
python_pkg/endurain_import/sources.py
Normal file
@ -0,0 +1,117 @@
|
|||||||
|
"""Where activity files come from.
|
||||||
|
|
||||||
|
Primary source is the WebDAV inbox the phone uploads into. The adb fallback
|
||||||
|
exists because RunnerUp has no background sync -- uploads only happen when the
|
||||||
|
user taps the upload button -- so a run can sit on the phone indefinitely. When
|
||||||
|
the phone is attached, its export directory is pulled into the same inbox and
|
||||||
|
from there follows the identical path, deduplicated by content hash.
|
||||||
|
|
||||||
|
This module is strictly read-only with respect to the phone. screen-locker's
|
||||||
|
workout verification reads the same directory over adb and is the gate that
|
||||||
|
unlocks the PC; nothing here may delete, move, or rewrite those files.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# RunnerUp's File synchronizer target on the phone.
|
||||||
|
PHONE_EXPORT_DIR = "/sdcard/Documents/RunnerUp"
|
||||||
|
ACTIVITY_SUFFIXES = (".tcx", ".gpx", ".fit")
|
||||||
|
_ADB_TIMEOUT = 60
|
||||||
|
# Resolved at call time so a missing adb is reported as "no device" rather than
|
||||||
|
# crashing the whole import run.
|
||||||
|
_ADB_BIN = shutil.which("adb") or "adb"
|
||||||
|
|
||||||
|
|
||||||
|
def inbox_files(inbox: Path) -> list[Path]:
|
||||||
|
"""Return activity files waiting in the WebDAV inbox, oldest first.
|
||||||
|
|
||||||
|
``processed/`` is skipped: it holds files this importer has already
|
||||||
|
delivered and is not rescanned.
|
||||||
|
"""
|
||||||
|
if not inbox.is_dir():
|
||||||
|
return []
|
||||||
|
found = [
|
||||||
|
path
|
||||||
|
for path in inbox.iterdir()
|
||||||
|
if path.is_file() and path.suffix.lower() in ACTIVITY_SUFFIXES
|
||||||
|
]
|
||||||
|
return sorted(found, key=lambda p: p.stat().st_mtime)
|
||||||
|
|
||||||
|
|
||||||
|
def _adb(args: list[str]) -> tuple[bool, str]:
|
||||||
|
"""Run an adb command, returning (ok, output)."""
|
||||||
|
try:
|
||||||
|
proc = subprocess.run(
|
||||||
|
[_ADB_BIN, *args],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=_ADB_TIMEOUT,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||||
|
return False, str(exc)
|
||||||
|
return proc.returncode == 0, proc.stdout + proc.stderr
|
||||||
|
|
||||||
|
|
||||||
|
def phone_attached() -> bool:
|
||||||
|
"""True when exactly one adb device is in the 'device' state."""
|
||||||
|
ok, out = _adb(["devices"])
|
||||||
|
if not ok:
|
||||||
|
return False
|
||||||
|
states = [
|
||||||
|
line.split("\t")[1].strip() for line in out.splitlines()[1:] if "\t" in line
|
||||||
|
]
|
||||||
|
return "device" in states
|
||||||
|
|
||||||
|
|
||||||
|
def pull_from_phone(inbox: Path) -> list[Path]:
|
||||||
|
"""Copy RunnerUp exports from the phone into ``inbox``.
|
||||||
|
|
||||||
|
Returns the paths newly written. Files already present in the inbox are
|
||||||
|
left alone; content-level deduplication happens later against the ledger,
|
||||||
|
so a name collision here is not authoritative either way.
|
||||||
|
"""
|
||||||
|
if not phone_attached():
|
||||||
|
_logger.info("no adb device attached; skipping phone fallback")
|
||||||
|
return []
|
||||||
|
|
||||||
|
ok, out = _adb(["shell", "ls", PHONE_EXPORT_DIR])
|
||||||
|
if not ok:
|
||||||
|
_logger.warning("could not list %s: %s", PHONE_EXPORT_DIR, out.strip())
|
||||||
|
return []
|
||||||
|
|
||||||
|
names = [
|
||||||
|
line.strip()
|
||||||
|
for line in out.splitlines()
|
||||||
|
if line.strip().lower().endswith(ACTIVITY_SUFFIXES)
|
||||||
|
]
|
||||||
|
inbox.mkdir(parents=True, exist_ok=True)
|
||||||
|
pulled: list[Path] = []
|
||||||
|
for name in names:
|
||||||
|
target = inbox / name
|
||||||
|
if target.exists():
|
||||||
|
continue
|
||||||
|
staged = inbox / f".{name}.partial"
|
||||||
|
ok, err = _adb(["pull", f"{PHONE_EXPORT_DIR}/{name}", str(staged)])
|
||||||
|
if not ok:
|
||||||
|
_logger.warning("adb pull failed for %s: %s", name, err.strip())
|
||||||
|
staged.unlink(missing_ok=True)
|
||||||
|
continue
|
||||||
|
# Rename only after a complete pull, so a partial transfer is never
|
||||||
|
# picked up as a whole activity by this or any concurrent run.
|
||||||
|
shutil.move(str(staged), str(target))
|
||||||
|
pulled.append(target)
|
||||||
|
|
||||||
|
if pulled:
|
||||||
|
_logger.info("pulled %d file(s) from the phone", len(pulled))
|
||||||
|
return pulled
|
||||||
1
python_pkg/endurain_import/tests/__init__.py
Normal file
1
python_pkg/endurain_import/tests/__init__.py
Normal file
@ -0,0 +1 @@
|
|||||||
|
"""Tests for the RunnerUp -> Endurain importer."""
|
||||||
95
python_pkg/endurain_import/tests/test_ledger.py
Normal file
95
python_pkg/endurain_import/tests/test_ledger.py
Normal file
@ -0,0 +1,95 @@
|
|||||||
|
"""Ledger behaviour, especially the two independent dedupe paths."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_pkg.endurain_import.ledger import (
|
||||||
|
Entry,
|
||||||
|
Ledger,
|
||||||
|
LedgerCorruptError,
|
||||||
|
activity_key,
|
||||||
|
file_digest,
|
||||||
|
now_iso,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _entry(sha: str = "abc", key: str = "RunnerUp_ts_Running") -> Entry:
|
||||||
|
return Entry(sha, key, "f.tcx", 7, now_iso())
|
||||||
|
|
||||||
|
|
||||||
|
def test_digest_is_content_addressed(tmp_path: Path) -> None:
|
||||||
|
a = tmp_path / "a.tcx"
|
||||||
|
b = tmp_path / "b.tcx"
|
||||||
|
a.write_text("same")
|
||||||
|
b.write_text("same")
|
||||||
|
assert file_digest(a) == file_digest(b)
|
||||||
|
|
||||||
|
b.write_text("different")
|
||||||
|
assert file_digest(a) != file_digest(b)
|
||||||
|
|
||||||
|
|
||||||
|
def test_activity_key_collapses_export_formats() -> None:
|
||||||
|
"""The .gpx and .tcx of one run must share a key."""
|
||||||
|
tcx = Path("RunnerUp_2026-08-22-23-51-04_Running.tcx")
|
||||||
|
gpx = Path("RunnerUp_2026-08-22-23-51-04_Running.gpx")
|
||||||
|
assert activity_key(tcx) == activity_key(gpx)
|
||||||
|
|
||||||
|
|
||||||
|
def test_activity_key_ignores_device_model() -> None:
|
||||||
|
"""WebDAV names embed Build.MODEL; adb names do not."""
|
||||||
|
webdav = Path("RunnerUp_Pixel_6a_2026-08-22-23-51-04_Running.tcx")
|
||||||
|
adb = Path("RunnerUp_2026-08-22-23-51-04_Running.tcx")
|
||||||
|
assert activity_key(webdav) == activity_key(adb)
|
||||||
|
|
||||||
|
|
||||||
|
def test_activity_key_separates_distinct_runs() -> None:
|
||||||
|
first = Path("RunnerUp_2026-08-22-23-51-04_Running.tcx")
|
||||||
|
second = Path("RunnerUp_2026-08-19-19-10-04_Running.tcx")
|
||||||
|
assert activity_key(first) != activity_key(second)
|
||||||
|
|
||||||
|
|
||||||
|
def test_activity_key_falls_back_to_stem() -> None:
|
||||||
|
assert activity_key(Path("whatever.tcx")) == "whatever"
|
||||||
|
|
||||||
|
|
||||||
|
def test_record_and_membership(tmp_path: Path) -> None:
|
||||||
|
ledger = Ledger(tmp_path / "l.json")
|
||||||
|
assert len(ledger) == 0
|
||||||
|
ledger.record(_entry())
|
||||||
|
assert "abc" in ledger
|
||||||
|
assert ledger.has_activity("RunnerUp_ts_Running")
|
||||||
|
assert not ledger.has_activity("other")
|
||||||
|
assert len(ledger) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_ledger_survives_reload(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "l.json"
|
||||||
|
Ledger(path).record(_entry())
|
||||||
|
assert "abc" in Ledger(path)
|
||||||
|
|
||||||
|
|
||||||
|
def test_corrupt_ledger_fails_closed(tmp_path: Path) -> None:
|
||||||
|
"""A truncated ledger must stop the run, not re-import everything."""
|
||||||
|
path = tmp_path / "l.json"
|
||||||
|
path.write_text("{not json")
|
||||||
|
with pytest.raises(LedgerCorruptError):
|
||||||
|
Ledger(path)
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_is_atomic(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "l.json"
|
||||||
|
ledger = Ledger(path)
|
||||||
|
ledger.record(_entry())
|
||||||
|
assert not path.with_suffix(".tmp").exists()
|
||||||
|
assert json.loads(path.read_text())["abc"]["activity_id"] == 7
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_dict_ledger_is_ignored(tmp_path: Path) -> None:
|
||||||
|
"""A JSON list is valid JSON but not a ledger; treat it as empty."""
|
||||||
|
path = tmp_path / "l.json"
|
||||||
|
path.write_text("[]")
|
||||||
|
assert len(Ledger(path)) == 0
|
||||||
113
python_pkg/endurain_import/tests/test_main.py
Normal file
113
python_pkg/endurain_import/tests/test_main.py
Normal file
@ -0,0 +1,113 @@
|
|||||||
|
"""Orchestration: routing, fail-closed behaviour and exit status."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_pkg.endurain_import.__main__ import _process, _route_rejected, main
|
||||||
|
from python_pkg.endurain_import.ledger import Entry, Ledger, now_iso
|
||||||
|
from python_pkg.endurain_import.upload import Outcome, Result
|
||||||
|
|
||||||
|
|
||||||
|
class _StubClient:
|
||||||
|
def __init__(self, result: Result) -> None:
|
||||||
|
self._result = result
|
||||||
|
self.uploads: list[Path] = []
|
||||||
|
|
||||||
|
def upload(self, path: Path) -> Result:
|
||||||
|
self.uploads.append(path)
|
||||||
|
return self._result
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def inbox(tmp_path: Path) -> Path:
|
||||||
|
path = tmp_path / "inbox"
|
||||||
|
path.mkdir()
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _file(inbox: Path, name: str = "RunnerUp_ts_Running.tcx") -> Path:
|
||||||
|
path = inbox / name
|
||||||
|
path.write_text("<xml/>")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def test_success_moves_to_processed_and_records(tmp_path: Path, inbox: Path) -> None:
|
||||||
|
path = _file(inbox)
|
||||||
|
ledger = Ledger(tmp_path / "l.json")
|
||||||
|
client = _StubClient(Result(Outcome.OK, 5, "ok"))
|
||||||
|
processed = inbox / "processed"
|
||||||
|
|
||||||
|
assert _process(path, client, ledger, processed, None) == "imported"
|
||||||
|
assert (processed / path.name).exists()
|
||||||
|
assert not path.exists()
|
||||||
|
assert len(ledger) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_known_digest_is_skipped_without_uploading(tmp_path: Path, inbox: Path) -> None:
|
||||||
|
path = _file(inbox)
|
||||||
|
ledger = Ledger(tmp_path / "l.json")
|
||||||
|
from python_pkg.endurain_import.ledger import activity_key, file_digest
|
||||||
|
|
||||||
|
ledger.record(Entry(file_digest(path), activity_key(path), path.name, 1, now_iso()))
|
||||||
|
client = _StubClient(Result(Outcome.OK, 9, "ok"))
|
||||||
|
|
||||||
|
assert _process(path, client, ledger, inbox / "processed", None) == "skipped"
|
||||||
|
assert client.uploads == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_same_run_other_format_is_skipped(tmp_path: Path, inbox: Path) -> None:
|
||||||
|
"""The .gpx twin of an imported .tcx must not create a second activity."""
|
||||||
|
tcx = _file(inbox, "RunnerUp_2026-08-22-23-51-04_Running.tcx")
|
||||||
|
ledger = Ledger(tmp_path / "l.json")
|
||||||
|
from python_pkg.endurain_import.ledger import activity_key, file_digest
|
||||||
|
|
||||||
|
ledger.record(Entry(file_digest(tcx), activity_key(tcx), tcx.name, 1, now_iso()))
|
||||||
|
|
||||||
|
gpx = inbox / "RunnerUp_2026-08-22-23-51-04_Running.gpx"
|
||||||
|
gpx.write_text("<gpx/>") # different bytes, same run
|
||||||
|
client = _StubClient(Result(Outcome.OK, 9, "ok"))
|
||||||
|
|
||||||
|
assert _process(gpx, client, ledger, inbox / "processed", None) == "skipped"
|
||||||
|
assert client.uploads == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_rejected_file_is_staged_and_left_in_place(tmp_path: Path, inbox: Path) -> None:
|
||||||
|
path = _file(inbox)
|
||||||
|
bulk = tmp_path / "bulk"
|
||||||
|
ledger = Ledger(tmp_path / "l.json")
|
||||||
|
client = _StubClient(Result(Outcome.REJECTED, None, "400"))
|
||||||
|
|
||||||
|
assert _process(path, client, ledger, inbox / "processed", bulk) == "rejected"
|
||||||
|
assert (bulk / path.name).exists()
|
||||||
|
assert path.exists(), "a rejected file must stay for inspection"
|
||||||
|
assert len(ledger) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_ambiguous_never_stages_to_bulk_import(tmp_path: Path, inbox: Path) -> None:
|
||||||
|
"""The whole point of the ambiguous class: do not risk a duplicate."""
|
||||||
|
path = _file(inbox)
|
||||||
|
bulk = tmp_path / "bulk"
|
||||||
|
ledger = Ledger(tmp_path / "l.json")
|
||||||
|
client = _StubClient(Result(Outcome.AMBIGUOUS, None, "timeout"))
|
||||||
|
|
||||||
|
assert _process(path, client, ledger, inbox / "processed", bulk) == "ambiguous"
|
||||||
|
assert not bulk.exists(), "ambiguous uploads must not be re-queued"
|
||||||
|
assert path.exists()
|
||||||
|
assert len(ledger) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_route_rejected_without_bulk_dir_is_safe(tmp_path: Path) -> None:
|
||||||
|
path = tmp_path / "f.tcx"
|
||||||
|
path.write_text("x")
|
||||||
|
_route_rejected(path, None)
|
||||||
|
assert path.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_api_key_exits_two(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.delenv("ENDURAIN_API_KEY", raising=False)
|
||||||
|
with pytest.raises(SystemExit) as excinfo:
|
||||||
|
main()
|
||||||
|
assert excinfo.value.code == 2
|
||||||
179
python_pkg/endurain_import/tests/test_orchestration.py
Normal file
179
python_pkg/endurain_import/tests/test_orchestration.py
Normal file
@ -0,0 +1,179 @@
|
|||||||
|
"""End-to-end orchestration: environment wiring and exit status."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_pkg.endurain_import.__main__ import _route_rejected, main
|
||||||
|
from python_pkg.endurain_import.upload import Outcome, Result
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def inbox(tmp_path: Path) -> Path:
|
||||||
|
path = tmp_path / "inbox"
|
||||||
|
path.mkdir()
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _file(inbox: Path, name: str = "RunnerUp_ts_Running.tcx") -> Path:
|
||||||
|
path = inbox / name
|
||||||
|
path.write_text("<xml/>")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
class _StubMainClient:
|
||||||
|
"""Stands in for EndurainClient across a whole main() run."""
|
||||||
|
|
||||||
|
def __init__(self, result: Result, *, reachable: bool = True) -> None:
|
||||||
|
self._result = result
|
||||||
|
self._reachable = reachable
|
||||||
|
|
||||||
|
def about(self) -> dict[str, object]:
|
||||||
|
if not self._reachable:
|
||||||
|
message = "connection refused"
|
||||||
|
raise OSError(message)
|
||||||
|
return {"version": "test"}
|
||||||
|
|
||||||
|
def upload(self, _path: Path) -> Result:
|
||||||
|
return self._result
|
||||||
|
|
||||||
|
|
||||||
|
def _env(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, inbox: Path, **extra: str
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setenv("ENDURAIN_API_KEY", "k")
|
||||||
|
monkeypatch.setenv("ENDURAIN_INBOX", str(inbox))
|
||||||
|
monkeypatch.setenv("ENDURAIN_STATE", str(tmp_path / "state"))
|
||||||
|
monkeypatch.setenv("ENDURAIN_NO_ADB", "1")
|
||||||
|
for key, value in extra.items():
|
||||||
|
monkeypatch.setenv(key, value)
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_client(monkeypatch: pytest.MonkeyPatch, client: _StubMainClient) -> None:
|
||||||
|
import python_pkg.endurain_import.__main__ as mod
|
||||||
|
|
||||||
|
monkeypatch.setattr(mod, "EndurainClient", lambda *_a, **_k: client)
|
||||||
|
|
||||||
|
|
||||||
|
def test_main_returns_one_when_unreachable(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, inbox: Path
|
||||||
|
) -> None:
|
||||||
|
_env(monkeypatch, tmp_path, inbox)
|
||||||
|
_patch_client(
|
||||||
|
monkeypatch,
|
||||||
|
_StubMainClient(Result(Outcome.OK, 1, "ok"), reachable=False),
|
||||||
|
)
|
||||||
|
assert main() == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_main_succeeds_with_empty_inbox(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, inbox: Path
|
||||||
|
) -> None:
|
||||||
|
_env(monkeypatch, tmp_path, inbox)
|
||||||
|
_patch_client(monkeypatch, _StubMainClient(Result(Outcome.OK, 1, "ok")))
|
||||||
|
assert main() == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_main_imports_and_returns_zero(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, inbox: Path
|
||||||
|
) -> None:
|
||||||
|
_file(inbox)
|
||||||
|
_env(monkeypatch, tmp_path, inbox)
|
||||||
|
_patch_client(monkeypatch, _StubMainClient(Result(Outcome.OK, 3, "ok")))
|
||||||
|
assert main() == 0
|
||||||
|
assert (inbox / "processed" / "RunnerUp_ts_Running.tcx").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_main_fails_closed_on_ambiguous(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, inbox: Path
|
||||||
|
) -> None:
|
||||||
|
_file(inbox)
|
||||||
|
_env(monkeypatch, tmp_path, inbox)
|
||||||
|
_patch_client(
|
||||||
|
monkeypatch, _StubMainClient(Result(Outcome.AMBIGUOUS, None, "timeout"))
|
||||||
|
)
|
||||||
|
assert main() == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_main_fails_closed_on_rejection(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, inbox: Path
|
||||||
|
) -> None:
|
||||||
|
_file(inbox)
|
||||||
|
_env(monkeypatch, tmp_path, inbox, ENDURAIN_BULK_DIR=str(tmp_path / "bulk"))
|
||||||
|
_patch_client(monkeypatch, _StubMainClient(Result(Outcome.REJECTED, None, "400")))
|
||||||
|
assert main() == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_main_prefers_tcx_over_gpx(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, inbox: Path
|
||||||
|
) -> None:
|
||||||
|
"""When both formats are pending, the richer TCX must be the one uploaded."""
|
||||||
|
stem = "RunnerUp_2026-08-22-23-51-04_Running"
|
||||||
|
(inbox / f"{stem}.gpx").write_text("<gpx/>")
|
||||||
|
(inbox / f"{stem}.tcx").write_text("<tcx/>")
|
||||||
|
_env(monkeypatch, tmp_path, inbox)
|
||||||
|
|
||||||
|
uploaded: list[str] = []
|
||||||
|
|
||||||
|
class _Recorder(_StubMainClient):
|
||||||
|
def upload(self, path: Path) -> Result:
|
||||||
|
uploaded.append(path.name)
|
||||||
|
return Result(Outcome.OK, 1, "ok")
|
||||||
|
|
||||||
|
_patch_client(monkeypatch, _Recorder(Result(Outcome.OK, 1, "ok")))
|
||||||
|
assert main() == 0
|
||||||
|
assert uploaded == [f"{stem}.tcx"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_main_runs_adb_fallback_when_enabled(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, inbox: Path
|
||||||
|
) -> None:
|
||||||
|
_env(monkeypatch, tmp_path, inbox)
|
||||||
|
monkeypatch.delenv("ENDURAIN_NO_ADB")
|
||||||
|
called: list[Path] = []
|
||||||
|
import python_pkg.endurain_import.__main__ as mod
|
||||||
|
|
||||||
|
def _record(path: Path) -> list[Path]:
|
||||||
|
called.append(path)
|
||||||
|
return []
|
||||||
|
|
||||||
|
monkeypatch.setattr(mod, "pull_from_phone", _record)
|
||||||
|
_patch_client(monkeypatch, _StubMainClient(Result(Outcome.OK, 1, "ok")))
|
||||||
|
assert main() == 0
|
||||||
|
assert called == [inbox]
|
||||||
|
|
||||||
|
|
||||||
|
def test_route_rejected_handles_unwritable_bulk_dir(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""A broken bulk_import must not take down the whole run."""
|
||||||
|
path = tmp_path / "f.tcx"
|
||||||
|
path.write_text("x")
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
def _boom(*_a: object, **_k: object) -> None:
|
||||||
|
message = "read-only filesystem"
|
||||||
|
raise OSError(message)
|
||||||
|
|
||||||
|
monkeypatch.setattr(shutil, "copy2", _boom)
|
||||||
|
_route_rejected(path, tmp_path / "bulk")
|
||||||
|
assert path.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_main_handles_requests_exception(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, inbox: Path
|
||||||
|
) -> None:
|
||||||
|
import requests
|
||||||
|
|
||||||
|
_env(monkeypatch, tmp_path, inbox)
|
||||||
|
|
||||||
|
class _Failing(_StubMainClient):
|
||||||
|
def about(self) -> dict[str, object]:
|
||||||
|
message = "refused"
|
||||||
|
raise requests.ConnectionError(message)
|
||||||
|
|
||||||
|
_patch_client(monkeypatch, _Failing(Result(Outcome.OK, 1, "ok")))
|
||||||
|
assert main() == 1
|
||||||
170
python_pkg/endurain_import/tests/test_sources.py
Normal file
170
python_pkg/endurain_import/tests/test_sources.py
Normal file
@ -0,0 +1,170 @@
|
|||||||
|
"""Inbox scanning and the adb fallback.
|
||||||
|
|
||||||
|
The fallback is read-only with respect to the phone: screen-locker reads the
|
||||||
|
same directory as its screen-unlock gate, so nothing here may delete or
|
||||||
|
rewrite those files.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from python_pkg.endurain_import import sources
|
||||||
|
|
||||||
|
|
||||||
|
def test_inbox_lists_only_activity_files(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "a.tcx").write_text("x")
|
||||||
|
(tmp_path / "b.gpx").write_text("x")
|
||||||
|
(tmp_path / "c.fit").write_text("x")
|
||||||
|
(tmp_path / "notes.txt").write_text("x")
|
||||||
|
names = {p.name for p in sources.inbox_files(tmp_path)}
|
||||||
|
assert names == {"a.tcx", "b.gpx", "c.fit"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_inbox_skips_processed_subdir(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "processed").mkdir()
|
||||||
|
(tmp_path / "processed" / "old.tcx").write_text("x")
|
||||||
|
assert sources.inbox_files(tmp_path) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_inbox_is_not_an_error(tmp_path: Path) -> None:
|
||||||
|
assert sources.inbox_files(tmp_path / "nope") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_inbox_is_ordered_oldest_first(tmp_path: Path) -> None:
|
||||||
|
first = tmp_path / "first.tcx"
|
||||||
|
second = tmp_path / "second.tcx"
|
||||||
|
first.write_text("x")
|
||||||
|
second.write_text("x")
|
||||||
|
import os
|
||||||
|
|
||||||
|
os.utime(first, (1000, 1000))
|
||||||
|
os.utime(second, (2000, 2000))
|
||||||
|
assert [p.name for p in sources.inbox_files(tmp_path)] == [
|
||||||
|
"first.tcx",
|
||||||
|
"second.tcx",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_phone_attached_parses_adb_devices(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
sources,
|
||||||
|
"_adb",
|
||||||
|
lambda _a: (True, "List of devices attached\nABC123\tdevice\n"),
|
||||||
|
)
|
||||||
|
assert sources.phone_attached()
|
||||||
|
|
||||||
|
|
||||||
|
def test_phone_unauthorised_is_not_attached(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
sources,
|
||||||
|
"_adb",
|
||||||
|
lambda _a: (True, "List of devices attached\nABC123\tunauthorized\n"),
|
||||||
|
)
|
||||||
|
assert not sources.phone_attached()
|
||||||
|
|
||||||
|
|
||||||
|
def test_pull_skips_when_no_device(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(sources, "phone_attached", lambda: False)
|
||||||
|
assert sources.pull_from_phone(tmp_path) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_pull_ignores_already_present_files(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(sources, "phone_attached", lambda: True)
|
||||||
|
monkeypatch.setattr(sources, "_adb", lambda _a: (True, "RunnerUp_ts_Running.tcx\n"))
|
||||||
|
(tmp_path / "RunnerUp_ts_Running.tcx").write_text("already here")
|
||||||
|
assert sources.pull_from_phone(tmp_path) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_pull_leaves_no_partial_file(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""A half-transferred file must never look like a complete activity."""
|
||||||
|
monkeypatch.setattr(sources, "phone_attached", lambda: True)
|
||||||
|
|
||||||
|
def _adb(args: list[str]) -> tuple[bool, str]:
|
||||||
|
if args[0] == "shell":
|
||||||
|
return True, "RunnerUp_ts_Running.tcx\n"
|
||||||
|
return False, "pull failed"
|
||||||
|
|
||||||
|
monkeypatch.setattr(sources, "_adb", _adb)
|
||||||
|
assert sources.pull_from_phone(tmp_path) == []
|
||||||
|
assert list(tmp_path.iterdir()) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_adb_handles_missing_binary(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""A machine with no adb reports 'no device', it does not crash the run."""
|
||||||
|
|
||||||
|
def _boom(*_a: object, **_k: object) -> None:
|
||||||
|
message = "adb not found"
|
||||||
|
raise OSError(message)
|
||||||
|
|
||||||
|
monkeypatch.setattr(subprocess, "run", _boom)
|
||||||
|
ok, out = sources._adb(["devices"])
|
||||||
|
assert not ok
|
||||||
|
assert "adb not found" in out
|
||||||
|
|
||||||
|
|
||||||
|
def test_adb_handles_timeout(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
def _slow(*_a: object, **_k: object) -> None:
|
||||||
|
raise subprocess.TimeoutExpired(cmd="adb", timeout=1)
|
||||||
|
|
||||||
|
monkeypatch.setattr(subprocess, "run", _slow)
|
||||||
|
ok, _ = sources._adb(["devices"])
|
||||||
|
assert not ok
|
||||||
|
|
||||||
|
|
||||||
|
def test_adb_returns_output_on_success(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
class _Proc:
|
||||||
|
returncode = 0
|
||||||
|
stdout = "out"
|
||||||
|
stderr = ""
|
||||||
|
|
||||||
|
monkeypatch.setattr(subprocess, "run", lambda *_a, **_k: _Proc())
|
||||||
|
ok, out = sources._adb(["devices"])
|
||||||
|
assert ok
|
||||||
|
assert out == "out"
|
||||||
|
|
||||||
|
|
||||||
|
def test_phone_attached_false_when_adb_fails(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(sources, "_adb", lambda _a: (False, "boom"))
|
||||||
|
assert not sources.phone_attached()
|
||||||
|
|
||||||
|
|
||||||
|
def test_pull_reports_unlistable_directory(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(sources, "phone_attached", lambda: True)
|
||||||
|
monkeypatch.setattr(sources, "_adb", lambda _a: (False, "no such dir"))
|
||||||
|
assert sources.pull_from_phone(tmp_path) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_successful_pull_moves_into_place(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(sources, "phone_attached", lambda: True)
|
||||||
|
|
||||||
|
def _adb(args: list[str]) -> tuple[bool, str]:
|
||||||
|
if args[0] == "shell":
|
||||||
|
return True, "RunnerUp_ts_Running.tcx\nnotes.txt\n"
|
||||||
|
Path(args[2]).write_text("pulled")
|
||||||
|
return True, ""
|
||||||
|
|
||||||
|
monkeypatch.setattr(sources, "_adb", _adb)
|
||||||
|
pulled = sources.pull_from_phone(tmp_path)
|
||||||
|
assert [p.name for p in pulled] == ["RunnerUp_ts_Running.tcx"]
|
||||||
|
assert (tmp_path / "RunnerUp_ts_Running.tcx").read_text() == "pulled"
|
||||||
|
assert not list(tmp_path.glob(".*partial"))
|
||||||
137
python_pkg/endurain_import/tests/test_upload.py
Normal file
137
python_pkg/endurain_import/tests/test_upload.py
Normal file
@ -0,0 +1,137 @@
|
|||||||
|
"""Upload outcome classification.
|
||||||
|
|
||||||
|
The distinction that matters: a definite rejection may be retried/staged, an
|
||||||
|
ambiguous one may NOT, because Endurain does not deduplicate and the activity
|
||||||
|
may already have been committed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from python_pkg.endurain_import.upload import EndurainClient, Outcome
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeResponse:
|
||||||
|
def __init__(self, status: int, body: object = None) -> None:
|
||||||
|
self.status_code = status
|
||||||
|
self._body = body
|
||||||
|
self.text = str(body)
|
||||||
|
|
||||||
|
def json(self) -> object:
|
||||||
|
if self._body is None:
|
||||||
|
message = "no json"
|
||||||
|
raise ValueError(message)
|
||||||
|
return self._body
|
||||||
|
|
||||||
|
|
||||||
|
def _client_with(monkeypatch: pytest.MonkeyPatch, outcome: object) -> EndurainClient:
|
||||||
|
client = EndurainClient("http://x", "k")
|
||||||
|
|
||||||
|
def _post(*_args: object, **_kwargs: object) -> object:
|
||||||
|
if isinstance(outcome, Exception):
|
||||||
|
raise outcome
|
||||||
|
return outcome
|
||||||
|
|
||||||
|
monkeypatch.setattr(client._session, "post", _post)
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def sample(tmp_path: Path) -> Path:
|
||||||
|
path = tmp_path / "RunnerUp_ts_Running.tcx"
|
||||||
|
path.write_text("<xml/>")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def test_success_extracts_activity_id(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, sample: Path
|
||||||
|
) -> None:
|
||||||
|
client = _client_with(monkeypatch, _FakeResponse(201, {"id": 42}))
|
||||||
|
result = client.upload(sample)
|
||||||
|
assert result.outcome is Outcome.OK
|
||||||
|
assert result.activity_id == 42
|
||||||
|
|
||||||
|
|
||||||
|
def test_success_without_parsable_body(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, sample: Path
|
||||||
|
) -> None:
|
||||||
|
client = _client_with(monkeypatch, _FakeResponse(200))
|
||||||
|
result = client.upload(sample)
|
||||||
|
assert result.outcome is Outcome.OK
|
||||||
|
assert result.activity_id is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_bare_integer_body_is_an_id(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, sample: Path
|
||||||
|
) -> None:
|
||||||
|
client = _client_with(monkeypatch, _FakeResponse(200, 9))
|
||||||
|
assert client.upload(sample).activity_id == 9
|
||||||
|
|
||||||
|
|
||||||
|
def test_client_error_is_rejected(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, sample: Path
|
||||||
|
) -> None:
|
||||||
|
client = _client_with(monkeypatch, _FakeResponse(400, {"detail": "bad"}))
|
||||||
|
assert client.upload(sample).outcome is Outcome.REJECTED
|
||||||
|
|
||||||
|
|
||||||
|
def test_server_error_is_ambiguous(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, sample: Path
|
||||||
|
) -> None:
|
||||||
|
"""5xx may mean the activity WAS created; retrying could duplicate it."""
|
||||||
|
client = _client_with(monkeypatch, _FakeResponse(503, {}))
|
||||||
|
assert client.upload(sample).outcome is Outcome.AMBIGUOUS
|
||||||
|
|
||||||
|
|
||||||
|
def test_transport_failure_is_ambiguous(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, sample: Path
|
||||||
|
) -> None:
|
||||||
|
client = _client_with(monkeypatch, requests.Timeout("timed out"))
|
||||||
|
assert client.upload(sample).outcome is Outcome.AMBIGUOUS
|
||||||
|
|
||||||
|
|
||||||
|
def test_api_key_header_is_set() -> None:
|
||||||
|
"""Endurain's OpenAPI schema mislabels this; X-API-Key is correct."""
|
||||||
|
client = EndurainClient("http://x/", "secret")
|
||||||
|
assert client._session.headers["X-API-Key"] == "secret"
|
||||||
|
|
||||||
|
|
||||||
|
def test_about_returns_parsed_body(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
client = EndurainClient("http://x", "k")
|
||||||
|
|
||||||
|
class _Resp:
|
||||||
|
status_code = 200
|
||||||
|
|
||||||
|
def raise_for_status(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def json(self) -> dict[str, str]:
|
||||||
|
return {"version": "0.19.0"}
|
||||||
|
|
||||||
|
monkeypatch.setattr(client._session, "get", lambda *_a, **_k: _Resp())
|
||||||
|
assert client.about() == {"version": "0.19.0"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_activity_id_ignores_non_integer_fields(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, sample: Path
|
||||||
|
) -> None:
|
||||||
|
client = _client_with(monkeypatch, _FakeResponse(200, {"id": "not-an-int"}))
|
||||||
|
assert client.upload(sample).activity_id is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_activity_id_from_alternate_key(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, sample: Path
|
||||||
|
) -> None:
|
||||||
|
client = _client_with(monkeypatch, _FakeResponse(200, {"activity_id": 12}))
|
||||||
|
assert client.upload(sample).activity_id == 12
|
||||||
|
|
||||||
|
|
||||||
|
def test_activity_id_none_for_list_body(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, sample: Path
|
||||||
|
) -> None:
|
||||||
|
client = _client_with(monkeypatch, _FakeResponse(200, ["a"]))
|
||||||
|
assert client.upload(sample).activity_id is None
|
||||||
122
python_pkg/endurain_import/upload.py
Normal file
122
python_pkg/endurain_import/upload.py
Normal file
@ -0,0 +1,122 @@
|
|||||||
|
"""Endurain API client for activity uploads.
|
||||||
|
|
||||||
|
Endurain does not deduplicate uploads, so the distinction between "definitely
|
||||||
|
rejected" and "might have been committed" decides whether a retry is safe:
|
||||||
|
|
||||||
|
* 4xx -- the server refused the file. Retrying is pointless but harmless;
|
||||||
|
the file is routed to Endurain's bulk_import for manual recovery.
|
||||||
|
* 5xx / timeout / connection loss -- the server may have created the activity
|
||||||
|
before the response was lost. Retrying blindly would duplicate it,
|
||||||
|
so the file is left in the inbox and reported as AMBIGUOUS. The
|
||||||
|
caller must not treat this as either success or a clean failure.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from enum import Enum
|
||||||
|
import logging
|
||||||
|
from typing import TYPE_CHECKING, Any, Protocol
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
UPLOAD_PATH = "/api/v1/activities/create/upload"
|
||||||
|
# The API key goes in X-API-Key. Endurain's own OpenAPI schema declares the
|
||||||
|
# APIKeyHeader security scheme as name "X-Client-Type", which is wrong --
|
||||||
|
# verified against the running server: X-API-Key with a bad key returns
|
||||||
|
# {"detail":"Invalid API key"} (parsed, then rejected), whereas X-Client-Type
|
||||||
|
# returns {"detail":"Not authenticated..."} (never read as a key at all).
|
||||||
|
_API_KEY_HEADER = "X-API-Key"
|
||||||
|
_TIMEOUT = 120
|
||||||
|
_HTTP_BAD_REQUEST = 400
|
||||||
|
_HTTP_SERVER_ERROR = 500
|
||||||
|
|
||||||
|
|
||||||
|
class SupportsUpload(Protocol):
|
||||||
|
"""The upload surface :func:`_process` needs.
|
||||||
|
|
||||||
|
Declared structurally so tests can substitute a stub without subclassing
|
||||||
|
the real client (and without mypy complaining about the swap).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def upload(self, path: Path) -> Result:
|
||||||
|
"""Upload one activity file and report what happened."""
|
||||||
|
... # pragma: no cover - structural typing only, never executed
|
||||||
|
|
||||||
|
|
||||||
|
class Outcome(Enum):
|
||||||
|
"""What happened to one upload attempt."""
|
||||||
|
|
||||||
|
OK = "ok"
|
||||||
|
REJECTED = "rejected"
|
||||||
|
AMBIGUOUS = "ambiguous"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Result:
|
||||||
|
"""The outcome of a single upload attempt."""
|
||||||
|
|
||||||
|
outcome: Outcome
|
||||||
|
activity_id: int | None
|
||||||
|
detail: str
|
||||||
|
|
||||||
|
|
||||||
|
class EndurainClient:
|
||||||
|
"""Minimal client for the endpoints this importer needs."""
|
||||||
|
|
||||||
|
def __init__(self, base_url: str, api_key: str) -> None:
|
||||||
|
"""Build a client for ``base_url`` authenticating with ``api_key``."""
|
||||||
|
self._base = base_url.rstrip("/")
|
||||||
|
self._session = requests.Session()
|
||||||
|
self._session.headers.update({_API_KEY_HEADER: api_key})
|
||||||
|
|
||||||
|
def about(self) -> dict[str, Any]:
|
||||||
|
"""Liveness probe; raises on any non-2xx."""
|
||||||
|
resp = self._session.get(f"{self._base}/api/v1/about", timeout=30)
|
||||||
|
resp.raise_for_status()
|
||||||
|
body: dict[str, Any] = resp.json()
|
||||||
|
return body
|
||||||
|
|
||||||
|
def upload(self, path: Path) -> Result:
|
||||||
|
"""Upload one activity file."""
|
||||||
|
try:
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
resp = self._session.post(
|
||||||
|
f"{self._base}{UPLOAD_PATH}",
|
||||||
|
files={"file": (path.name, handle)},
|
||||||
|
timeout=_TIMEOUT,
|
||||||
|
)
|
||||||
|
except requests.RequestException as exc:
|
||||||
|
# The request may or may not have reached the server.
|
||||||
|
return Result(Outcome.AMBIGUOUS, None, f"transport error: {exc}")
|
||||||
|
|
||||||
|
if resp.status_code >= _HTTP_SERVER_ERROR:
|
||||||
|
return Result(Outcome.AMBIGUOUS, None, f"server error {resp.status_code}")
|
||||||
|
if resp.status_code >= _HTTP_BAD_REQUEST:
|
||||||
|
return Result(
|
||||||
|
Outcome.REJECTED,
|
||||||
|
None,
|
||||||
|
f"{resp.status_code}: {resp.text[:200]}",
|
||||||
|
)
|
||||||
|
return Result(Outcome.OK, _activity_id(resp), "uploaded")
|
||||||
|
|
||||||
|
|
||||||
|
def _activity_id(resp: requests.Response) -> int | None:
|
||||||
|
"""Best-effort extraction of the new activity id from a success body."""
|
||||||
|
try:
|
||||||
|
body = resp.json()
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
if isinstance(body, int):
|
||||||
|
return body
|
||||||
|
if isinstance(body, dict):
|
||||||
|
for key in ("id", "activity_id"):
|
||||||
|
value = body.get(key)
|
||||||
|
if isinstance(value, int):
|
||||||
|
return value
|
||||||
|
return None
|
||||||
Loading…
Reference in New Issue
Block a user