From cc19d568a3c75ae583fc412576ef68b70116226c Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 9 Aug 2026 23:51:06 +0200 Subject: [PATCH] Add aur-publish: hands-off AUR publishing for the Flutter apps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AUR registration is currently returning a bare nginx 503 server-side, so the three packages (todo-flutter, habit-stack, diet-guard-app) are built and verified but cannot be published yet. This runs the whole thing unattended whenever signup reopens. anubis_gate.py exists because the AUR sits behind Anubis: every HTML request gets a proof-of-work challenge page carrying HTTP *200*, so a plain curl reports 200 for a page that is actually unreachable. That is how the 503 got misdiagnosed twice as a local/browser/IPv6 problem. The module solves the challenge and reports the status of the real page behind it. Fail-closed by construction: each package is built, namcap'd, installed and LAUNCHED before it is pushed, and check_aur_rules() blocks the push on the mechanically checkable submission rules (maintainer line, x86_64, no replaces=, not already in an official repo, .SRCINFO matching the PKGBUILD). Verified by injecting a replaces= violation: the push was blocked and reported. A failing package is skipped; the others still publish. Signup itself stays manual — it needs an email confirmation, and creating accounts is not something this should do on the user's behalf. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Aoa1nKWRrAbowz7Hfd3tgL --- aur-publish/README.md | 45 +++++ aur-publish/anubis_gate.py | 138 +++++++++++++ aur-publish/publish.sh | 395 +++++++++++++++++++++++++++++++++++++ 3 files changed, 578 insertions(+) create mode 100644 aur-publish/README.md create mode 100644 aur-publish/anubis_gate.py create mode 100755 aur-publish/publish.sh diff --git a/aur-publish/README.md b/aur-publish/README.md new file mode 100644 index 0000000..58a5782 --- /dev/null +++ b/aur-publish/README.md @@ -0,0 +1,45 @@ +# aur-publish + +Hands-off publishing of kuhy's Flutter apps to the AUR. + +```bash +~/utils/aur-publish/publish.sh # wait for registration, then publish +~/utils/aur-publish/publish.sh --dry-run # everything except the push +~/utils/aur-publish/publish.sh --skip-wait # account already works +~/utils/aur-publish/publish.sh --only habit-stack +``` + +## What it does + +1. Polls `/register` until AUR signup reopens, opens it in a browser and + prints the SSH key to paste. **Signup is the only manual step** — it + requires email confirmation. +2. Waits for `ssh aur@aur.archlinux.org` to authenticate. +3. Then, unattended, per package: resolve the newest `vX.Y.Z` tag → set + `pkgver` → `updpkgsums` → `makepkg -C` → `namcap` → AUR rule check → + `pacman -U` → launch check → `.SRCINFO` → commit → push. + +Every package is built, installed and **launched** before it is pushed, so a +broken package cannot reach the AUR. A package failing any gate is skipped and +reported; the others still publish. Safe to re-run. + +## anubis_gate.py + +The AUR sits behind [Anubis](https://anubis.techaro.lol/), which answers HTML +requests with a proof-of-work challenge page carrying **HTTP 200**. A plain +`curl` therefore reports 200 for a page that is actually unreachable — which is +exactly how "is registration back?" gets answered wrong. This module solves the +challenge and reports the status of the *real* page behind it. + +```bash +python3 anubis_gate.py /register # prints the true status code +``` + +## AUR rules enforced before pushing + +The AUR does not review uploads, but per the submission guidelines "packages +that violate the rules may be deleted without warning". `check_aur_rules()` +blocks a push on the mechanically checkable ones: `# Maintainer:` line, +`x86_64` in `arch`, no `replaces=` (reserved for renames — use `conflicts`), +not already in an official repo, `pkgdesc`/`url`/`license` present, and +`.SRCINFO` matching the PKGBUILD. diff --git a/aur-publish/anubis_gate.py b/aur-publish/anubis_gate.py new file mode 100644 index 0000000..79de9bd --- /dev/null +++ b/aur-publish/anubis_gate.py @@ -0,0 +1,138 @@ +"""Fetch an AUR URL from behind the Anubis anti-bot gate. + +The AUR sits behind Anubis (https://anubis.techaro.lol/), which answers every +HTML request with a proof-of-work challenge page carrying HTTP **200**. A bare +`curl` therefore reports 200 for a page that is actually unreachable, which is +exactly how "is /register back up?" gets answered wrong. + +This module solves the challenge and reports the status of the *real* page +behind it, so the caller can distinguish: + + * 200 -> genuinely available + * 503 -> reached the site; that endpoint is disabled server-side + * other-> whatever the app actually said + +Exit codes mirror the HTTP status so shell callers can branch on them: +0 for 200, 1 for anything else, 2 for a local/transport failure. +""" + +from __future__ import annotations + +import argparse +import hashlib +import http.cookiejar +import json +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +BASE = "https://aur.archlinux.org" +UA = "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0" +CHALLENGE_RE = re.compile( + r'', + re.S, +) +PASS_PATH = "/.within.website/x/cmd/anubis/api/pass-challenge" +# A solve at difficulty 4 takes ~0.01s; this ceiling only guards a server-side +# difficulty spike, so the poller degrades instead of spinning forever. +MAX_NONCE = 50_000_000 + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Keep redirects visible instead of silently following them.""" + + def redirect_request(self, *_args, **_kwargs): # noqa: D102 - see class doc + return None + + +def _opener() -> urllib.request.OpenerDirector: + jar = http.cookiejar.CookieJar() + opener = urllib.request.build_opener( + urllib.request.HTTPCookieProcessor(jar), _NoRedirect + ) + opener.addheaders = [ + ("User-Agent", UA), + ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + ] + return opener + + +def _solve(random_data: str, difficulty: int) -> tuple[int, str, float]: + """Find a nonce whose sha256 digest starts with `difficulty` zero hex chars.""" + prefix = "0" * difficulty + started = time.time() + for nonce in range(MAX_NONCE): + digest = hashlib.sha256(f"{random_data}{nonce}".encode()).hexdigest() + if digest.startswith(prefix): + return nonce, digest, time.time() - started + raise RuntimeError(f"no nonce found below {MAX_NONCE} at difficulty {difficulty}") + + +def fetch(path: str, timeout: int = 30) -> tuple[int, str]: + """Return (status, body) for `path`, clearing Anubis first if challenged.""" + opener = _opener() + url = f"{BASE}{path}" + + try: + with opener.open(url, timeout=timeout) as resp: + body = resp.read().decode("utf-8", "replace") + status = resp.status + except urllib.error.HTTPError as exc: + return exc.code, exc.read().decode("utf-8", "replace") + + match = CHALLENGE_RE.search(body) + if not match: + return status, body + + challenge = json.loads(match.group(1))["challenge"] + nonce, digest, elapsed = _solve( + challenge["randomData"], int(challenge["difficulty"]) + ) + query = urllib.parse.urlencode( + { + "id": challenge["id"], + "response": digest, + "nonce": str(nonce), + "redir": path, + "elapsedTime": str(int(elapsed * 1000)), + } + ) + try: + opener.open(f"{BASE}{PASS_PATH}?{query}", timeout=timeout) + except urllib.error.HTTPError: + # A redirect or non-2xx here is fine; what matters is the cookie the + # opener's jar just captured. + pass + + try: + with opener.open(url, timeout=timeout) as resp: + return resp.status, resp.read().decode("utf-8", "replace") + except urllib.error.HTTPError as exc: + return exc.code, exc.read().decode("utf-8", "replace") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("path", help="AUR path, e.g. /register") + parser.add_argument( + "--print-body", action="store_true", help="write the page body to stdout" + ) + args = parser.parse_args() + + try: + status, body = fetch(args.path) + except (urllib.error.URLError, RuntimeError, TimeoutError) as exc: + print(f"transport error: {exc}", file=sys.stderr) + return 2 + + print(status) + if args.print_body: + sys.stdout.write(body) + return 0 if status == 200 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/aur-publish/publish.sh b/aur-publish/publish.sh new file mode 100755 index 0000000..193bf4e --- /dev/null +++ b/aur-publish/publish.sh @@ -0,0 +1,395 @@ +#!/bin/bash + +# ============================================================================ +# Publish kuhy's three Flutter apps to the AUR, hands-off. +# +# Run this once AUR registration is back. It: +# 1. Waits for https://aur.archlinux.org/register to stop returning 503, +# then opens it in a browser and prints the SSH key to paste. +# (Signup is the ONLY manual step: it needs an email confirmation, and +# creating accounts is not something this script will do for you.) +# 2. Waits for `ssh aur@aur.archlinux.org` to authenticate. +# 3. From there, fully unattended, for each package: +# newest release tag -> pkgver -> updpkgsums -> makepkg -C -> namcap +# -> pacman -U -> launch check -> .SRCINFO -> clone AUR repo +# -> commit -> push. +# +# Every package is BUILT, INSTALLED and LAUNCHED before it is pushed, so a +# broken package cannot reach the AUR. A package that fails any gate is +# skipped and reported; the others still publish. +# +# Idempotent and resumable: re-running skips work that is already done, and +# an already-published package is updated rather than duplicated. +# ============================================================================ + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +readonly GATE="$SCRIPT_DIR/anubis_gate.py" +readonly AUR_HOST="aur.archlinux.org" +readonly SSH_KEY="$HOME/.ssh/aur" +readonly AUR_ROOT="$HOME/aur" + +# package:repo:pubspec-subdir:binary-name:expected-startup-string +readonly PACKAGES=( + "todo-flutter:$HOME/todo:.:todo:serving on http://localhost:8730" + "habit-stack:$HOME/habit_stack:.:habit-stack:serving on http://localhost:8731" + "diet-guard-app:$HOME/diet-guard:app:diet-guard-app:serving on http://localhost:8732" +) + +POLL_SECONDS=300 +SKIP_WAIT=0 +DRY_RUN=0 +ONLY="" +FAILED=() +PUBLISHED=() + +usage() { + cat <\033[0m %s\n' "$*"; } +info() { printf ' %s\n' "$*"; } +warn() { printf '\033[1;33m ! %s\033[0m\n' "$*" >&2; } +die() { printf '\033[1;31m==> ERROR: %s\033[0m\n' "$*" >&2; exit 1; } + +require_tools() { + local missing=() + local tool + for tool in git makepkg updpkgsums namcap ssh python3 curl; do + command -v "$tool" >/dev/null 2>&1 || missing+=("$tool") + done + if ((${#missing[@]})); then + info "Installing missing tools: ${missing[*]}" + # namcap/updpkgsums come from namcap + pacman-contrib; the rest are base. + sudo pacman -S --needed --noconfirm namcap pacman-contrib git openssh python \ + || die "could not install: ${missing[*]}" + fi +} + +# --------------------------------------------------------------------------- +# Phase 1: wait for registration, then for the key to work +# --------------------------------------------------------------------------- + +register_status() { + python3 "$GATE" /register 2>/dev/null | head -1 +} + +open_browser() { + local url="$1" + local browser + for browser in xdg-open thorium-browser chromium google-chrome-stable firefox librewolf; do + if command -v "$browser" >/dev/null 2>&1; then + "$browser" "$url" >/dev/null 2>&1 & + info "Opened $url in $browser" + return 0 + fi + done + warn "No browser found; open this yourself: $url" +} + +ensure_key() { + if [[ ! -f "$SSH_KEY" ]]; then + log "Generating an AUR SSH key (none at $SSH_KEY)" + ssh-keygen -t ed25519 -f "$SSH_KEY" -N '' -C 'kuhy@aur' + fi + if ! grep -q "Host $AUR_HOST" "$HOME/.ssh/config" 2>/dev/null; then + log "Adding $AUR_HOST to ~/.ssh/config" + mkdir -p "$HOME/.ssh" + cat >> "$HOME/.ssh/config" <&1 | + grep -qiE 'interactive shell is disabled|Welcome to AUR' +} + +wait_for_registration() { + local status + status="$(register_status)" + if [[ "$status" == "200" ]]; then + log "AUR registration is OPEN" + else + log "AUR registration is closed (HTTP $status). Polling every ${POLL_SECONDS}s." + info "Ctrl-C to stop; re-run any time, nothing is lost." + while :; do + sleep "$POLL_SECONDS" + status="$(register_status)" + if [[ "$status" == "200" ]]; then + log "Registration is BACK (HTTP 200)" + break + fi + printf ' %s still HTTP %s\n' "$(date +%H:%M)" "$status" + done + fi + + open_browser "https://$AUR_HOST/register" + cat </dev/null | + awk '{print $2}' | + sed 's|refs/tags/||' | + grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | + sed 's/^v//' | + sort -V | + tail -1 +} + +verify_installed() { + local binary="$1" expected="$2" output + output="$(timeout 15 "/usr/bin/$binary" 2>&1 | head -3 || true)" + if grep -qF "$expected" <<<"$output"; then + return 0 + fi + # A wrapper that finds its own instance already running is still a healthy + # binary, so treat that as a pass rather than a false failure. + if grep -qiE 'already running' <<<"$output"; then + info "already running — treating as healthy" + return 0 + fi + warn "unexpected startup output: $output" + return 1 +} + +# The AUR does not review uploads, but "packages that violate the rules may be +# deleted without warning" (AUR submission guidelines). These are the rules that +# are mechanically checkable, enforced here so a violation blocks the push +# rather than surfacing later as a deletion request. +check_aur_rules() { + local pkg="$1" problems=() + + grep -q '^# Maintainer:' PKGBUILD \ + || problems+=("missing '# Maintainer:' comment on line 1") + + grep -qE "^arch=\(.*x86_64" PKGBUILD \ + || problems+=("x86_64 not in arch=() — packages without it are not allowed") + + # `replaces` makes pacman rip out the named package on the next -Sy, which + # is only correct for a genuine rename. `conflicts` is the right tool. + grep -q '^replaces=' PKGBUILD \ + && problems+=("uses replaces=; AUR rules reserve it for renames — use conflicts=") + + # A package already in an official repo must never be duplicated. + if pacman -Si "$pkg" >/dev/null 2>&1; then + problems+=("$pkg exists in an official repo — must not be submitted") + fi + + grep -qE '^(pkgdesc|url|license)=' PKGBUILD \ + || problems+=("missing pkgdesc/url/license") + + # A stale .SRCINFO is the single most common upload rejection. + if ! diff -q <(makepkg --printsrcinfo) .SRCINFO >/dev/null 2>&1; then + problems+=(".SRCINFO does not match PKGBUILD") + fi + + if ((${#problems[@]})); then + warn "$pkg: AUR rule violations:" + printf ' - %s\n' "${problems[@]}" + return 1 + fi + info "AUR rule checks passed" + return 0 +} + +publish_one() { + local pkg="$1" repo="$2" binary="$4" expected="$5" + local dir="$AUR_ROOT/$pkg" + local version + + log "$pkg" + + [[ -d "$repo" ]] || { warn "$pkg: no repo at $repo"; return 1; } + [[ -f "$dir/PKGBUILD" ]] || { warn "$pkg: no PKGBUILD at $dir"; return 1; } + + version="$(newest_tag "$repo")" + [[ -n "$version" ]] || { warn "$pkg: no vX.Y.Z tag on the remote"; return 1; } + info "newest release tag: v$version" + + cd "$dir" + sed -i -E "s/^pkgver=.*/pkgver=$version/" PKGBUILD + # The tarball for a tag contains the PKGBUILD, so the checksum can only be + # computed after the tag exists — never committed ahead of time. + updpkgsums >/dev/null 2>&1 || { warn "$pkg: updpkgsums failed (tag missing?)"; return 1; } + + info "building" + if ! env PATH=/usr/bin:/bin:/usr/local/bin makepkg -Cf --noconfirm >/dev/null 2>&1; then + warn "$pkg: makepkg failed" + return 1 + fi + + local built + built="$(find . -maxdepth 1 -name "$pkg-$version-*.pkg.tar.zst" | head -1)" + [[ -n "$built" ]] || { warn "$pkg: no package produced"; return 1; } + + # namcap's "ELF files outside of a valid path ('opt/')" is a false positive + # for /opt-installed apps, so only genuinely new errors are worth failing on. + local findings + findings="$(namcap "$built" 2>&1 | grep -E ' E: ' | grep -v "outside of a valid path" || true)" + if [[ -n "$findings" ]]; then + warn "$pkg: namcap errors:" + printf ' %s\n' "$findings" + return 1 + fi + + info "installing and launching" + sudo pacman -U --noconfirm "$built" >/dev/null 2>&1 \ + || { warn "$pkg: pacman -U failed"; return 1; } + verify_installed "$binary" "$expected" || { warn "$pkg: launch check failed"; return 1; } + + # Generated before the rule check, which verifies it matches the PKGBUILD. + makepkg --printsrcinfo > .SRCINFO + check_aur_rules "$pkg" || return 1 + + # The AUR repo is a separate remote living alongside the build dir; adding + # it here (rather than cloning into a fresh path) keeps the verified + # PKGBUILD and the thing being pushed byte-identical. + if [[ ! -d .git ]]; then + git init -q -b master . + git remote add origin "ssh://aur@$AUR_HOST/$pkg.git" + fi + git remote get-url origin >/dev/null 2>&1 \ + || git remote add origin "ssh://aur@$AUR_HOST/$pkg.git" + + # Fetch first: an existing AUR package must be built on, not overwritten. + if git fetch -q origin master 2>/dev/null; then + git reset -q --soft FETCH_HEAD 2>/dev/null || true + fi + + cat > .gitignore <<'EOF' +# Build products; the AUR tracks only the recipe. +*.pkg.tar.zst +*.tar.gz +src/ +pkg/ +EOF + + git add PKGBUILD .SRCINFO .gitignore + if git diff --cached --quiet; then + info "already up to date on the AUR; nothing to push" + PUBLISHED+=("$pkg v$version (unchanged)") + return 0 + fi + + git -c user.name='Krzysztof Rudnicki' \ + -c user.email='krzysztofrudnicki0@gmail.com' \ + commit -q -m "$pkg $version-1" + + if ((DRY_RUN)); then + info "DRY RUN — not pushing" + PUBLISHED+=("$pkg v$version (dry run)") + return 0 + fi + + info "pushing to the AUR" + git push -q origin master 2>&1 | tail -3 || { warn "$pkg: push failed"; return 1; } + PUBLISHED+=("$pkg v$version") + return 0 +} + +main() { + require_tools + [[ -f "$GATE" ]] || die "missing $GATE" + mkdir -p "$AUR_ROOT" + + ensure_key + if ((SKIP_WAIT)); then + ssh_works || die "SSH key is not accepted yet; drop --skip-wait to wait for it" + else + wait_for_registration + wait_for_ssh + fi + + local entry pkg repo sub binary expected + for entry in "${PACKAGES[@]}"; do + IFS=: read -r pkg repo sub binary expected <<<"$entry" + [[ -z "$ONLY" || "$ONLY" == "$pkg" ]] || continue + if ! publish_one "$pkg" "$repo" "$sub" "$binary" "$expected"; then + FAILED+=("$pkg") + fi + done + + log "Summary" + if ((${#PUBLISHED[@]})); then + printf ' published: %s\n' "${PUBLISHED[@]}" + fi + if ((${#FAILED[@]})); then + printf '\033[1;31m FAILED: %s\033[0m\n' "${FAILED[@]}" + info "Nothing broken was pushed; fix and re-run (safe to repeat)." + return 1 + fi + if ((${#PUBLISHED[@]} == 0)); then + warn "nothing to do" + return 1 + fi + printf '\n Check: https://aur.archlinux.org/packages/todo-flutter\n' + return 0 +} + +while [[ $# -gt 0 ]]; do + case $1 in + --skip-wait) SKIP_WAIT=1; shift ;; + --dry-run) DRY_RUN=1; shift ;; + --only) ONLY="$2"; shift 2 ;; + --poll) POLL_SECONDS="$2"; shift 2 ;; + -h|--help) usage ;; + *) die "Unknown option: $1" ;; + esac +done + +main "$@"