kuhyx
71e4a6115e
Merges ~/ck3-knowledge and ~/kcd-knowledge, which were 90% duplicate code and
neither of which had ever been committed anywhere.
Works on any wiki, not just MediaWiki:
- sources.py splits the engine-specific parts (enumerate, fetch) behind a Source
protocol. MediaWikiSource is preferred whenever an api.php exists;
SitemapHtmlSource handles custom engines with no API (verified on Fextralife).
Everything downstream was already source-agnostic.
- scripts/detect_wiki.py probes /api.php, /w/api.php then /sitemap.xml and prints
the config to use. The API path and article URL vary independently — Fandom
serves /api.php but /wiki/{Title}.
- Profiles are plain env files, since config.py was already fully env-driven.
Per-wiki behaviour that must be measured rather than assumed:
- Subpage policy. On the CK3 wiki "A/B" is a mod; on the KCD wiki it is the
sequel, and rejecting subpages deletes 441 pages of wanted content.
- Noise sections. "Other languages" tables were 7% of KCD chunks and matched
English queries by repeating every title in eight languages.
- Variant tagging for wikis covering two editions, with scopes that exclude the
other edition but keep both/unknown pages.
Build performance: length bucketing in embed_efficiently is 2.7x (fastembed pads
each batch to its longest member; 184-char median vs 2000-char max). With noise
sections that is 61 min -> 20 min on a 6,140-page wiki at half the workers.
Fandom portable-infoboxes are transformed to tables before markdownify —
otherwise stat values survive but detach from their labels.
100% branch coverage, ruff clean.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
79 lines
3.0 KiB
Bash
Executable File
79 lines
3.0 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# ============================================================================
|
|
# run.sh — orchestrate a wiki knowledge base
|
|
# ./run.sh install create venv + install deps
|
|
# ./run.sh scrape [--limit N] mirror the wiki into corpus/
|
|
# ./run.sh index build FTS5 + embeddings index
|
|
# ./run.sh search "<query>" hybrid search from the CLI
|
|
# ./run.sh serve run the MCP server (stdio)
|
|
# ./run.sh check ruff + pytest (with coverage)
|
|
# ============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
readonly VENV_PY="${SCRIPT_DIR}/.venv/bin/python"
|
|
|
|
need_venv() {
|
|
if [[ ! -x "$VENV_PY" ]]; then
|
|
echo "venv missing — run: ./run.sh install" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
usage() {
|
|
grep -E '^# \./run\.sh' "${BASH_SOURCE[0]}" | sed 's/^# //'
|
|
exit "${1:-0}"
|
|
}
|
|
|
|
# A profile is just an env file (see profiles/README.md) — config.py reads every
|
|
# setting from WIKI_KB_* variables, so no plugin machinery is needed.
|
|
# A profile is just an env file (see profiles/README.md) — config.py reads every
|
|
# setting from WIKI_KB_* variables, so no plugin machinery is needed. Parsed line by
|
|
# line rather than sourced: values legitimately contain spaces ("other languages"),
|
|
# which `source` would try to execute.
|
|
load_profile() {
|
|
local name="$1" file="${SCRIPT_DIR}/profiles/${1}.env" line key value
|
|
[[ -f "$file" ]] || {
|
|
echo "No such profile: $name (available: $(cd "${SCRIPT_DIR}/profiles" && \
|
|
ls *.env 2>/dev/null | sed 's/\.env$//' | tr '\n' ' '))" >&2
|
|
exit 1
|
|
}
|
|
while IFS= read -r line || [[ -n "$line" ]]; do
|
|
[[ -z "$line" || "$line" == \#* ]] && continue
|
|
key="${line%%=*}"
|
|
value="${line#*=}"
|
|
value="${value//\$HOME/$HOME}" # only $HOME is expanded, deliberately
|
|
export "$key=$value"
|
|
done < "$file"
|
|
echo "profile: $name -> ${WIKI_KB_HOST:-?} (root ${WIKI_KB_ROOT:-?})" >&2
|
|
}
|
|
|
|
main() {
|
|
# --profile <name> may precede the command; it only sets env vars.
|
|
if [[ "${1:-}" == "--profile" ]]; then
|
|
[[ $# -ge 2 ]] || { echo "--profile needs a name" >&2; exit 1; }
|
|
load_profile "$2"
|
|
shift 2
|
|
fi
|
|
local cmd="${1:-}"
|
|
[[ $# -gt 0 ]] && shift || true
|
|
case "$cmd" in
|
|
install) exec "${SCRIPT_DIR}/install.sh" ;;
|
|
scrape) need_venv; exec "$VENV_PY" -u -m wikikb.scraper "$@" ;;
|
|
index) need_venv; exec "$VENV_PY" -u -m wikikb.indexer "$@" ;;
|
|
search) need_venv; exec "$VENV_PY" -u -m wikikb.search "$@" ;;
|
|
serve) need_venv; exec "$VENV_PY" -u -m wikikb.server "$@" ;;
|
|
check)
|
|
need_venv
|
|
"$VENV_PY" -m ruff check "${SCRIPT_DIR}/wikikb" "${SCRIPT_DIR}/tests"
|
|
"$VENV_PY" -m pytest --cov=wikikb --cov-report=term-missing "${SCRIPT_DIR}/tests"
|
|
;;
|
|
""|-h|--help) usage 0 ;;
|
|
*) echo "Unknown command: $cmd" >&2; usage 1 ;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|