#!/usr/bin/env bash # scripts/migration-smoke.sh — rehearse a release's EF migrations against a COPY of the real prod # database before promoting the new image (ersatztv#315). Data-plane release rigor. # # WHY: CI's `migrations` job only proves a migration is well-formed against a FRESH, empty DB # (model-drift + apply-to-fresh per provider). It cannot prove the migration applies cleanly to the # accumulated prod SQLite — real row volume, historical values, and the post-migration data steps # ErsatzTV runs on startup (DatabaseMigratorService → DbInitializer + PopulatePathHashes over the real # MediaFile table). A migration green on a fresh DB can still fail or corrupt on prod. This smoke boots # the NEW image against a THROWAWAY copy of the latest prod backup, waits for the migrator to finish, # and reports pass/fail — so a bad migration is caught on a copy, not mid-deploy on live data. # # It is designed to run ON the docker host (bumblebee) as a Komodo pre-deploy step (server-management # owns that wiring — see ersatztv#315), or by hand before a release. It NEVER touches the live prod DB. # # Usage: # scripts/migration-smoke.sh --image [--db ] [--timeout ] # # --image Image to rehearse — the one about to be PROMOTED (e.g. # 192.168.1.95:3000/timothy/ersatztv:26.6.0). Required. # --db A prod SQLite backup file to copy and migrate. Default: the newest # $ETV_BACKUP_DIR/*/ersatztv.sqlite3 (ETV_BACKUP_DIR defaults to # ~/downloadswarm/ersatztv-backups — where the Komodo pre-deploy hook writes them). # --timeout Seconds to wait for migrations to finish (default 180). Big prod DBs + the path-hash # backfill can take a while on first upgrade. # # Exit 0 = the new image applied all pending migrations to the prod-copy and booted cleanly. # Exit 1 = a migration failed / the container exited before finishing / it never booted (logs dumped). # Exit 2 = usage / precondition error (no image, no backup found, docker missing). # # The container, the temp config dir, and the DB copy are always cleaned up on exit. set -euo pipefail IMAGE="" DB="" TIMEOUT=180 BACKUP_DIR="${ETV_BACKUP_DIR:-$HOME/downloadswarm/ersatztv-backups}" die() { echo "migration-smoke: $*" >&2; exit 2; } while [ $# -gt 0 ]; do case "$1" in --image) IMAGE="${2:-}"; shift 2 ;; --db) DB="${2:-}"; shift 2 ;; --timeout) TIMEOUT="${2:-}"; shift 2 ;; -h|--help) sed -n '2,40p' "$0"; exit 0 ;; *) die "unknown argument: $1 (see --help)" ;; esac done command -v docker >/dev/null 2>&1 || die "docker not found on PATH (run this on the docker host)" [ -n "$IMAGE" ] || die "--image is required (the image about to be promoted)" case "$TIMEOUT" in ''|*[!0-9]*) die "--timeout must be a positive integer of seconds (got: '$TIMEOUT')";; esac [ "$TIMEOUT" -gt 0 ] || die "--timeout must be greater than 0" # Resolve the backup DB: explicit --db, else newest ersatztv.sqlite3 under $BACKUP_DIR. if [ -z "$DB" ]; then DB=$(find "$BACKUP_DIR" -type f -name 'ersatztv.sqlite3' -printf '%T@ %p\n' 2>/dev/null \ | sort -rn | head -1 | cut -d' ' -f2-) [ -n "$DB" ] || die "no ersatztv.sqlite3 backup found under $BACKUP_DIR (pass --db explicitly)" echo "migration-smoke: using newest backup: $DB" fi [ -f "$DB" ] || die "backup DB not found: $DB" NAME="etv-migsmoke-$$" WORK="$(mktemp -d)" # shellcheck disable=SC2329 # invoked via 'trap cleanup EXIT' cleanup() { docker rm -f "$NAME" >/dev/null 2>&1 || true # The container runs as root and writes root-owned files into $WORK/config (cache, logs, search-index, # data-protection), which a non-root invoker cannot delete. Remove them from inside a throwaway root # container that mounts $WORK, then drop the (host-owned) temp dir — otherwise each run leaks the # multi-hundred-MB DB copy + config. if [ -n "${IMAGE:-}" ]; then docker run --rm -v "$WORK:/work" --entrypoint /bin/sh "$IMAGE" -c 'rm -rf /work/config' >/dev/null 2>&1 || true fi rm -rf "$WORK" 2>/dev/null || true } trap cleanup EXIT # Seed a throwaway config dir with a WRITABLE copy of the backup (single checkpointed file — we do NOT # copy any -wal/-shm; a backup snapshot is self-contained, and the container will create its own WAL). mkdir -p "$WORK/config" cp "$DB" "$WORK/config/ersatztv.sqlite3" chmod u+rw "$WORK/config/ersatztv.sqlite3" SIZE=$(du -h "$WORK/config/ersatztv.sqlite3" | cut -f1) echo "migration-smoke: image=$IMAGE db-copy=${SIZE} timeout=${TIMEOUT}s" if ! docker pull "$IMAGE" >/dev/null 2>&1; then # Not fatal (offline / locally-built images), but for a mutable tag (:latest) a stale local copy # would rehearse the wrong image — so we log the resolved image ID below for the operator to verify. echo "migration-smoke: WARNING — pull failed; rehearsing whatever '$IMAGE' resolves to locally" fi IMG_ID=$(docker image inspect "$IMAGE" --format '{{.Id}}' 2>/dev/null || echo "unknown") echo "migration-smoke: rehearsing $IMAGE ($IMG_ID)" # Boot the new image against the prod-copy. DatabaseMigratorService (a BackgroundService) applies # pending migrations on startup; it logs "Applying database migrations" then "Done applying database # migrations", and on failure the host stops (default BackgroundServiceExceptionBehavior = StopHost), # so the container exits. We gate PASS on the "Done" line, FAIL on early exit / a migration exception. # --memory-swap equal to --memory disables swap for this container. Without it Docker defaults # --memory-swap to 2x --memory, so `--memory 2g` alone silently grants 2g RAM + 2g of swap # (ersatztv#406). That matters here specifically: this smoke runs on the PROD host in the release # path, so a runaway migration should die loudly against its cap rather than quietly swap out the # box that is serving media. docker run -d --name "$NAME" --memory 2g --memory-swap 2g \ -e ETV_CONFIG_FOLDER=/config \ -e ETV_TRANSCODE_FOLDER=/tmp/etv/transcode \ -v "$WORK/config:/config" \ "$IMAGE" >/dev/null || die "docker run failed for $IMAGE" # PASS gate = the migrator's completion log line. FAIL = the container exiting before it (a failed # migration faults the BackgroundService → default StopHost → the host stops → the container exits, so # early-exit IS the reliable migration-failure signal), or the timeout below. We deliberately do NOT # grep logs for error strings: ErsatzTV's startup is noisy (library scans against absent media mounts, # EF connection retries) and a broad error-regex would false-FAIL a good migration. DONE_RE='Done applying database migrations' mig_done=0 deadline=$((SECONDS + TIMEOUT)) while [ $SECONDS -lt $deadline ]; do if docker logs "$NAME" 2>&1 | grep -qE "$DONE_RE"; then mig_done=1; break; fi if [ -z "$(docker ps -q --filter name="$NAME" --filter status=running)" ]; then echo "migration-smoke: FAIL — container exited before finishing migrations (a failed migration stops the host):" docker logs "$NAME" 2>&1 | tail -n 40 exit 1 fi sleep 2 done if [ "$mig_done" != "1" ]; then echo "migration-smoke: FAIL — migrations did not finish within ${TIMEOUT}s:"; docker logs "$NAME" 2>&1 | tail -n 40 exit 1 fi echo "migration-smoke: migrations applied cleanly to the prod-copy." # Boot sanity: confirm the app then serves a DB-backed IPTV endpoint (proves DatabaseIsReady + the # app is healthy post-migration, not just that the migrator finished). The image ships python3. serve_ok=0 for _ in $(seq 1 30); do if docker exec "$NAME" python3 -c "import urllib.request,sys; urllib.request.urlopen('http://localhost:8409/iptv/channels.m3u',timeout=5)" >/dev/null 2>&1; then serve_ok=1; break fi sleep 2 done if [ "$serve_ok" != "1" ]; then echo "migration-smoke: FAIL — migrations finished but the app did not serve /iptv/channels.m3u post-boot:" docker logs "$NAME" 2>&1 | tail -n 40 exit 1 fi echo "migration-smoke: PASS — $IMAGE migrated the prod-copy and booted healthy." exit 0