ersatztv/scripts/e2e-functional.sh

#!/usr/bin/env bash
# scripts/e2e-functional.sh — drive the manual live-E2E functional flows against a RUNNING
# ErsatzTV instance and assert their HTTP contracts. This is the codified, automatable half of the
# ad-hoc curl scenarios sessions have been running by hand (ersatztv#299); it runs both locally and
# in CI (the `functional-e2e` job in .gitea/workflows/docker-build.yml).
#
# It does NOT boot the app — pair it with scripts/e2e-local.sh (which builds/launches and prints the
# CONFIG_DIR), or point it at any already-running instance:
#
#   scripts/e2e-functional.sh [BASE_URL] [CONFIG_DIR]
#
#   BASE_URL     default http://localhost:8409  (the app's UI+API port; ETV_UI_PORT)
#   CONFIG_DIR   the instance's config folder — REQUIRED, so we can read its machine api key
#                ($CONFIG_DIR/api.key). The /api surface is fail-closed (Api:RequireKeyForReads
#                defaults true), so every /api call sends X-Api-Key.
#
# Scope:
#   1. Legacy -> SPA redirect sweep (+ the /api,/artwork never-redirect exemption).       [curl-only]
#   2. Machine-key data-plane: read-gate, library-scan lifecycle (404/202/scan-status),
#      optimistic-concurrency If-Match/412.                                               [curl-only]
#   3. Lock-contention 409s: scan-in-progress + external-collections-in-progress          [seeds DB;
#      (ersatztv#363) — see that section's header for how each is made deterministic.      scan=ffmpeg]
#   4. Auth / CSRF / security-stamp (setup-claim, read-gate, CSRF gate, login, logout+revoke).
#
# Section 3 is the first that isn't curl-only: it seeds rows the API can't create (a LibraryPath and
# a media-source) straight into the running instance's SQLite DB (via python3's stdlib sqlite3, whose
# busy-timeout retry serializes behind the app's writer) and, for the scan flow, synthesizes media
# with ffmpeg. Both 409s
# are made DETERMINISTIC (no racy sleep-and-hope) — see the section header for how. The scan flow is
# skipped (advisory) when ffmpeg is absent; the collections flow needs neither ffmpeg nor a scanner.
#
# Deliberately still OUT of scope (tracked as ersatztv#363 follow-ups):
#   - the playout-build lock 409 + isLocked projection (#215) — needs a seeded schedule/playout and a
#     build window wide enough to observe without racing,
#   - the genuinely UI-interactive Playwright (headless) flows — a separate CI browser-tooling lift.
#
# Exit status: 0 if every assertion passed, 1 if any failed. Assertions keep running after a
# failure so one run reports the full picture.

set -uo pipefail

BASE_URL="${1:-http://localhost:8409}"
CONFIG_DIR="${2:-}"

if [ -z "$CONFIG_DIR" ]; then
  echo "error: CONFIG_DIR (arg 2) is required — needed to read the machine api key ($CONFIG_DIR/api.key)." >&2
  echo "usage: scripts/e2e-functional.sh [BASE_URL] CONFIG_DIR" >&2
  exit 2
fi

KEY_FILE="$CONFIG_DIR/api.key"
if [ ! -f "$KEY_FILE" ]; then
  echo "error: $KEY_FILE not found. Is the instance running with ETV_CONFIG_FOLDER=$CONFIG_DIR?" >&2
  exit 2
fi
API_KEY="$(cat "$KEY_FILE")"

PASS=0
FAIL=0
FAILURES=()

# ---- assertion helpers -------------------------------------------------------------------------

# ok/bad: record a single assertion result.
ok()  { PASS=$((PASS + 1)); printf '  \033[32mPASS\033[0m %s\n' "$1"; }
bad() { FAIL=$((FAIL + 1)); FAILURES+=("$1"); printf '  \033[31mFAIL\033[0m %s\n' "$1"; }

# status_of METHOD URL [extra curl args...] -> prints the HTTP status code.
status_of() {
  local method="$1" url="$2"; shift 2
  curl -s -o /dev/null -w '%{http_code}' -X "$method" "$@" "$url"
}

# expect_status "desc" EXPECTED METHOD URL [extra curl args...]
expect_status() {
  local desc="$1" expected="$2" method="$3" url="$4"; shift 4
  local got; got="$(status_of "$method" "$url" "$@")"
  if [ "$got" = "$expected" ]; then ok "$desc (=$expected)"; else bad "$desc — expected $expected, got $got  [$method $url]"; fi
}

# api(): curl against the /api surface with the machine key + JSON content type.
api() { curl -s -H "X-Api-Key: $API_KEY" -H 'Content-Type: application/json' "$@"; }

# json_field FILE FIELD -> value of a top-level JSON field (via python3, always present here).
json_field() { python3 -c "import json,sys; print(json.load(open('$1')).get('$2',''))"; }

section() { printf '\n\033[1m== %s ==\033[0m\n' "$1"; }

echo "Functional E2E against $BASE_URL (config: $CONFIG_DIR)"

# ---- 1. Legacy -> SPA redirect sweep ------------------------------------------------------------
# The legacy branch 302s GET requests to their /app equivalent; /api, /artwork, /docs, /openapi are
# NEVER redirected (they 4xx from their own handlers/fallback). See ErsatzTV/LegacyUiRedirects.cs +
# Startup.cs MapFallback, and docs/blazor-route-parity.md.
section "Legacy -> SPA redirects"

# redirects_to "src" "expected /app location suffix"
redirects_to() {
  local src="$1" want="$2"
  local code loc
  code="$(curl -s -o /dev/null -w '%{http_code}' "$BASE_URL$src")"
  loc="$(curl -s -o /dev/null -w '%{redirect_url}' "$BASE_URL$src")"
  if [ "$code" = "302" ] && [ "$loc" = "$BASE_URL$want" ]; then
    ok "$src -> 302 $want"
  else
    bad "$src — expected 302 -> $BASE_URL$want, got $code -> ${loc:-<none>}"
  fi
}
redirects_to "/"                               "/app"
redirects_to "/channels"                       "/app/channels"
redirects_to "/channels/5"                     "/app/edit-channel/5"
redirects_to "/settings/ffmpeg"                "/app/settings/streaming"
redirects_to "/media/movies"                   "/app/media?kind=movies"
redirects_to "/media/sources/plex/3/libraries" "/app/libraries/plex/3/sync"

# The exempt prefixes must NOT redirect to /app — they 4xx from their own handlers (an unknown
# /api path 404s; /artwork rejects a non-artwork path 400). Assert only "did not 302 to /app".
never_redirects() {
  local src="$1"
  local code loc
  code="$(curl -s -o /dev/null -w '%{http_code}' "$BASE_URL$src")"
  loc="$(curl -s -o /dev/null -w '%{redirect_url}' "$BASE_URL$src")"
  if [ "$code" != "302" ] && [ -z "$loc" ] && [ "${code:0:1}" = "4" ]; then
    ok "$src -> not redirected (${code})"
  else
    bad "$src — expected a non-redirect 4xx, got $code -> ${loc:-<none>}"
  fi
}
never_redirects "/api/does-not-exist"
never_redirects "/artwork/does-not-exist"

# ---- 2. Machine-key data-plane contracts (scan lifecycle + If-Match/412) ------------------------
# These use the machine key (X-Api-Key), which is independent of the session and does NOT claim the
# local admin — so the fresh-config setup-claim flow in section 3 still sees setupRequired:true.
section "Read gate (Api:RequireKeyForReads = true)"
expect_status "GET /api/v1/channels without a key -> 401" 401 GET "$BASE_URL/api/v1/channels"
expect_status "GET /api/v1/channels with the machine key -> 200" 200 GET "$BASE_URL/api/v1/channels" \
  -H "X-Api-Key: $API_KEY"

section "Library scan lifecycle contract"
# Create an empty local library (no LibraryPath) — enough to exercise the enqueue/status contract
# without any media on disk (QueueLibraryScanByLibraryIdHandler never inspects paths).
LIB_JSON="$(mktemp)"
api -X POST "$BASE_URL/api/v1/libraries/local" -d '{"name":"E2E Functional Library"}' -o "$LIB_JSON" -w ''
LIB_ID="$(json_field "$LIB_JSON" id)"
if [ -n "$LIB_ID" ]; then ok "created local library id=$LIB_ID"; else bad "could not create local library (body: $(cat "$LIB_JSON"))"; fi
expect_status "POST /api/v1/libraries/$LIB_ID/scan -> 202 (queued)" 202 POST "$BASE_URL/api/v1/libraries/$LIB_ID/scan" \
  -H "X-Api-Key: $API_KEY" -H 'Content-Type: application/json' -d '{}'
expect_status "POST /api/v1/libraries/99999/scan -> 404 (unknown library)" 404 POST "$BASE_URL/api/v1/libraries/99999/scan" \
  -H "X-Api-Key: $API_KEY" -H 'Content-Type: application/json' -d '{}'
expect_status "GET /api/v1/libraries/scan-status -> 200" 200 GET "$BASE_URL/api/v1/libraries/scan-status" \
  -H "X-Api-Key: $API_KEY"

section "Optimistic concurrency (If-Match / 412)"
# rerun-collections is the reference concurrency-aware editor: GET emits an ETag, PUT enforces
# If-Match. It needs an existing collection to target, so create one first.
COLL_JSON="$(mktemp)"
api -X POST "$BASE_URL/api/v1/collections" -d '{"name":"E2E Functional Collection"}' -o "$COLL_JSON" -w ''
COLL_ID="$(json_field "$COLL_JSON" id)"
RC_JSON="$(mktemp)"
api -X POST "$BASE_URL/api/v1/rerun-collections" -o "$RC_JSON" -w '' -d "{
  \"name\":\"E2E Functional Rerun\",\"collectionType\":\"Collection\",\"selectedId\":$COLL_ID,
  \"firstRunPlaybackOrder\":\"Chronological\",\"rerunPlaybackOrder\":\"Chronological\"}"
RC_ID="$(json_field "$RC_JSON" id)"
if [ -n "$RC_ID" ]; then ok "created rerun-collection id=$RC_ID (targets collection $COLL_ID)"; else bad "could not create rerun-collection (body: $(cat "$RC_JSON"))"; fi

# GET must surface the current version as a quoted-integer ETag.
ETAG="$(api -D - -o /dev/null "$BASE_URL/api/v1/rerun-collections/$RC_ID" | awk 'tolower($1)=="etag:"{print $2}' | tr -d '\r')"
if [ -n "$ETAG" ]; then ok "GET rerun-collection $RC_ID surfaces ETag $ETAG"; else bad "GET rerun-collection $RC_ID has no ETag header"; fi

RC_BODY="{\"name\":\"E2E Functional Rerun v2\",\"selectedId\":$COLL_ID,\"firstRunPlaybackOrder\":\"Chronological\",\"rerunPlaybackOrder\":\"Chronological\"}"
# A well-formed but non-current version must be rejected 412 (999999 can never be the live version here).
expect_status "PUT with a stale If-Match -> 412" 412 PUT "$BASE_URL/api/v1/rerun-collections/$RC_ID" \
  -H "X-Api-Key: $API_KEY" -H 'Content-Type: application/json' -H 'If-Match: "999999"' -d "$RC_BODY"
# The current ETag must be accepted.
expect_status "PUT with the current If-Match -> 200" 200 PUT "$BASE_URL/api/v1/rerun-collections/$RC_ID" \
  -H "X-Api-Key: $API_KEY" -H 'Content-Type: application/json' -H "If-Match: $ETAG" -d "$RC_BODY"
# A malformed (non-integer) If-Match is a client error.
expect_status "PUT with a malformed If-Match -> 400" 400 PUT "$BASE_URL/api/v1/rerun-collections/$RC_ID" \
  -H "X-Api-Key: $API_KEY" -H 'Content-Type: application/json' -H 'If-Match: not-a-version' -d "$RC_BODY"

# ---- 3. Lock contention (409): scan-in-progress + collections-in-progress -----------------------
# The IEntityLocker 409 contracts the first cut deferred (ersatztv#299 -> #363). Both are made
# DETERMINISTIC (no sleep-and-hope) by only firing the racing request once the lock is provably held:
#
#   A. Library-scan "already scanning" 409. A real scan must be in flight, so we seed a Shows library
#      with enough media that the scanner subprocess (out-of-process, one ffprobe per file) runs for a
#      few seconds. The scan-status "active" window is a strict SUBSET of the scan lock's held window
#      (ScannerProxyService.StartScan happens after locker.LockLibrary, EndScan before UnlockLibrary --
#      ErsatzTV/Services/ScannerService.cs), so once GET /libraries/scan-status shows the library
#      active the lock is provably held and a second POST .../scan is a 409 -- deterministic bar a
#      tiny residual TOCTOU gap (observe-active -> 2nd POST lands) that the multi-second scan covers.
#
#   B. External-collections "already scanning" 409 (per-family lock). The controller takes the lock
#      SYNCHRONOUSLY before returning 202 (JellyfinMediaSourcesController), so the 202 itself proves
#      the lock is held (race-free by construction); pointing the seeded source at a non-routable
#      address keeps the background sync hung so the window stays wide open. GET
#      /media-sources/collections-scan-status corroborates.
#
# Seeding writes rows the API can't create (LibraryPath, media-source) directly into the instance's
# SQLite DB via python3's stdlib sqlite3 (already a hard dep of this harness); the connection's
# busy-timeout retry serializes this second writer behind the app. Runs on the machine key, before the
# auth section (which claims the admin) -- these never claim it. Flow A is skipped (advisory) without
# ffmpeg; Flow B needs neither ffmpeg nor a scanner.
section "Lock contention (409): scan-in-progress + collections-in-progress"

DB="$CONFIG_DIR/ersatztv.sqlite3"

# seed_library_path PATH LIBRARY_ID -- attach a folder to a built-in local library (parameterized);
# prints the new LibraryPath row id so the caller can detect a silent seeding failure.
seed_library_path() {
  python3 - "$DB" "$1" "$2" <<'PY'
import sqlite3, sys
con = sqlite3.connect(sys.argv[1], timeout=10)   # busy-timeout retry serializes this second writer
cur = con.cursor()
cur.execute("INSERT INTO LibraryPath (Path, LibraryId) VALUES (?, ?)", (sys.argv[2], int(sys.argv[3])))
con.commit()
print(cur.lastrowid)
PY
}

# seed_jellyfin_source ADDRESS -- create MediaSource + JellyfinMediaSource + JellyfinConnection rows,
# print the new media-source id. The row only needs to EXIST for the {id} to resolve (404 otherwise);
# a non-routable ADDRESS makes the background collections sync hang so the family lock stays held.
seed_jellyfin_source() {
  python3 - "$DB" "$1" <<'PY'
import sqlite3, sys
con = sqlite3.connect(sys.argv[1], timeout=10)   # busy-timeout retry serializes this second writer
cur = con.cursor()
cur.execute("INSERT INTO MediaSource DEFAULT VALUES")
msid = cur.lastrowid
cur.execute("INSERT INTO JellyfinMediaSource (Id, ServerName, OperatingSystem) VALUES (?, 'E2E-Lock-Jellyfin', 'linux')", (msid,))
cur.execute("INSERT INTO JellyfinConnection (Address, JellyfinMediaSourceId) VALUES (?, ?)", (sys.argv[2], msid))
con.commit()
print(msid)
PY
}

skip() { printf '  \033[33mSKIP\033[0m %s\n' "$1"; }

if [ ! -f "$DB" ]; then
  bad "expected the instance SQLite DB at $DB (needed to seed the lock-flow rows)"
else
  # --- Flow A: library-scan already-scanning 409 (needs seeded media + the scanner subprocess) ---
  if command -v ffmpeg >/dev/null 2>&1; then
    SEED_ROOT="$(mktemp -d)/tv"
    SEED_SEASON="$SEED_ROOT/Show Lock/Season 01"
    mkdir -p "$SEED_SEASON"
    SEED_SRC="$(mktemp -d)/seed.mkv"
    if ffmpeg -y -loglevel error -f lavfi -i testsrc=duration=1:size=160x120:rate=5 \
         -c:v libx264 -pix_fmt yuv420p "$SEED_SRC" >/dev/null 2>&1; then
      # 60 distinct episode files (the scanner ffprobes each) -> a multi-second scan even on fast CI
      # I/O, so the poll->fire gap (one HTTP round-trip) is comfortably inside the lock window.
      for n in $(seq -w 1 60); do cp "$SEED_SRC" "$SEED_SEASON/Show Lock - s01e$n.mkv"; done
      # Shows is the built-in library Id=2 (docs/e2e-local.md recipe). Check the seed succeeded -- a
      # silent failure would leave library 2 with no paths, the scan a no-op, and the 409 un-asserted.
      if [ -z "$(seed_library_path "$SEED_ROOT" 2)" ]; then
        bad "could not seed a LibraryPath row for the scan-lock flow (library 2)"
      else
        # No ?deep -- local scans always ForceScan (ForceScanLocalLibrary.ForceScan is hardcoded true;
        # `deep` only matters for Plex/Jellyfin/Emby), so a fresh library 2 scan runs unconditionally.
        expect_status "POST /libraries/2/scan -> 202 (queued)" 202 POST "$BASE_URL/api/v1/libraries/2/scan" \
          -H "X-Api-Key: $API_KEY" -H 'Content-Type: application/json' -d '{}'
        # Poll scan-status until the scan is provably in flight, then fire the racing scan. The active
        # window is a strict subset of the lock's held window, so this is deterministic bar a tiny
        # residual TOCTOU gap (observe-active -> 2nd POST lands) that the multi-second scan covers.
        # Match the id exactly (trailing comma) so "libraryId":2 can't match "libraryId":20 etc.
        scan_active=""
        for _ in $(seq 1 100); do
          if api "$BASE_URL/api/v1/libraries/scan-status" | grep -q '"libraryId":2,'; then scan_active=1; break; fi
          sleep 0.1
        done
        if [ -n "$scan_active" ]; then
          ok "scan-status shows library 2 active (scan lock provably held)"
          expect_status "second POST /libraries/2/scan while scanning -> 409" 409 POST "$BASE_URL/api/v1/libraries/2/scan" \
            -H "X-Api-Key: $API_KEY" -H 'Content-Type: application/json' -d '{}'
        else
          # Never observed active within 10s: don't assert a race we can't prove we won (advisory skip).
          skip "library-2 scan never observed active within 10s -> not asserting the 409 (seed=60 files)"
        fi
      fi
    else
      bad "ffmpeg present but failed to synthesize the seed clip for the scan-lock flow"
    fi
  else
    skip "ffmpeg not found -> skipping the library-scan 409 flow (advisory; needs seeded media + scanner)"
  fi

  # --- Flow B: external-collections already-scanning 409 (per-family lock; no ffmpeg/scanner) ---
  # TEST-NET-1 (RFC 5737) is guaranteed non-routable, so the background sync hangs; but Flow B is
  # race-free regardless (the 202 already proves the lock held -- the hang only widens the margin).
  MSID="$(seed_jellyfin_source 'http://192.0.2.1:8096')"
  if [ -n "$MSID" ]; then
    ok "seeded Jellyfin media source id=$MSID (non-routable -> sync hangs, lock stays held)"
    expect_status "POST /media-sources/jellyfin/$MSID/scan-collections -> 202" 202 POST "$BASE_URL/api/v1/media-sources/jellyfin/$MSID/scan-collections" \
      -H "X-Api-Key: $API_KEY" -H 'Content-Type: application/json' -d '{}'
    # The lock is taken synchronously before the 202, so it is provably held now; corroborate via status.
    if api "$BASE_URL/api/v1/media-sources/collections-scan-status" | grep -q '"family":"jellyfin"'; then
      ok "collections-scan-status shows the jellyfin family locked"
    else
      bad "collections-scan-status did not show the jellyfin family locked after a 202"
    fi
    expect_status "second POST .../jellyfin/$MSID/scan-collections while scanning -> 409" 409 POST "$BASE_URL/api/v1/media-sources/jellyfin/$MSID/scan-collections" \
      -H "X-Api-Key: $API_KEY" -H 'Content-Type: application/json' -d '{}'
    expect_status "POST .../jellyfin/99999/scan-collections (unknown source) -> 404" 404 POST "$BASE_URL/api/v1/media-sources/jellyfin/99999/scan-collections" \
      -H "X-Api-Key: $API_KEY" -H 'Content-Type: application/json' -d '{}'
  else
    bad "could not seed a Jellyfin media source row for the collections-lock flow"
  fi
fi

# ---- 4. Auth / CSRF / security-stamp flow -------------------------------------------------------
# Runs last: setup-claim is a one-shot on a fresh config and logout revokes the session. All prior
# sections used the machine key, which does not claim the admin, so setupRequired is still true here.
section "Auth: setup-claim, CSRF gate, login, logout + stamp revocation"

CFG_JSON="$(mktemp)"
curl -s "$BASE_URL/api/v1/auth/config" -o "$CFG_JSON"
if [ "$(json_field "$CFG_JSON" setupRequired)" = "True" ]; then ok "fresh config: setupRequired=true"; else bad "expected setupRequired=true on a fresh config (body: $(cat "$CFG_JSON"))"; fi

JAR="$(mktemp)"
# Claim the local admin (setup is CSRF-gated like every mutation).
expect_status "POST /api/v1/auth/setup -> 200 (claims admin, issues session)" 200 POST "$BASE_URL/api/v1/auth/setup" \
  -c "$JAR" -H 'Content-Type: application/json' -H 'X-CSRF: 1' -d '{"username":"admin","password":"e2e-pass-1234"}'
curl -s "$BASE_URL/api/v1/auth/config" -o "$CFG_JSON"
if [ "$(json_field "$CFG_JSON" setupRequired)" = "False" ]; then ok "after claim: setupRequired=false"; else bad "expected setupRequired=false after claim (body: $(cat "$CFG_JSON"))"; fi
expect_status "POST /api/v1/auth/setup again -> 409 (already configured)" 409 POST "$BASE_URL/api/v1/auth/setup" \
  -H 'Content-Type: application/json' -H 'X-CSRF: 1' -d '{"username":"admin2","password":"e2e-pass-1234"}'

# CSRF gate on a general /api session mutation: a POST with the session cookie but no X-CSRF is 403.
expect_status "session mutation without X-CSRF -> 403" 403 POST "$BASE_URL/api/v1/collections" \
  -b "$JAR" -H 'Content-Type: application/json' -d '{"name":"should-be-csrf-blocked"}'

# Login: wrong password 401, correct 200 (fresh cookie jar).
expect_status "login with wrong password -> 401" 401 POST "$BASE_URL/api/v1/auth/login" \
  -H 'Content-Type: application/json' -H 'X-CSRF: 1' -d '{"username":"admin","password":"nope"}'
JAR2="$(mktemp)"
expect_status "login with correct password -> 200" 200 POST "$BASE_URL/api/v1/auth/login" \
  -c "$JAR2" -H 'Content-Type: application/json' -H 'X-CSRF: 1' -d '{"username":"admin","password":"e2e-pass-1234"}'

# Logout is CSRF-gated; and after it, the same cookie is invalidated by the rotated security stamp.
expect_status "authed GET before logout -> 200" 200 GET "$BASE_URL/api/v1/auth/machine-key" -b "$JAR2"
expect_status "logout without X-CSRF -> 403" 403 POST "$BASE_URL/api/v1/auth/logout" -b "$JAR2"
expect_status "logout with X-CSRF -> 204" 204 POST "$BASE_URL/api/v1/auth/logout" -b "$JAR2" -H 'X-CSRF: 1'
expect_status "authed GET after logout (stamp revoked) -> 401" 401 GET "$BASE_URL/api/v1/auth/machine-key" -b "$JAR2"

# ---- summary ------------------------------------------------------------------------------------
printf '\n\033[1m== Summary ==\033[0m\n'
printf 'passed: %d   failed: %d\n' "$PASS" "$FAIL"
if [ "$FAIL" -ne 0 ]; then
  printf '\033[31mfailed assertions:\033[0m\n'
  for f in "${FAILURES[@]}"; do printf '  - %s\n' "$f"; done
  exit 1
fi
echo "all functional-E2E assertions passed."