ersatztv/scripts/security-scan.sh

#!/usr/bin/env bash
# scripts/security-scan.sh — out-of-ecosystem, black-box security scan of a release image (ersatztv#314).
#
# WHY: every OTHER security check we run is in-ecosystem / white-box (SonarAnalyzer, NetArchTest, the
# adversarial fork + Codex review passes, the api-docs/format/decisions CI gates, `dotnet list package
# --vulnerable`). They reason about our source the way we do, so they share our blind spots. This drives
# the RUNNING product from outside our C#/review stack — OWASP ZAP as an authenticated DAST against every
# `/api/v1` endpoint, plus semgrep as a SAST cross-check — to catch the classes structural review misses:
# missing/incorrect security headers, an endpoint that forgot its auth attribute, verb tampering,
# injection reflections, known-CVE fingerprints. It is a #197 exit criterion — a HARD GATE before remote
# exposure — and should be re-run each release (see docs/ci-cd.md → Security scanning).
#
# It runs on a docker host with registry access and semgrep on its non-interactive PATH (docs/ci-cd.md →
# Security scanning → Where/when), against a THROWAWAY container booted from the image under test —
# NEVER the deployed prod (or test) container: the authenticated active scan sends attack payloads to
# write endpoints (creating/deleting rows, triggering playout builds), so it must target a disposable
# instance with a fresh empty config volume. The container is torn down on exit.
#
# Usage:
#   scripts/security-scan.sh [IMAGE] [PORT]
#     IMAGE  container image to scan          (default: 192.168.1.95:3000/timothy/ersatztv:latest)
#     PORT   host port for the throwaway app  (default: 8411)
#   Env:
#     ETV_SCAN_OUT   host dir for reports      (default: /tmp/etv-scan-out)
#     ETV_SCAN_SKIP_SEMGREP=1  skip the SAST pass (DAST only) — the ONLY skip; a missing semgrep otherwise exits 2
#
# Reports land in $ETV_SCAN_OUT: zap-api-report.{html,md,json}. zap-api-scan.py's exit code is NOT a
# simple pass/fail — its wrapper contract (ersatztv#338, found via the #335 exact-image release scan
# returning 2 for a clean-enough report):
#   0     clean — no FAIL or WARN alerts
#   2     WARN  — only WARN-level alerts (triage, but NOT release-blocking) — do not fail the job on this
#   1     FAIL  — at least one FAIL-level alert — release-blocking
#   124   our own `timeout` wrapper killed a hung wrapper (see the NOTE below) — report is still usable
#   other the scanner/tool itself errored (bad args, crash) — not a scan result at all
# classify_zap_exit() (below) is the single place this contract is encoded. The SCRIPT's own exit is not
# ZAP's raw code; every non-zero exit is release-blocking:
#   0     ZAP clean or WARN-only, AND the SAST half ran clean (or was opted out: ETV_SCAN_SKIP_SEMGREP=1)
#   1     ZAP FAIL / timeout / tool error, or the throwaway target never came up or its key was unreadable
#   2     NOT ZAP's WARN: a scanner half did not run — semgrep missing (checked before the scan) or erroring
#         (after it), or docker missing. If ZAP already classified non-zero, that status stands and the
#         semgrep error is printed alongside it.
# A setup command that fails under `set -e` (e.g. `docker pull`) exits with that command's own status.

set -euo pipefail

# classify_zap_exit RC — sets ZAP_CLASS_MSG to an unambiguous one-line classification of ZAP's raw exit
# code and returns the exit status this SCRIPT should use for that outcome (see contract above). Kept as
# a standalone function (not inlined) so `--selftest` can exercise it without docker/ZAP.
classify_zap_exit() {
  local rc="$1"
  case "$rc" in
    0)   ZAP_CLASS_MSG="PASS — no FAIL or WARN alerts"; return 0 ;;
    2)   ZAP_CLASS_MSG="WARN — WARN-level alerts only; triage required, NOT release-blocking"; return 0 ;;
    1)   ZAP_CLASS_MSG="FAIL — at least one FAIL-level alert; release-blocking, do not ship"; return 1 ;;
    124) ZAP_CLASS_MSG="TIMEOUT — wrapper hit ETV_SCAN_TIMEOUT; triage the partial report before shipping"; return 1 ;;
    *)   ZAP_CLASS_MSG="TOOL ERROR (raw exit $rc) — scanner itself failed, this is not a scan result"; return 1 ;;
  esac
}

# require_semgrep — the SAST half is part of the gate: a missing semgrep is an ERROR naming the tool, never a
# warning that still exits 0 (ersatztv#931 — a v26.16.0 release gate could pass with its SAST half never run).
# The only skip is the explicit opt-out ETV_SCAN_SKIP_SEMGREP=1. Standalone so `--selftest` can exercise it.
require_semgrep() {
  [ "${ETV_SCAN_SKIP_SEMGREP:-0}" = "1" ] && return 0
  command -v semgrep >/dev/null && return 0
  echo "error: semgrep not found on PATH — the SAST half of the release gate cannot run. Install semgrep, or set ETV_SCAN_SKIP_SEMGREP=1 to skip it explicitly (DAST only)." >&2
  return 2
}

# run_semgrep ROOT — the SAST pass. Without --error semgrep exits 0 on findings (they print above), so ANY
# non-zero exit means semgrep itself failed (bad config, registry unreachable, crash): the SAST half did not
# run, which must fail the gate exactly like a missing semgrep (ersatztv#931). Returns semgrep's own status.
run_semgrep() {
  semgrep scan --config p/security-audit --config p/secrets --config p/csharp \
    --exclude web/node_modules --exclude design-system --metrics off --quiet "$1"
}

if [ "${1:-}" = "--selftest" ]; then
  # Lightweight regression check for the classification contract above (ersatztv#338 Done-when) — no
  # docker/ZAP required, so it can run anywhere (incl. off bumblebee, incl. in per-PR CI if ever added).
  selftest_failed=0
  selftest_check() {
    local rc="$1" want_status="$2" want_substr="$3" got_status
    if classify_zap_exit "$rc"; then got_status=0; else got_status=$?; fi
    if [ "$got_status" != "$want_status" ] || [[ "$ZAP_CLASS_MSG" != *"$want_substr"* ]]; then
      echo "SELFTEST FAIL: rc=$rc -> status=$got_status msg='$ZAP_CLASS_MSG' (want status=$want_status, msg containing '$want_substr')" >&2
      selftest_failed=1
    else
      echo "selftest ok: rc=$rc -> status=$got_status ($ZAP_CLASS_MSG)"
    fi
  }
  selftest_check 0 0 "PASS"
  selftest_check 2 0 "WARN"
  selftest_check 1 1 "FAIL"
  selftest_check 124 1 "TIMEOUT"
  selftest_check 3 1 "TOOL ERROR"
  selftest_check 77 1 "TOOL ERROR"
  # semgrep gate (ersatztv#931): absent -> non-zero naming the tool; opt-out -> 0; present -> 0.
  semgrep_check() {
    local label="$1" want_status="$2" want_substr="$3" got_status err
    err="$(require_semgrep 2>&1 >/dev/null)" && got_status=0 || got_status=$?
    if [ "$got_status" != "$want_status" ] || [[ "$err" != *"$want_substr"* ]]; then
      echo "SELFTEST FAIL: semgrep gate ($label) -> status=$got_status msg='$err' (want status=$want_status, msg containing '$want_substr')" >&2
      return 1
    else
      echo "selftest ok: semgrep gate ($label) -> status=$got_status"
    fi
  }
  sg_dir="$(mktemp -d)"; sg_empty="$(mktemp -d)"
  printf '#!/bin/sh\n' > "$sg_dir/semgrep"; chmod +x "$sg_dir/semgrep"
  (PATH="$sg_empty"; unset ETV_SCAN_SKIP_SEMGREP; semgrep_check "absent" 2 "semgrep") || selftest_failed=1
  (PATH="$sg_empty"; ETV_SCAN_SKIP_SEMGREP=0; semgrep_check "absent, skip=0" 2 "semgrep") || selftest_failed=1
  (PATH="$sg_empty"; ETV_SCAN_SKIP_SEMGREP=1; semgrep_check "absent, explicit opt-out" 0 "") || selftest_failed=1
  (PATH="$sg_dir"; unset ETV_SCAN_SKIP_SEMGREP; semgrep_check "present" 0 "") || selftest_failed=1
  # End-to-end through the real main path (pins the call sites, not just the helper). Absent semgrep + no
  # docker on PATH: must exit 2 with the semgrep message BEFORE the docker check (which also exits 2).
  out="$(env -u ETV_SCAN_SKIP_SEMGREP PATH="$sg_empty" "$BASH" "${BASH_SOURCE[0]}" 2>&1)" && rc=0 || rc=$?
  if [ "$rc" = 2 ] && [[ "$out" == *"error: semgrep not found"* ]]; then
    echo "selftest ok: script, semgrep absent -> exit 2 naming semgrep"
  else
    echo "SELFTEST FAIL: script with semgrep absent -> rc=$rc out='$out' (want rc=2 naming semgrep)" >&2; selftest_failed=1
  fi
  out="$(ETV_SCAN_SKIP_SEMGREP=1 PATH="$sg_empty" "$BASH" "${BASH_SOURCE[0]}" 2>&1)" && rc=0 || rc=$?
  if [ "$rc" = 2 ] && [[ "$out" == *"docker not found"* ]] && [[ "$out" != *"semgrep not found"* ]]; then
    echo "selftest ok: script, explicit opt-out -> gate skipped (stops at docker check)"
  else
    echo "SELFTEST FAIL: script with opt-out -> rc=$rc out='$out' (want to pass the semgrep gate)" >&2; selftest_failed=1
  fi
  # Full runs through the MAIN-path SAST block, with shimmed docker/curl/timeout (ZAP reports clean, so the
  # script's exit is decided by the semgrep half alone). The semgrep shim records each call in
  # $ETV_SCAN_OUT/semgrep.called and exits $SG_SHIM_RC, so a case can assert both the exit and whether
  # SAST actually ran. These pin the main-path opt-out test and the sg_rc check, not just the helper.
  sg_shim="$(mktemp -d)"; sg_out="$(mktemp -d)"
  printf '#!/bin/sh\n[ "$1" = exec ] && echo shimkey\nexit 0\n' > "$sg_shim/docker"
  printf '#!/bin/sh\necho 200\n' > "$sg_shim/curl"
  printf '#!/bin/sh\nshift\nexec "$@"\n' > "$sg_shim/timeout"
  printf '#!/bin/sh\necho called >> "$ETV_SCAN_OUT/semgrep.called"\n[ "${SG_SHIM_RC:-0}" = 0 ] || echo "fake semgrep: registry unreachable" >&2\nexit "${SG_SHIM_RC:-0}"\n' > "$sg_shim/semgrep"
  chmod +x "$sg_shim"/*
  # full_check LABEL SG_SHIM_RC SKIP WANT_RC WANT_CALLED WANT_SUBSTR — SKIP is "unset" or the value to export.
  full_check() {
    local label="$1" sg_shim_rc="$2" skip="$3" want_rc="$4" want_called="$5" want_substr="$6" out rc called
    rm -f "$sg_out/semgrep.called"
    if [ "$skip" = unset ]; then
      out="$(env -u ETV_SCAN_SKIP_SEMGREP SG_SHIM_RC="$sg_shim_rc" PATH="$sg_shim:$PATH" ETV_SCAN_OUT="$sg_out" "$BASH" "${BASH_SOURCE[0]}" 2>&1)" && rc=0 || rc=$?
    else
      out="$(ETV_SCAN_SKIP_SEMGREP="$skip" SG_SHIM_RC="$sg_shim_rc" PATH="$sg_shim:$PATH" ETV_SCAN_OUT="$sg_out" "$BASH" "${BASH_SOURCE[0]}" 2>&1)" && rc=0 || rc=$?
    fi
    if [ -e "$sg_out/semgrep.called" ]; then called=1; else called=0; fi
    if [ "$rc" = "$want_rc" ] && [ "$called" = "$want_called" ] && [[ "$out" == *"$want_substr"* ]]; then
      echo "selftest ok: script, $label -> exit $rc, semgrep called=$called"
    else
      echo "SELFTEST FAIL: script, $label -> rc=$rc called=$called (want rc=$want_rc called=$want_called, output containing '$want_substr')" >&2
      selftest_failed=1
    fi
  }
  full_check "semgrep succeeds" 0 unset 0 1 "==> semgrep SAST cross-check"
  full_check "semgrep succeeds, skip=0" 0 0 0 1 "==> semgrep SAST cross-check"
  full_check "semgrep fails" 2 unset 2 1 "error: semgrep failed (exit 2)"
  full_check "explicit opt-out, semgrep present" 0 1 0 0 "semgrep SAST skipped (ETV_SCAN_SKIP_SEMGREP=1)"
  rm -rf "$sg_dir" "$sg_empty" "$sg_shim" "$sg_out"
  if [ "$selftest_failed" = 0 ]; then
    echo "selftest: all ZAP exit classifications and semgrep-gate cases OK"; exit 0
  else
    echo "selftest: FAILURES ABOVE" >&2; exit 1
  fi
fi

IMAGE="${1:-192.168.1.95:3000/timothy/ersatztv:latest}"
PORT="${2:-8411}"
OUT="${ETV_SCAN_OUT:-/tmp/etv-scan-out}"
TARGET_NAME="etv-secscan-target"
ZAP_IMAGE="ghcr.io/zaproxy/zaproxy:stable"

# Fail fast, before the image pull and the multi-minute ZAP run, if the SAST half cannot run.
require_semgrep || exit $?

command -v docker >/dev/null || { echo "error: docker not found — run this on the docker host (bumblebee)" >&2; exit 2; }

CFG="$(mktemp -d /tmp/etv-secscan-cfg-XXXXXX)"
cleanup() {
  # Reap the ZAP scanner container too — in the trap (not just inline after the scan) so an interrupt
  # (SIGINT/SIGTERM) or a `timeout` kill mid-scan doesn't leave it running.
  docker ps --filter ancestor="$ZAP_IMAGE" -q | xargs -r docker kill >/dev/null 2>&1 || true
  docker rm -f "$TARGET_NAME" >/dev/null 2>&1 || true
  rm -rf "$CFG" 2>/dev/null || true
}
trap cleanup EXIT

echo "==> pulling $IMAGE"
docker pull "$IMAGE" >/dev/null

echo "==> booting throwaway scan target ($TARGET_NAME) on :$PORT with a fresh config volume"
docker rm -f "$TARGET_NAME" >/dev/null 2>&1 || true
docker run -d --name "$TARGET_NAME" -p "$PORT:8409" -v "$CFG":/config "$IMAGE" >/dev/null

echo -n "==> waiting for the app to serve /app/ "
ready=0
for _ in $(seq 1 60); do
  if [ "$(curl -s -o /dev/null -w '%{http_code}' "http://localhost:$PORT/app/" 2>/dev/null)" = "200" ]; then
    ready=1; echo "ready"; break
  fi
  echo -n "."; sleep 3
done
[ "$ready" = 1 ] || { echo "FAILED: app never became ready"; docker logs "$TARGET_NAME" 2>&1 | tail -30; exit 1; }

# The machine key is generated at /config/api.key (0600, root-owned on the host mount) — read it from
# inside the container. ZAP injects it as X-Api-Key on every request via a replacer rule so the scan
# reaches the authenticated (`RequiresAuthentication` + RequireKeyForReads) surface, not just the /app shell.
KEY="$(docker exec "$TARGET_NAME" cat /config/api.key 2>/dev/null)"
[ -n "$KEY" ] || { echo "FAILED: could not read /config/api.key"; exit 1; }
echo "==> got machine key (${#KEY} chars); it will be injected as X-Api-Key on every request"

mkdir -p "$OUT"; chmod 777 "$OUT"

# zap-api-scan.py imports the OpenAPI spec (served as a static file at /openapi/v1.json) so it exercises
# every declared /api/v1 operation — not just what a spider finds — then runs passive + active scanners.
# The reports are written when the scan finishes but BEFORE the wrapper exits; zap-api-scan has been seen
# to hang in post-scan cleanup (draining the passive queue / holding a connection to a streaming endpoint),
# so the whole run is wrapped in `timeout` — a hung wrapper is killed and the already-written report is
# still used. ETV_SCAN_TIMEOUT (default 45m) bounds it.
echo "==> running authenticated OWASP ZAP API scan (this can take several minutes)"
set +e
timeout "${ETV_SCAN_TIMEOUT:-45m}" docker run --rm --network host -v "$OUT":/zap/wrk:rw "$ZAP_IMAGE" \
  zap-api-scan.py -t "http://localhost:$PORT/openapi/v1.json" -f openapi \
  -r zap-api-report.html -w zap-api-report.md -J zap-api-report.json \
  -z "replacer.full_list(0).description=apikey;replacer.full_list(0).enabled=true;replacer.full_list(0).matchtype=REQ_HEADER;replacer.full_list(0).matchstr=X-Api-Key;replacer.full_list(0).regex=false;replacer.full_list(0).replacement=$KEY"
zap_rc=$?
[ "$zap_rc" = 124 ] && echo "NOTE: ZAP wrapper hit the ${ETV_SCAN_TIMEOUT:-45m} timeout (usually a post-scan cleanup hang) — the report written before the hang is still valid; triage it."
classify_zap_exit "$zap_rc"
zap_class_status=$?
set -e
# (The ZAP container + target + temp dir are reaped by cleanup() on the EXIT trap — every path, incl.
# a `timeout` kill or Ctrl-C mid-scan.)
echo "==> ZAP report: $OUT/zap-api-report.html (+ .md/.json)"
echo "==> ZAP result: $ZAP_CLASS_MSG (raw exit $zap_rc)"

if [ "${ETV_SCAN_SKIP_SEMGREP:-0}" = "1" ]; then
  echo "==> semgrep SAST skipped (ETV_SCAN_SKIP_SEMGREP=1)"
else
  echo "==> semgrep SAST cross-check (repo root)"
  REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
  sg_rc=0
  run_semgrep "$REPO_ROOT" || sg_rc=$?
  if [ "$sg_rc" != 0 ]; then
    echo "error: semgrep failed (exit $sg_rc) — the SAST half did not run; this is a gate failure, not a finding" >&2
    [ "$zap_class_status" != 0 ] || zap_class_status=2
  fi
fi

exit "$zap_class_status"