ersatztv/scripts/ci-toolchain-image-resolves.sh

#!/usr/bin/env bash
# Preflight: does the PINNED CI toolchain image still exist in the registry? (ersatztv#772)
#
# WHY THIS EXISTS. `docker-build.yml` pins its five `container:` jobs to an immutable
# `ersatztv-ci:<sha>`. Between 2026-08-11 and 2026-08-13 that tag was deleted from the Gitea
# registry and every one of those jobs — including BOTH required contexts — died after 1-2s with
#
#   Error response from daemon: failed to resolve reference "…/ersatztv-ci:<the pinned sha>": not found
#
# buried in each job's log. Nothing said "your toolchain image is gone", so the natural first
# reading was "my diff broke the build". This job says it in one line, in a job whose NAME says it.
#
# "Immutable" was taken to mean "will always exist", and those are different claims. The cause was
# an owner-level Gitea package cleanup rule (keep_count 15, remove_days 1, remove_pattern `.*`, and
# a keep_pattern no 7-hex sha can match), so a pinned tag is deleted once 15 newer versions of the
# package exist. The rule lives in the registry's repo — the durable fix is
# timothy/server-management#842 — and THIS script does not fix it. It converts a five-job pull
# failure into one actionable message, which is all a consumer of someone else's registry can do.
#
# WHY IT DOES NOT GATE THE CONTAINER JOBS with `needs:`. Serialising five jobs behind a checkout +
# one curl would tax every green run to speed up the rare red one, and the container jobs already
# fail fast (1-2s) when the pull fails. This runs in PARALLEL: the diagnosis is present the moment
# anyone looks, and the happy path pays nothing.
#
# UNKNOWN IS NOT A PASS. Warning and exiting 0 on every answer that is not 200 or
# 404 makes "curl is missing from this runner", "the
# registry moved", and "DNS changed" all indistinguishable from a healthy pin — a job that is green
# forever having checked nothing, in a file whose header claims the opposite. Unknown answers are
# RETRIED (they are usually transient) and then FAIL. The message stays distinct from the deleted
# case: "could not verify" and "IS GONE" send an operator to different places.
#
# Env (all optional except the credential; the defaults are the live values):
#   ETV_CI_REGISTRY        registry host:port                 (default 192.168.1.95:3000)
#   ETV_CI_IMAGE_REPO      package path inside the registry   (default timothy/ersatztv-ci)
#   ETV_CI_WORKFLOW        workflow file to read the pin from (default .gitea/workflows/docker-build.yml)
#   ETV_CI_ATTEMPTS        tries per pin before an unknown becomes a failure (default 3)
#   ETV_CI_RETRY_SECONDS   pause between those tries           (default 5)
#   ETV_REGISTRY_AUTH      user:pass — REQUIRED; the registry rejects anonymous reads with 401
set -euo pipefail

registry="${ETV_CI_REGISTRY:-192.168.1.95:3000}"
image_repo="${ETV_CI_IMAGE_REPO:-timothy/ersatztv-ci}"
workflow="${ETV_CI_WORKFLOW:-.gitea/workflows/docker-build.yml}"

fail() { printf '::error::ci-toolchain-image-resolves: %s\n' "$*" >&2; exit 1; }

[ -f "$workflow" ] || fail "cannot read $workflow to find the toolchain pin"

# The same expression `pr-checks.yml::ci-image-pin` greps with, so the two cannot disagree about
# what "the pin" is. Note it is written so THIS line cannot match itself: the character after the
# colon here is `[`, which is not in [0-9a-f].
pins=$(grep -oE 'ersatztv-ci:[0-9a-f]+' "$workflow" | cut -d: -f2 | sort -u || true)
[ -n "$pins" ] || fail "no ersatztv-ci pin found in $workflow — if the grep pattern stopped matching, fix it here and in pr-checks.yml::ci-image-pin together"

# No credentials is NOT a pass. An unauthenticated read of this registry is a 401 for every tag,
# present or deleted, so a run without them would report "cannot tell" for a live pin and for a
# deleted one alike — the shape where a guard reports green having checked nothing.
#
# The EMPTY-halves check is the one that matters in CI and is easy to miss: an absent secret does
# not arrive here as an unset variable. `ETV_REGISTRY_AUTH: ${{ secrets.REGISTRY_USER }}:${{ ... }}`
# interpolates a missing secret to the empty string, so the job passes the non-empty string ":".
# Testing only the unset case would leave the production shape uncovered.
auth="${ETV_REGISTRY_AUTH:-}"
[ -n "$auth" ] || fail "ETV_REGISTRY_AUTH (user:pass) is unset, so the registry cannot be queried — this check refuses to report a pass it did not establish"
case "$auth" in
  *:*) ;;
  *) fail "ETV_REGISTRY_AUTH must be user:pass, got a value with no ':' — the registry cannot be queried and this check refuses to report a pass it did not establish" ;;
esac
[ -n "${auth%%:*}" ] && [ -n "${auth#*:}" ] \
  || fail "ETV_REGISTRY_AUTH has an empty half (user or password) — this is what an ABSENT REGISTRY_USER/REGISTRY_PASSWORD secret interpolates to, not a credential. Fix the secrets rather than reading an unauthenticated 401 as could-not-tell."

accept='application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json'
attempts="${ETV_CI_ATTEMPTS:-3}"
retry_seconds="${ETV_CI_RETRY_SECONDS:-5}"
rc=0

# One GET, returning "<code> <is-a-manifest>". The body is fetched rather than a HEAD sent, because
# HTTP 200 alone does not mean "the manifest is there": a proxy, a captive login page or an error
# document all answer 200 with a body that is not a manifest, and a check that reads only the status
# line reports those as "resolves". A manifest always carries `schemaVersion`, so the body is matched
# for it — with a shell `case`, so nothing depends on jq being installed and no pipeline can invert
# the result on a large body.
probe() {
  local url="$1" resp code body
  # `-u` puts the credential in argv, visible to `ps` for the length of the call — and this job has
  # no `container:`, so that is the shared host. Kept because it is the shape every other curl caller
  # in scripts/ already uses (`ci-detect-already-validated.sh`, `pr-changed-files.sh`,
  # `select-queue.sh`, `issue-qualification-audit.sh`): fixing one site would leave the class intact
  # and the codebase inconsistent. The class is tracked in ersatztv#821.
  resp=$(curl -s -w '\n%{http_code}' -u "$auth" -H "Accept: $accept" "$url") || resp=""
  [ -n "$resp" ] || { printf '000 no\n'; return 0; }
  code=${resp##*$'\n'}
  body=${resp%$'\n'*}
  case "$body" in
    *'"schemaVersion"'*) printf '%s yes\n' "$code" ;;
    *) printf '%s no\n' "$code" ;;
  esac
}

for pin in $pins; do
  url="http://$registry/v2/$image_repo/manifests/$pin"
  attempt=1
  while : ; do
    read -r code is_manifest <<EOF
$(probe "$url")
EOF
    case "$code" in
      200|404|401|403) break ;;
    esac
    # Only the unknown answers are retried: 200/404 are answers, and an auth failure will not cure
    # itself. A transient registry is the common case for the rest, and absorbing it here is what
    # lets the unknown be a FAILURE at the end rather than a warning nobody reads.
    [ "$attempt" -lt "$attempts" ] || break
    attempt=$((attempt + 1))
    sleep "$retry_seconds"
  done

  case "$code" in
    200)
      if [ "$is_manifest" = "yes" ]; then
        printf 'ci-toolchain-image-resolves: %s/%s:%s resolves (HTTP 200, manifest present)\n' "$registry" "$image_repo" "$pin"
      else
        printf '::error::ci-toolchain-image-resolves: %s/%s:%s answered HTTP 200 with a body that is not a manifest (no schemaVersion). Something is answering for the registry — a proxy, a login page, or an error document. The pin was NOT verified.\n' \
          "$registry" "$image_repo" "$pin" >&2
        rc=1
      fi
      ;;
    404)
      # The one unambiguous answer, and the outage this exists for.
      printf '::error::ci-toolchain-image-resolves: the pinned CI toolchain image %s/%s:%s IS GONE from the registry (HTTP 404). Every container: job in docker-build.yml will fail at image pull, including both required contexts, and NO diff caused it. Recovery does not need CI: rebuild that exact tag from the commit it names and push it — see docs/ci-cd.md -> "CI toolchain image" -> "When the pinned tag disappears". Root cause + the durable fix: timothy/server-management#842.\n' \
        "$registry" "$image_repo" "$pin" >&2
      rc=1
      ;;
    401|403)
      # `fail` rather than `rc=1`: unlike a 404, this says nothing about the pin, and it will say
      # the same thing about every remaining one. Abandoning the loop keeps the log to one cause.
      fail "the registry rejected these credentials (HTTP $code) for $registry/$image_repo:$pin, so the pin could not be checked. Fix REGISTRY_USER/REGISTRY_PASSWORD rather than reading this as a pass."
      ;;
    *)
      # NOT gone, and NOT a pass either. Deliberately worded apart from the 404 message: this sends
      # an operator to the registry's health, not to a rebuild of a tag that may be sitting there.
      printf '::error::ci-toolchain-image-resolves: could NOT VERIFY %s/%s:%s after %s attempt(s) (last answer: HTTP %s). This is not evidence the image is gone — it is evidence the check could not run, which fails rather than passing so the preflight cannot quietly become a no-op.\n' \
        "$registry" "$image_repo" "$pin" "$attempts" "$code" >&2
      rc=1
      ;;
  esac
done

exit "$rc"