ersatztv/scripts/ci-toolchain-image-resolves.sh

#!/usr/bin/env bash
# Preflight: does the PINNED CI toolchain image still exist in the registry? (ersatztv#772)
#
# WHY THIS EXISTS. `docker-build.yml` pins its five `container:` jobs to an immutable
# `ersatztv-ci:<sha>`. Between 2026-08-11 and 2026-08-13 that tag was deleted from the Gitea
# registry and every one of those jobs — including BOTH required contexts — died after 1-2s with
#
#   Error response from daemon: failed to resolve reference "…/ersatztv-ci:<the pinned sha>": not found
#
# buried in each job's log. Nothing said "your toolchain image is gone", so the natural first
# reading was "my diff broke the build". This job says it in one line, in a job whose NAME says it.
#
# "Immutable" was taken to mean "will always exist", and those are different claims. The cause was
# an owner-level Gitea package cleanup rule (keep_count 15, remove_days 1, remove_pattern `.*`, and
# a keep_pattern no 7-hex sha can match), so a pinned tag is deleted once 15 newer versions of the
# package exist. The rule lives in the registry's repo — the durable fix is
# timothy/server-management#842 — and THIS script does not fix it. It converts a five-job pull
# failure into one actionable message, which is all a consumer of someone else's registry can do.
#
# WHY IT DOES NOT GATE THE CONTAINER JOBS with `needs:`. Serialising five jobs behind a checkout +
# one curl would tax every green run to speed up the rare red one, and the container jobs already
# fail fast (1-2s) when the pull fails. This runs in PARALLEL: the diagnosis is present the moment
# anyone looks, and the happy path pays nothing.
#
# UNKNOWN IS NOT A PASS. Warning and exiting 0 on every answer that is not 200 or
# 404 makes "curl is missing from this runner", "the
# registry moved", and "DNS changed" all indistinguishable from a healthy pin — a job that is green
# forever having checked nothing, in a file whose header claims the opposite. Unknown answers are
# RETRIED (they are usually transient) and then FAIL. The message stays distinct from the deleted
# case: "could not verify" and "IS GONE" send an operator to different places.
#
# IT READS THE REGISTRY ANONYMOUSLY, AND THAT IS LOAD-BEARING RATHER THAN INCIDENTAL. This job runs
# on the `pull_request` route, where the workflow YAML is supplied by the PR head, so it may hold no
# stored secret (ersatztv#885, `ci.pr-route-carries-no-stored-credential`). A naive unauthenticated
# GET of a manifest is a 401 for every tag, present or deleted — which is why this used to demand a
# credential — but that 401 is a Bearer CHALLENGE, and this registry issues an anonymous pull token
# for a public package against it. Measured 2026-09-04: the token endpoint answers 200 with no
# credential, that token reads the pinned manifest and its config blob (200), and a tag that does
# not exist answers 404 rather than 401 — so the deleted-tag diagnosis this whole script exists for
# survives the change. What does NOT survive it is `timothy/ersatztv` or its `ersatztv-ci` package
# being made private: the token leg then refuses, and this fails loudly with a message that names
# that cause rather than reporting could-not-tell.
#
# Env (all optional; the defaults are the live values):
#   ETV_CI_REGISTRY        registry host:port                 (default 192.168.1.95:3000)
#   ETV_CI_IMAGE_REPO      package path inside the registry   (default timothy/ersatztv-ci)
#   ETV_CI_WORKFLOW        workflow file to read the pin from (default .gitea/workflows/docker-build.yml)
#   ETV_CI_ATTEMPTS        tries per pin before an unknown becomes a failure (default 3)
#   ETV_CI_RETRY_SECONDS   pause between those tries           (default 5)
set -euo pipefail

registry="${ETV_CI_REGISTRY:-192.168.1.95:3000}"
image_repo="${ETV_CI_IMAGE_REPO:-timothy/ersatztv-ci}"
workflow="${ETV_CI_WORKFLOW:-.gitea/workflows/docker-build.yml}"

fail() { printf '::error::ci-toolchain-image-resolves: %s\n' "$*" >&2; exit 1; }

[ -f "$workflow" ] || fail "cannot read $workflow to find the toolchain pin"

# The same expression `pr-checks.yml::ci-image-pin` greps with, so the two cannot disagree about
# what "the pin" is. Note it is written so THIS line cannot match itself: the character after the
# colon here is `[`, which is not in [0-9a-f].
pins=$(grep -oE 'ersatztv-ci:[0-9a-f]+' "$workflow" | cut -d: -f2 | sort -u || true)
[ -n "$pins" ] || fail "no ersatztv-ci pin found in $workflow — if the grep pattern stopped matching, fix it here and in pr-checks.yml::ci-image-pin together"

accept='application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json'
attempts="${ETV_CI_ATTEMPTS:-3}"
retry_seconds="${ETV_CI_RETRY_SECONDS:-5}"
rc=0

# The anonymous pull token, acquired at most once and reused for every pin. Empty until the
# registry challenges; `token_leg_done` separates "not tried yet" from "tried and got nothing", so a
# registry that is genuinely refusing anonymous reads is not re-asked once per pin.
token=""
token_leg_done=0
headers=$(mktemp)
trap 'rm -f "$headers"' EXIT

# One directive out of a `Www-Authenticate: Bearer realm="…",service="…"` challenge. The realm is
# read from the challenge rather than assumed, so a registry that moves its token endpoint is
# followed instead of guessed at; `service` is optional (this registry issues a token without it,
# measured 2026-09-04) and is passed through when the challenge names one.
challenge_field() {
  printf '%s' "$1" | sed -n "s/.*[ ,]$2=\"\([^\"]*\)\".*/\1/p" | head -1
}

# The token leg. Returns non-zero on every shape that leaves us without a bearer — no challenge, no
# realm in it, a token endpoint that will not answer, or an answer carrying no token. Each of those
# is "could not establish anonymous access", which the caller turns into a REFUSAL rather than a
# could-not-tell: an empty token would otherwise fall through to a second 401 and read as an
# ordinary auth failure with no cause named.
acquire_token() {
  local challenge realm service url body
  token_leg_done=1
  challenge=$(tr -d '\r' < "$headers" | sed -n 's/^[Ww][Ww][Ww]-[Aa]uthenticate:[[:space:]]*//p' | head -1)
  [ -n "$challenge" ] || return 1
  realm=$(challenge_field "$challenge" realm)
  [ -n "$realm" ] || return 1
  service=$(challenge_field "$challenge" service)
  url="$realm?scope=repository:$image_repo:pull"
  # Spelled as a full `if` rather than `[ … ] && …`: as a bare statement the latter returns the
  # test's exit status, which is 1 whenever `service` is absent — a legal challenge shape.
  if [ -n "$service" ]; then url="$url&service=$service"; fi
  body=$(curl -s "$url") || return 1
  token=$(printf '%s' "$body" | sed -n 's/.*"token"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -1)
  [ -n "$token" ] || return 1
  return 0
}

# One GET, recording the HTTP code and whether the body is a manifest. The body is fetched rather
# than a HEAD sent, because
# HTTP 200 alone does not mean "the manifest is there": a proxy, a captive login page or an error
# document all answer 200 with a body that is not a manifest, and a check that reads only the status
# line reports those as "resolves". A manifest always carries `schemaVersion`, so the body is matched
# for it — with a shell `case`, so nothing depends on jq being installed and no pipeline can invert
# the result on a large body.
#
# It ASSIGNS `code`/`is_manifest` rather than printing them, and so does `probe` — because the token
# and the "already tried the token leg" flag must survive from one pin to the next, and a
# `$(probe …)` command substitution runs in a SUBSHELL whose assignments are discarded. Reading the
# answer through a subshell would silently re-run the whole two-leg exchange for every pin.
request() {
  local url="$1" resp body
  if [ -n "$token" ]; then
    resp=$(curl -s -w '\n%{http_code}' -D "$headers" -H "Authorization: Bearer $token" -H "Accept: $accept" "$url") || resp=""
  else
    resp=$(curl -s -w '\n%{http_code}' -D "$headers" -H "Accept: $accept" "$url") || resp=""
  fi
  if [ -z "$resp" ]; then
    code="000"
    is_manifest="no"
    return 0
  fi
  code=${resp##*$'\n'}
  body=${resp%$'\n'*}
  case "$body" in
    *'"schemaVersion"'*) is_manifest="yes" ;;
    *) is_manifest="no" ;;
  esac
}

# The two legs of an anonymous registry read: the plain GET, and — only if it is challenged and we
# have not already tried — the token exchange followed by ONE retry carrying the bearer. A 401 that
# survives the retry is left as a 401 and refused by the caller; it is never rounded off to
# could-not-tell, because the cause (this package is no longer readable without a credential) sends
# an operator somewhere quite different from a flaky registry.
probe() {
  request "$1"
  if [ "$code" = "401" ] && [ "$token_leg_done" -eq 0 ]; then
    if acquire_token; then
      request "$1"
    fi
  fi
}

code=""
is_manifest="no"
for pin in $pins; do
  url="http://$registry/v2/$image_repo/manifests/$pin"
  attempt=1
  while : ; do
    probe "$url"
    case "$code" in
      200|404|401|403) break ;;
    esac
    # Only the unknown answers are retried: 200/404 are answers, and an auth failure will not cure
    # itself. A transient registry is the common case for the rest, and absorbing it here is what
    # lets the unknown be a FAILURE at the end rather than a warning nobody reads.
    [ "$attempt" -lt "$attempts" ] || break
    attempt=$((attempt + 1))
    sleep "$retry_seconds"
  done

  case "$code" in
    200)
      if [ "$is_manifest" = "yes" ]; then
        printf 'ci-toolchain-image-resolves: %s/%s:%s resolves (HTTP 200, manifest present)\n' "$registry" "$image_repo" "$pin"
      else
        printf '::error::ci-toolchain-image-resolves: %s/%s:%s answered HTTP 200 with a body that is not a manifest (no schemaVersion). Something is answering for the registry — a proxy, a login page, or an error document. The pin was NOT verified.\n' \
          "$registry" "$image_repo" "$pin" >&2
        rc=1
      fi
      ;;
    404)
      # The one unambiguous answer, and the outage this exists for.
      printf '::error::ci-toolchain-image-resolves: the pinned CI toolchain image %s/%s:%s IS GONE from the registry (HTTP 404). Every container: job in docker-build.yml will fail at image pull, including both required contexts, and NO diff caused it. Recovery does not need CI: rebuild that exact tag from the commit it names and push it — see docs/ci-cd.md -> "CI toolchain image" -> "When the pinned tag disappears". Root cause + the durable fix: timothy/server-management#842.\n' \
        "$registry" "$image_repo" "$pin" >&2
      rc=1
      ;;
    401|403)
      # `fail` rather than `rc=1`: unlike a 404, this says nothing about the pin, and it will say
      # the same thing about every remaining one. Abandoning the loop keeps the log to one cause.
      #
      # THREE ways to get here, and they send an operator to three different places, so they are
      # worded apart — the same reason `404` and `could NOT VERIFY` are. Each message states only
      # what actually ran, because a message naming a step that did not happen is evidence for a
      # diagnosis nobody performed:
      #
      #   * a refusal that survived a GOOD token is an answer about this PACKAGE, and since
      #     ersatztv#885 it is not a preflight-only problem: every `container:` job pulls the same
      #     image with no credential, so they fail at image pull too, including both required
      #     contexts;
      #   * a challenge that yielded no usable token is an infrastructure answer about the TOKEN
      #     ENDPOINT;
      #   * a refusal carrying NO Bearer challenge at all never reached the token leg — `probe`
      #     enters it on a 401 only, so a first-read 403 (or a 401 with no `Www-Authenticate`, which
      #     the token leg then abandons) leaves `token` empty having asked for nothing. This is an
      #     answer about ACCESS to the registry, and the branch order below is `token` first
      #     precisely so this case cannot borrow either of the other two mechanisms.
      #
      # A failed token leg is NOT retried, deliberately: `token_leg_done` is set before the attempt
      # so a registry genuinely refusing anonymous reads is asked once rather than once per pin. The
      # cost is that a transient token-endpoint outage fails the job on its first try; the message
      # below says so rather than blaming the package.
      if [ -n "$token" ]; then
        fail "the registry refused an ANONYMOUS read (HTTP $code) of $registry/$image_repo:$pin even after a Bearer token was obtained, so the pin could not be checked. Every container: job pulls this image without a credential too, so they will fail at image pull. Check that timothy/ersatztv and its ersatztv-ci package are still PUBLIC — do not read this as a pass."
      fi
      if [ "$token_leg_done" -eq 1 ]; then
        fail "could NOT OBTAIN an anonymous pull token for $registry/$image_repo:$pin — the registry answered HTTP $code and the token leg produced none: either there was no Www-Authenticate challenge, or it named no realm, or the token endpoint did not answer with a token. The pin was NOT checked. Look at the registry's token endpoint, not at the pin."
      fi
      fail "the registry refused an ANONYMOUS read (HTTP $code) of $registry/$image_repo:$pin WITHOUT issuing a Bearer challenge, so no token was ever requested and the pin could not be checked. A challenge is what tells a client where a token can be had; an outright refusal is an answer about ACCESS to the registry, not about the pin. Every container: job pulls this image without a credential too, so they will fail at image pull. Check that timothy/ersatztv and its ersatztv-ci package are still PUBLIC, and that nothing (a proxy, an ACL) is answering for the registry — do not read this as a pass."
      ;;
    *)
      # NOT gone, and NOT a pass either. Deliberately worded apart from the 404 message: this sends
      # an operator to the registry's health, not to a rebuild of a tag that may be sitting there.
      printf '::error::ci-toolchain-image-resolves: could NOT VERIFY %s/%s:%s after %s attempt(s) (last answer: HTTP %s). This is not evidence the image is gone — it is evidence the check could not run, which fails rather than passing so the preflight cannot quietly become a no-op.\n' \
        "$registry" "$image_repo" "$pin" "$attempts" "$code" >&2
      rc=1
      ;;
  esac
done

exit "$rc"