#!/usr/bin/env bash # Preflight: does the PINNED CI toolchain image still exist in the registry? (ersatztv#772) # # WHY THIS EXISTS. `docker-build.yml` pins its five `container:` jobs to an immutable # `ersatztv-ci:`. Between 2026-08-11 and 2026-08-13 that tag was deleted from the Gitea # registry and every one of those jobs — including BOTH required contexts — died after 1-2s with # # Error response from daemon: failed to resolve reference "…/ersatztv-ci:": not found # # buried in each job's log. Nothing said "your toolchain image is gone", so the natural first # reading was "my diff broke the build". This job says it in one line, in a job whose NAME says it. # # "Immutable" was taken to mean "will always exist", and those are different claims. The cause was # an owner-level Gitea package cleanup rule (keep_count 15, remove_days 1, remove_pattern `.*`, and # a keep_pattern no 7-hex sha can match), so a pinned tag is deleted once 15 newer versions of the # package exist. The rule lives in the registry's repo — the durable fix is # timothy/server-management#842 — and THIS script does not fix it. It converts a five-job pull # failure into one actionable message, which is all a consumer of someone else's registry can do. # # WHY IT DOES NOT GATE THE CONTAINER JOBS with `needs:`. Serialising five jobs behind a checkout + # one curl would tax every green run to speed up the rare red one, and the container jobs already # fail fast (1-2s) when the pull fails. This runs in PARALLEL: the diagnosis is present the moment # anyone looks, and the happy path pays nothing. # # UNKNOWN IS NOT A PASS. Warning and exiting 0 on every answer that is not 200 or # 404 makes "curl is missing from this runner", "the # registry moved", and "DNS changed" all indistinguishable from a healthy pin — a job that is green # forever having checked nothing, in a file whose header claims the opposite. Unknown answers are # RETRIED (they are usually transient) and then FAIL. The message stays distinct from the deleted # case: "could not verify" and "IS GONE" send an operator to different places. # # IT READS THE REGISTRY ANONYMOUSLY, AND THAT IS LOAD-BEARING RATHER THAN INCIDENTAL. This job runs # on the `pull_request` route, where the workflow YAML is supplied by the PR head, so it may hold no # stored secret (ersatztv#885, `ci.pr-route-carries-no-stored-credential`). A naive unauthenticated # GET of a manifest is a 401 for every tag, present or deleted — which is why this used to demand a # credential — but that 401 is a Bearer CHALLENGE, and this registry issues an anonymous pull token # for a public package against it. Measured 2026-09-04: the token endpoint answers 200 with no # credential, that token reads the pinned manifest and its config blob (200), and a tag that does # not exist answers 404 rather than 401 — so the deleted-tag diagnosis this whole script exists for # survives the change. What does NOT survive it is `timothy/ersatztv` or its `ersatztv-ci` package # being made private: the token leg then refuses, and this fails loudly with a message that names # that cause rather than reporting could-not-tell. # # Env (all optional; the defaults are the live values): # ETV_CI_REGISTRY registry host:port (default 192.168.1.95:3000) # ETV_CI_IMAGE_REPO package path inside the registry (default timothy/ersatztv-ci) # ETV_CI_WORKFLOW workflow file to read the pin from (default .gitea/workflows/docker-build.yml) # ETV_CI_ATTEMPTS tries per pin before an unknown becomes a failure (default 3) # ETV_CI_RETRY_SECONDS pause between those tries (default 5) set -euo pipefail registry="${ETV_CI_REGISTRY:-192.168.1.95:3000}" image_repo="${ETV_CI_IMAGE_REPO:-timothy/ersatztv-ci}" workflow="${ETV_CI_WORKFLOW:-.gitea/workflows/docker-build.yml}" fail() { printf '::error::ci-toolchain-image-resolves: %s\n' "$*" >&2; exit 1; } [ -f "$workflow" ] || fail "cannot read $workflow to find the toolchain pin" # The same expression `pr-checks.yml::ci-image-pin` greps with, so the two cannot disagree about # what "the pin" is. Note it is written so THIS line cannot match itself: the character after the # colon here is `[`, which is not in [0-9a-f]. pins=$(grep -oE 'ersatztv-ci:[0-9a-f]+' "$workflow" | cut -d: -f2 | sort -u || true) [ -n "$pins" ] || fail "no ersatztv-ci pin found in $workflow — if the grep pattern stopped matching, fix it here and in pr-checks.yml::ci-image-pin together" accept='application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json' attempts="${ETV_CI_ATTEMPTS:-3}" retry_seconds="${ETV_CI_RETRY_SECONDS:-5}" rc=0 # The anonymous pull token, acquired at most once and reused for every pin. Empty until the # registry challenges; `token_leg_done` separates "not tried yet" from "tried and got nothing", so a # registry that is genuinely refusing anonymous reads is not re-asked once per pin. token="" token_leg_done=0 headers=$(mktemp) trap 'rm -f "$headers"' EXIT # One directive out of a `Www-Authenticate: Bearer realm="…",service="…"` challenge. The realm is # read from the challenge rather than assumed, so a registry that moves its token endpoint is # followed instead of guessed at; `service` is optional (this registry issues a token without it, # measured 2026-09-04) and is passed through when the challenge names one. challenge_field() { printf '%s' "$1" | sed -n "s/.*[ ,]$2=\"\([^\"]*\)\".*/\1/p" | head -1 } # The token leg. Returns non-zero on every shape that leaves us without a bearer — no challenge, no # realm in it, a token endpoint that will not answer, or an answer carrying no token. Each of those # is "could not establish anonymous access", which the caller turns into a REFUSAL rather than a # could-not-tell: an empty token would otherwise fall through to a second 401 and read as an # ordinary auth failure with no cause named. acquire_token() { local challenge realm service url body token_leg_done=1 challenge=$(tr -d '\r' < "$headers" | sed -n 's/^[Ww][Ww][Ww]-[Aa]uthenticate:[[:space:]]*//p' | head -1) [ -n "$challenge" ] || return 1 realm=$(challenge_field "$challenge" realm) [ -n "$realm" ] || return 1 service=$(challenge_field "$challenge" service) url="$realm?scope=repository:$image_repo:pull" # Spelled as a full `if` rather than `[ … ] && …`: as a bare statement the latter returns the # test's exit status, which is 1 whenever `service` is absent — a legal challenge shape. if [ -n "$service" ]; then url="$url&service=$service"; fi body=$(curl -s "$url") || return 1 token=$(printf '%s' "$body" | sed -n 's/.*"token"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -1) [ -n "$token" ] || return 1 return 0 } # One GET, recording the HTTP code and whether the body is a manifest. The body is fetched rather # than a HEAD sent, because # HTTP 200 alone does not mean "the manifest is there": a proxy, a captive login page or an error # document all answer 200 with a body that is not a manifest, and a check that reads only the status # line reports those as "resolves". A manifest always carries `schemaVersion`, so the body is matched # for it — with a shell `case`, so nothing depends on jq being installed and no pipeline can invert # the result on a large body. # # It ASSIGNS `code`/`is_manifest` rather than printing them, and so does `probe` — because the token # and the "already tried the token leg" flag must survive from one pin to the next, and a # `$(probe …)` command substitution runs in a SUBSHELL whose assignments are discarded. Reading the # answer through a subshell would silently re-run the whole two-leg exchange for every pin. request() { local url="$1" resp body if [ -n "$token" ]; then resp=$(curl -s -w '\n%{http_code}' -D "$headers" -H "Authorization: Bearer $token" -H "Accept: $accept" "$url") || resp="" else resp=$(curl -s -w '\n%{http_code}' -D "$headers" -H "Accept: $accept" "$url") || resp="" fi if [ -z "$resp" ]; then code="000" is_manifest="no" return 0 fi code=${resp##*$'\n'} body=${resp%$'\n'*} case "$body" in *'"schemaVersion"'*) is_manifest="yes" ;; *) is_manifest="no" ;; esac } # The two legs of an anonymous registry read: the plain GET, and — only if it is challenged and we # have not already tried — the token exchange followed by ONE retry carrying the bearer. A 401 that # survives the retry is left as a 401 and refused by the caller; it is never rounded off to # could-not-tell, because the cause (this package is no longer readable without a credential) sends # an operator somewhere quite different from a flaky registry. probe() { request "$1" if [ "$code" = "401" ] && [ "$token_leg_done" -eq 0 ]; then if acquire_token; then request "$1" fi fi } code="" is_manifest="no" for pin in $pins; do url="http://$registry/v2/$image_repo/manifests/$pin" attempt=1 while : ; do probe "$url" case "$code" in 200|404|401|403) break ;; esac # Only the unknown answers are retried: 200/404 are answers, and an auth failure will not cure # itself. A transient registry is the common case for the rest, and absorbing it here is what # lets the unknown be a FAILURE at the end rather than a warning nobody reads. [ "$attempt" -lt "$attempts" ] || break attempt=$((attempt + 1)) sleep "$retry_seconds" done case "$code" in 200) if [ "$is_manifest" = "yes" ]; then printf 'ci-toolchain-image-resolves: %s/%s:%s resolves (HTTP 200, manifest present)\n' "$registry" "$image_repo" "$pin" else printf '::error::ci-toolchain-image-resolves: %s/%s:%s answered HTTP 200 with a body that is not a manifest (no schemaVersion). Something is answering for the registry — a proxy, a login page, or an error document. The pin was NOT verified.\n' \ "$registry" "$image_repo" "$pin" >&2 rc=1 fi ;; 404) # The one unambiguous answer, and the outage this exists for. printf '::error::ci-toolchain-image-resolves: the pinned CI toolchain image %s/%s:%s IS GONE from the registry (HTTP 404). Every container: job in docker-build.yml will fail at image pull, including both required contexts, and NO diff caused it. Recovery does not need CI: rebuild that exact tag from the commit it names and push it — see docs/ci-cd.md -> "CI toolchain image" -> "When the pinned tag disappears". Root cause + the durable fix: timothy/server-management#842.\n' \ "$registry" "$image_repo" "$pin" >&2 rc=1 ;; 401|403) # `fail` rather than `rc=1`: unlike a 404, this says nothing about the pin, and it will say # the same thing about every remaining one. Abandoning the loop keeps the log to one cause. # # THREE ways to get here, and they send an operator to three different places, so they are # worded apart — the same reason `404` and `could NOT VERIFY` are. Each message states only # what actually ran, because a message naming a step that did not happen is evidence for a # diagnosis nobody performed: # # * a refusal that survived a GOOD token is an answer about this PACKAGE, and since # ersatztv#885 it is not a preflight-only problem: every `container:` job pulls the same # image with no credential, so they fail at image pull too, including both required # contexts; # * a challenge that yielded no usable token is an infrastructure answer about the TOKEN # ENDPOINT; # * a FIRST-READ 403 never reached the token leg at all — `probe` enters it on a 401 only — # so nothing was ever asked for. This is an answer about ACCESS to the registry. Note the # boundary: a 401 carrying no `Www-Authenticate` DOES enter the token leg and abandon it, # so it lands in the row above, not this one. The branch order below is `token` first # precisely so this case cannot borrow either of the other two mechanisms. # # A failed token leg is NOT retried, deliberately: `token_leg_done` is set before the attempt # so a registry genuinely refusing anonymous reads is asked once rather than once per pin. The # cost is that a transient token-endpoint outage fails the job on its first try; the message # below says so rather than blaming the package. if [ -n "$token" ]; then fail "the registry refused an ANONYMOUS read (HTTP $code) of $registry/$image_repo:$pin even after a Bearer token was obtained, so the pin could not be checked. Every container: job pulls this image without a credential too, so they will fail at image pull. Check that timothy/ersatztv and its ersatztv-ci package are still PUBLIC — do not read this as a pass." fi if [ "$token_leg_done" -eq 1 ]; then fail "could NOT OBTAIN an anonymous pull token for $registry/$image_repo:$pin — the registry answered HTTP $code and the token leg produced none: either there was no Www-Authenticate challenge, or it named no realm, or the token endpoint did not answer with a token. The pin was NOT checked. Look at the registry's token endpoint, not at the pin." fi fail "the registry refused an ANONYMOUS read (HTTP $code) of $registry/$image_repo:$pin WITHOUT issuing a Bearer challenge, so no token was ever requested and the pin could not be checked. A challenge is what tells a client where a token can be had; an outright refusal is an answer about ACCESS to the registry, not about the pin. Every container: job pulls this image without a credential too, so they will fail at image pull. Check that timothy/ersatztv and its ersatztv-ci package are still PUBLIC, and that nothing (a proxy, an ACL) is answering for the registry — do not read this as a pass." ;; *) # NOT gone, and NOT a pass either. Deliberately worded apart from the 404 message: this sends # an operator to the registry's health, not to a rebuild of a tag that may be sitting there. printf '::error::ci-toolchain-image-resolves: could NOT VERIFY %s/%s:%s after %s attempt(s) (last answer: HTTP %s). This is not evidence the image is gone — it is evidence the check could not run, which fails rather than passing so the preflight cannot quietly become a no-op.\n' \ "$registry" "$image_repo" "$pin" "$attempts" "$code" >&2 rc=1 ;; esac done exit "$rc"