#!/usr/bin/env bash # Preflight: does the PINNED CI toolchain image still exist in the registry? (ersatztv#772) # # WHY THIS EXISTS. `docker-build.yml` pins its five `container:` jobs to an immutable # `ersatztv-ci:`. Between 2026-08-11 and 2026-08-13 that tag was deleted from the Gitea # registry and every one of those jobs — including BOTH required contexts — died after 1-2s with # # Error response from daemon: failed to resolve reference "…/ersatztv-ci:": not found # # buried in each job's log. Nothing said "your toolchain image is gone", so the natural first # reading was "my diff broke the build". This job says it in one line, in a job whose NAME says it. # # "Immutable" was taken to mean "will always exist", and those are different claims. The cause was # an owner-level Gitea package cleanup rule (keep_count 15, remove_days 1, remove_pattern `.*`, and # a keep_pattern no 7-hex sha can match), so a pinned tag is deleted once 15 newer versions of the # package exist. The rule lives in the registry's repo — the durable fix is # timothy/server-management#842 — and THIS script does not fix it. It converts a five-job pull # failure into one actionable message, which is all a consumer of someone else's registry can do. # # WHY IT DOES NOT GATE THE CONTAINER JOBS with `needs:`. Serialising five jobs behind a checkout + # one curl would tax every green run to speed up the rare red one, and the container jobs already # fail fast (1-2s) when the pull fails. This runs in PARALLEL: the diagnosis is present the moment # anyone looks, and the happy path pays nothing. # # UNKNOWN IS NOT A PASS. Warning and exiting 0 on every answer that is not 200 or # 404 makes "curl is missing from this runner", "the # registry moved", and "DNS changed" all indistinguishable from a healthy pin — a job that is green # forever having checked nothing, in a file whose header claims the opposite. Unknown answers are # RETRIED (they are usually transient) and then FAIL. The message stays distinct from the deleted # case: "could not verify" and "IS GONE" send an operator to different places. # # Env (all optional except the credential; the defaults are the live values): # ETV_CI_REGISTRY registry host:port (default 192.168.1.95:3000) # ETV_CI_IMAGE_REPO package path inside the registry (default timothy/ersatztv-ci) # ETV_CI_WORKFLOW workflow file to read the pin from (default .gitea/workflows/docker-build.yml) # ETV_CI_ATTEMPTS tries per pin before an unknown becomes a failure (default 3) # ETV_CI_RETRY_SECONDS pause between those tries (default 5) # ETV_REGISTRY_AUTH user:pass — REQUIRED; the registry rejects anonymous reads with 401 set -euo pipefail registry="${ETV_CI_REGISTRY:-192.168.1.95:3000}" image_repo="${ETV_CI_IMAGE_REPO:-timothy/ersatztv-ci}" workflow="${ETV_CI_WORKFLOW:-.gitea/workflows/docker-build.yml}" fail() { printf '::error::ci-toolchain-image-resolves: %s\n' "$*" >&2; exit 1; } [ -f "$workflow" ] || fail "cannot read $workflow to find the toolchain pin" # The same expression `pr-checks.yml::ci-image-pin` greps with, so the two cannot disagree about # what "the pin" is. Note it is written so THIS line cannot match itself: the character after the # colon here is `[`, which is not in [0-9a-f]. pins=$(grep -oE 'ersatztv-ci:[0-9a-f]+' "$workflow" | cut -d: -f2 | sort -u || true) [ -n "$pins" ] || fail "no ersatztv-ci pin found in $workflow — if the grep pattern stopped matching, fix it here and in pr-checks.yml::ci-image-pin together" # No credentials is NOT a pass. An unauthenticated read of this registry is a 401 for every tag, # present or deleted, so a run without them would report "cannot tell" for a live pin and for a # deleted one alike — the shape where a guard reports green having checked nothing. # # The EMPTY-halves check is the one that matters in CI and is easy to miss: an absent secret does # not arrive here as an unset variable. `ETV_REGISTRY_AUTH: ${{ secrets.REGISTRY_USER }}:${{ ... }}` # interpolates a missing secret to the empty string, so the job passes the non-empty string ":". # Testing only the unset case would leave the production shape uncovered. auth="${ETV_REGISTRY_AUTH:-}" [ -n "$auth" ] || fail "ETV_REGISTRY_AUTH (user:pass) is unset, so the registry cannot be queried — this check refuses to report a pass it did not establish" case "$auth" in *:*) ;; *) fail "ETV_REGISTRY_AUTH must be user:pass, got a value with no ':' — the registry cannot be queried and this check refuses to report a pass it did not establish" ;; esac [ -n "${auth%%:*}" ] && [ -n "${auth#*:}" ] \ || fail "ETV_REGISTRY_AUTH has an empty half (user or password) — this is what an ABSENT REGISTRY_USER/REGISTRY_PASSWORD secret interpolates to, not a credential. Fix the secrets rather than reading an unauthenticated 401 as could-not-tell." accept='application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json' attempts="${ETV_CI_ATTEMPTS:-3}" retry_seconds="${ETV_CI_RETRY_SECONDS:-5}" rc=0 # One GET, returning " ". The body is fetched rather than a HEAD sent, because # HTTP 200 alone does not mean "the manifest is there": a proxy, a captive login page or an error # document all answer 200 with a body that is not a manifest, and a check that reads only the status # line reports those as "resolves". A manifest always carries `schemaVersion`, so the body is matched # for it — with a shell `case`, so nothing depends on jq being installed and no pipeline can invert # the result on a large body. probe() { local url="$1" resp code body # `-u` puts the credential in argv, visible to `ps` for the length of the call — and this job has # no `container:`, so that is the shared host. Kept because it is the shape every other curl caller # in scripts/ already uses (`ci-detect-already-validated.sh`, `pr-changed-files.sh`, # `select-queue.sh`, `issue-qualification-audit.sh`): fixing one site would leave the class intact # and the codebase inconsistent. The class is tracked in ersatztv#821. resp=$(curl -s -w '\n%{http_code}' -u "$auth" -H "Accept: $accept" "$url") || resp="" [ -n "$resp" ] || { printf '000 no\n'; return 0; } code=${resp##*$'\n'} body=${resp%$'\n'*} case "$body" in *'"schemaVersion"'*) printf '%s yes\n' "$code" ;; *) printf '%s no\n' "$code" ;; esac } for pin in $pins; do url="http://$registry/v2/$image_repo/manifests/$pin" attempt=1 while : ; do read -r code is_manifest <&2 rc=1 fi ;; 404) # The one unambiguous answer, and the outage this exists for. printf '::error::ci-toolchain-image-resolves: the pinned CI toolchain image %s/%s:%s IS GONE from the registry (HTTP 404). Every container: job in docker-build.yml will fail at image pull, including both required contexts, and NO diff caused it. Recovery does not need CI: rebuild that exact tag from the commit it names and push it — see docs/ci-cd.md -> "CI toolchain image" -> "When the pinned tag disappears". Root cause + the durable fix: timothy/server-management#842.\n' \ "$registry" "$image_repo" "$pin" >&2 rc=1 ;; 401|403) # `fail` rather than `rc=1`: unlike a 404, this says nothing about the pin, and it will say # the same thing about every remaining one. Abandoning the loop keeps the log to one cause. fail "the registry rejected these credentials (HTTP $code) for $registry/$image_repo:$pin, so the pin could not be checked. Fix REGISTRY_USER/REGISTRY_PASSWORD rather than reading this as a pass." ;; *) # NOT gone, and NOT a pass either. Deliberately worded apart from the 404 message: this sends # an operator to the registry's health, not to a rebuild of a tag that may be sitting there. printf '::error::ci-toolchain-image-resolves: could NOT VERIFY %s/%s:%s after %s attempt(s) (last answer: HTTP %s). This is not evidence the image is gone — it is evidence the check could not run, which fails rather than passing so the preflight cannot quietly become a no-op.\n' \ "$registry" "$image_repo" "$pin" "$attempts" "$code" >&2 rc=1 ;; esac done exit "$rc"