From 4263cf7919f7dedd08911ec7f2bd490de574848f Mon Sep 17 00:00:00 2001 From: Timothy Date: Mon, 20 Jul 2026 22:23:51 +0200 Subject: [PATCH 1/2] ci(508): move both `docker build` jobs off the `small` lane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `runs-on: small` carried two jobs that are not small: docker-build.yml's image build, and ci-image.yml's toolchain buildx. The second reads as lightweight because it is "docker-only, no toolchain needed — it *builds* the toolchain", which is true and yet describes the heaviest job in the lane. A lane's per-job memory cap is set by its worst member, not its median, so these two pinned `small` at --memory=10g. On bumblebee's 25 GiB — also the prod media host — that permits exactly ONE slot, and four jobs shared it. So "widen the lane" and "keep the heavy jobs" were never simultaneously available. The symptom that forces the issue is not queue wait. A saturated lane also wedges DISPATCHED jobs in act's setup phase: >10 min in_progress, no log file written at all, then failure, before Checkout runs. That is where "decisions.md is a known flake, just rerun it" came from — the rerun works only because it lands after load clears, so a capacity problem read as a bug in the guard. With both builds on ubuntu-latest, `small` is a checkout plus a `git diff` and server-management#639 caps it at 1 GiB, widening it to 4 slots across two hosts while committing LESS RAM to CI than the single slot did. so it cannot be dispatched until the jobs it would queue behind have finished. refs #508 Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitea/workflows/ci-image.yml | 11 ++++-- .gitea/workflows/docker-build.yml | 20 +++++++---- docs/ci-cd.md | 56 +++++++++++++++++++++++++------ docs/decisions.md | 26 ++++++++++++++ 4 files changed, 94 insertions(+), 19 deletions(-) diff --git a/.gitea/workflows/ci-image.yml b/.gitea/workflows/ci-image.yml index b0fb53fae..6c3be2e09 100644 --- a/.gitea/workflows/ci-image.yml +++ b/.gitea/workflows/ci-image.yml @@ -53,9 +53,14 @@ env: jobs: build: name: Build & push CI image - # `small` = the small-jobs runner lane. This is a docker-only job (no toolchain needed — - # it *builds* the toolchain), same as docker-build.yml's `build` job. - runs-on: small + # Moved off `small` with docker-build.yml's `build` (server-management#639). Being + # "docker-only" made it look lightweight, but it is a full buildx of the .NET + # toolchain image — the heaviest thing that ran in that lane. `small` is now + # git-only and capped at 1g per job, which would OOM this build. + # + # Rare trigger (pushes touching docker/ci + a weekly cron), so it costs the + # ubuntu-latest lane almost nothing, and ci-runner (.127) runs no prod workload. + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.gitea/workflows/docker-build.yml b/.gitea/workflows/docker-build.yml index 691e3abeb..872c0aeb7 100644 --- a/.gitea/workflows/docker-build.yml +++ b/.gitea/workflows/docker-build.yml @@ -496,12 +496,20 @@ jobs: build: name: Build & push image (amd64) - # `small` = the dedicated small-jobs runner lane (server-management#574). - # On PR runs this job only resolves its skip, but Gitea still dispatches it - # as a task — on the ubuntu-latest runners that skip queued behind long - # builds (observed 31 min). Real builds (main/tags) run on bumblebee, - # capped at 4 CPUs / 10g. - runs-on: small + # Moved back off `small` (server-management#639). This is the one HEAVY job that + # was still in that lane, and its 10g requirement was what pinned the lane's + # per-job cap at 10g — which in turn capped the lane at ONE slot on a 25 GiB + # host. Four jobs sharing one slot is what starved the git-only checks in act's + # setup phase (>10 min, no logs, then fail). With this job gone, `small` is + # git-only and can run wide and tiny on two hosts. + # + # The `ubuntu-latest` queueing that sent it to `small` in the first place + # (server-management#574: a PR-run skip stuck 31 min behind long builds) does not + # come back, because `needs: [test, migrations]` means this job cannot be + # dispatched until those two have already finished — by which point the lane it + # was queueing behind has drained. Real builds (main/tags) get the full + # ubuntu-latest allotment: 4 CPUs / 10g on ci-runner (.127). + runs-on: ubuntu-latest needs: [test, migrations] if: github.event_name != 'pull_request' steps: diff --git a/docs/ci-cd.md b/docs/ci-cd.md index fbfe1819a..b56431a43 100644 --- a/docs/ci-cd.md +++ b/docs/ci-cd.md @@ -101,16 +101,30 @@ its superseded run, and image builds still serialize within their own ref. Do NO (History: originally one global group serializing ALL runs for the single runner — with three runners that starved the queue; changed 2026-07-11, server-management#574.) -Three runners serve the fork (server-management#570/#574): `ci-runner` (VM 127 pve4, -`ubuntu-latest`, 2 slots), `bumblebee-runner` (bumblebee, `ubuntu-latest`, 2 slots, -jobs capped `--cpus=4 --memory=10g` so CI can't starve prod media playback), and -`small-runner` (bumblebee, label **`small`**, 4 slots) — the small-jobs lane. The -`build` and `docs-reminder` jobs use `runs-on: small`: Gitea dispatches a job as a -runner task even when its `if` skips it, and those skip-tasks used to wait behind -long builds (observed 31 min) stalling every PR run. +**Four** runners serve the fork (server-management#570/#574/#639): -**Lane assignment (ersatztv#390).** The `ubuntu-latest` lane has **4 slots** (2 + 2) and the -`small` lane has 4. A 2026-07-17 audit of the Actions API found the `ubuntu-latest` lane +| Runner | Host | Label | Slots | Per-job cap | +|---|---|---|---|---| +| `ci-runner` | VM 127 (pve4) — no prod workload | `ubuntu-latest` | 4 | `--cpus=4 --memory=10g` | +| `bumblebee-runner` | bumblebee — prod media | `ubuntu-latest` | 2 | `--cpus=4 --memory=10g --cpu-shares=256` | +| `small-runner` | bumblebee — prod media | **`small`** | 2 | `--cpus=1 --memory=1g --cpu-shares=256` | +| `jazz-small-runner` | jazz — prod media (#633) | **`small`** | 2 | `--cpus=1 --memory=1g --cpu-shares=128` | + +The `small` lane exists because Gitea dispatches a job as a runner task **even when its +`if` skips it**, and those skip-tasks used to wait behind long builds (observed 31 min), +stalling every PR run. `--cpu-shares` below the default 1024 is what makes a runner on a +prod media host acceptable: under contention CI loses to the transcoders (ersatztv 1536 / +jellyfin), which are the reason those hosts exist. + +**`small` is git-only, and that is load-bearing (server-management#639).** Everything in +the lane is a checkout plus a `git diff`: `decisions-guard`, `ci-image-pin`, +`docs-reminder`. Nothing there runs a compiler or a `docker build`, which is why the lane +can be capped at 1 GiB per job. Route a heavy job here and it will OOM — give it +`ubuntu-latest`, or its own label on `ci-runner`, the only host with no prod workload. + +**Lane assignment (ersatztv#390).** *Slot counts below are as-of 2026-07-17; the table above is +current.* At the time, the `ubuntu-latest` lane had **4 slots** (2 + 2) and the +`small` lane 4. A 2026-07-17 audit of the Actions API found the `ubuntu-latest` lane saturated and the `small` lane idle — **queue wait exceeded every job's runtime**: | Job | Runtime | Queue wait | Lane | @@ -143,6 +157,26 @@ gone, so it is no longer a reason to keep the lane large. `api-docs` on an API-t Queue wait is still a dominant cost and capacity is server-management's boundary — tracked in **server-management#604**. The redundant triple-build behind those runtimes is **ersatztv#398**. +**The other failure mode: setup-phase starvation (server-management#639, 2026-07-20).** The +table above measures *queue wait* — time before a job is dispatched. A saturated lane also +produces a second, much more confusing symptom: a job that *is* dispatched, sits `in_progress` +for >10 minutes, writes **no log file at all** (`OpenLogs … .log.zst: file does not exist`), +and then fails — wedged in act's job-**setup** phase, before Checkout. Same-config siblings +that started 90s earlier finished in seconds; a concurrent job's log showed a normally-fast +compile taking a 7-minute gap between projects. This is the origin of the "`decisions.md` is a +known flake, just rerun it" folklore: the rerun succeeds only because it lands after load +clears, so the guard's logic gets blamed for a capacity problem. + +The fix was **not** more capacity for its own sake. `small` was stuck at one slot because it +still held two heavy jobs — `docker-build.yml`'s image build and `ci-image.yml`'s toolchain +buildx (the latter reads as lightweight because it is "docker-only", but it is the heaviest +thing that ran in the lane) — and their 10 GiB requirement set the lane's per-job cap, which +on a 25 GiB host permits exactly one slot. Moving both to `ubuntu-latest` made the lane +genuinely tiny, so it could widen to **4 slots across two hosts while committing less RAM to +CI than the single slot did**. `docker-build.yml`'s `build` does not re-create #574's +skip-task queueing, because `needs: [test, migrations]` means it cannot be dispatched until +the lane it would queue behind has already drained. + ### CI build memory: no persistent compiler servers (ersatztv#406) Roslyn's `VBCSCompiler` is a **persistent** compiler server — it outlives the `dotnet build` that @@ -533,7 +567,9 @@ ersatztv#299 seeded-media/scanner E2E follow-ups will need. `ci-image.yml` triggers on pushes touching `docker/ci/**`, `workflow_dispatch`, and a weekly Monday 05:00 UTC cron (base-image security updates; Gitea registers `schedule` only from `main`). It runs on -the `small` lane and, like `docker-build.yml`, needs BuildKit's inline `http = true` for the HTTP +`ubuntu-latest` — it was on `small` until server-management#639, where "docker-only" was found to be +a poor proxy for "small": this is a full buildx of the .NET toolchain image, the heaviest job in that +lane. Like `docker-build.yml`, it needs BuildKit's inline `http = true` for the HTTP registry. Renovate tracks the Dockerfile's image pins (`dockerfile` manager, see `renovate.json`). **Three container-specific gotchas** — worth knowing if you add a job or a step: diff --git a/docs/decisions.md b/docs/decisions.md index 492532cd7..68323d5e4 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -2440,3 +2440,29 @@ ErsatzTV. Full design: `docs/superpowers/specs/2026-07-20-qsv-native-decode-desi - **Native decode is Linux-only.** Guarded with `!OperatingSystem.IsWindows()` in the QSV builder — FFmpeg has no `vaapi` hwaccel on Windows (and Windows QSV capabilities are over-reported), so on Windows a QSV profile keeps QSV decode regardless of the flag. +## 2026-07-20 — `runs-on: small` means git-only; the two `docker build` jobs move to `ubuntu-latest` (server-management#639) + +- **The `small` lane is defined by what a job *does*, not by how long it usually takes.** Both jobs + removed from it here were justified as small on a runtime argument that only held in the common case: + `docker-build.yml`'s `build` is a 1-second skip on PR runs (but a real image build on main/tags), and + `ci-image.yml`'s `build` was reasoned about as "docker-only, no toolchain needed — it *builds* the + toolchain", which is true and yet describes the single heaviest job in the lane. The lane's per-job + memory cap is set by its worst member, not its median, so both of these forced `--memory=10g`. +- **That cap, not a capacity decision, is what pinned the lane at one slot.** 10 GiB per slot on a + 25 GiB host that also runs prod media permits exactly one — the sum-of-caps rule from #406/#604 + (6 slots × 10 GiB on a 25 GiB host produced load 340 and 21 GiB of swap). So "widen the lane" and + "keep the heavy jobs" were never simultaneously available; the earlier note in the runner config had + parked the widening indefinitely behind moving the lane to a different host. +- **Fixing the cap dominates fixing the capacity.** With both builds on `ubuntu-latest`, `small` is a + checkout plus a `git diff`, cappable at 1 GiB, so it widened from 1 slot to **4 across two hosts + while committing less RAM to CI than the single slot did**. A second runner was added on jazz at + `--cpu-shares=128` — CI on a prod media host is only acceptable while it loses every scheduling + contest to the transcoders. +- **The symptom this fixes is not queue wait.** A saturated lane also wedges *dispatched* jobs in act's + setup phase: >10 min `in_progress`, **no log file written at all**, then failure, before Checkout + runs. That produced the standing "`decisions.md` is a known flake, just rerun it" belief — the rerun + works only because it lands after load clears, so a capacity problem read as a bug in the guard. A + job that fails with zero log output is evidence about the runner, not about the job. +- **#574's skip-task queueing does not return** by moving `build` back to `ubuntu-latest`: + `needs: [test, migrations]` means it cannot be dispatched until the jobs it would have queued behind + have already finished. -- 2.47.3 From 5709bf5a2c0027e68280a61173b6fa601e71bb98 Mon Sep 17 00:00:00 2001 From: Timothy Date: Mon, 20 Jul 2026 22:29:56 +0200 Subject: [PATCH 2/2] ci(508): bump the CI toolchain pin to the image this branch published MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Editing .gitea/workflows/ci-image.yml is itself a trigger path for ci-image.yml, so the previous commit republished the toolchain image at its own sha. `ci-image-pin` then correctly failed: the pin still named 07048b8 while `git log -1 -- docker/ci .gitea/workflows/ci-image.yml` resolved to 4263cf79. The image content is unchanged — that commit only moved the job's `runs-on:` — but the pin's contract is "the pin equals the last commit that touched the image sources", not "the bytes differ", so it has to move. Verified 4263cf7 is actually published to the registry before pinning it; a pin that doesn't resolve to a real image would break every `container:` job at once. This is why the bump lands as a SEPARATE commit: the tag is the short sha of the pushed branch tip, so a single commit could never contain its own sha. Splitting it makes the branch self-consistent — the source-touching commit stays the last one to touch those paths, and this one carries the matching pin. refs #508 Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitea/workflows/docker-build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitea/workflows/docker-build.yml b/.gitea/workflows/docker-build.yml index 872c0aeb7..937ecaf02 100644 --- a/.gitea/workflows/docker-build.yml +++ b/.gitea/workflows/docker-build.yml @@ -25,7 +25,7 @@ name: Build ErsatzTV Image # cannot read the workflow `env` context. **Bump all five together**; see docs/ci-cd.md -> # "CI toolchain image" for the two-step procedure. # -# CI image pin: 192.168.1.95:3000/timothy/ersatztv-ci:07048b8 +# CI image pin: 192.168.1.95:3000/timothy/ersatztv-ci:4263cf7 # # DOCS-ONLY SKIP (ersatztv#416): a change that touches only docs/** or *.md has nothing for the # heavy jobs to validate. `test`, `migrations`, `functional-e2e` and `build` each run @@ -103,7 +103,7 @@ jobs: name: Build & test (.NET) runs-on: ubuntu-latest container: - image: 192.168.1.95:3000/timothy/ersatztv-ci:07048b8 + image: 192.168.1.95:3000/timothy/ersatztv-ci:4263cf7 credentials: username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} @@ -258,7 +258,7 @@ jobs: name: EF migration integrity (SQLite + MySql) runs-on: ubuntu-latest container: - image: 192.168.1.95:3000/timothy/ersatztv-ci:07048b8 + image: 192.168.1.95:3000/timothy/ersatztv-ci:4263cf7 credentials: username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} @@ -417,7 +417,7 @@ jobs: # v* tag builds. if: github.event_name == 'pull_request' || github.ref == 'refs/heads/main' container: - image: 192.168.1.95:3000/timothy/ersatztv-ci:07048b8 + image: 192.168.1.95:3000/timothy/ersatztv-ci:4263cf7 credentials: username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} @@ -785,7 +785,7 @@ jobs: # 48 GiB at capacity 4 + a bumblebee overflow slot), which fixes the queue at the source. runs-on: ubuntu-latest container: - image: 192.168.1.95:3000/timothy/ersatztv-ci:07048b8 + image: 192.168.1.95:3000/timothy/ersatztv-ci:4263cf7 credentials: username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} @@ -880,7 +880,7 @@ jobs: # move to a lighter lane is a server-management capacity call (#604). runs-on: ubuntu-latest container: - image: 192.168.1.95:3000/timothy/ersatztv-ci:07048b8 + image: 192.168.1.95:3000/timothy/ersatztv-ci:4263cf7 credentials: username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} -- 2.47.3