diff --git a/.gitea/workflows/ci-image.yml b/.gitea/workflows/ci-image.yml index b0fb53fae..6c3be2e09 100644 --- a/.gitea/workflows/ci-image.yml +++ b/.gitea/workflows/ci-image.yml @@ -53,9 +53,14 @@ env: jobs: build: name: Build & push CI image - # `small` = the small-jobs runner lane. This is a docker-only job (no toolchain needed — - # it *builds* the toolchain), same as docker-build.yml's `build` job. - runs-on: small + # Moved off `small` with docker-build.yml's `build` (server-management#639). Being + # "docker-only" made it look lightweight, but it is a full buildx of the .NET + # toolchain image — the heaviest thing that ran in that lane. `small` is now + # git-only and capped at 1g per job, which would OOM this build. + # + # Rare trigger (pushes touching docker/ci + a weekly cron), so it costs the + # ubuntu-latest lane almost nothing, and ci-runner (.127) runs no prod workload. + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.gitea/workflows/docker-build.yml b/.gitea/workflows/docker-build.yml index 691e3abeb..937ecaf02 100644 --- a/.gitea/workflows/docker-build.yml +++ b/.gitea/workflows/docker-build.yml @@ -25,7 +25,7 @@ name: Build ErsatzTV Image # cannot read the workflow `env` context. **Bump all five together**; see docs/ci-cd.md -> # "CI toolchain image" for the two-step procedure. # -# CI image pin: 192.168.1.95:3000/timothy/ersatztv-ci:07048b8 +# CI image pin: 192.168.1.95:3000/timothy/ersatztv-ci:4263cf7 # # DOCS-ONLY SKIP (ersatztv#416): a change that touches only docs/** or *.md has nothing for the # heavy jobs to validate. `test`, `migrations`, `functional-e2e` and `build` each run @@ -103,7 +103,7 @@ jobs: name: Build & test (.NET) runs-on: ubuntu-latest container: - image: 192.168.1.95:3000/timothy/ersatztv-ci:07048b8 + image: 192.168.1.95:3000/timothy/ersatztv-ci:4263cf7 credentials: username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} @@ -258,7 +258,7 @@ jobs: name: EF migration integrity (SQLite + MySql) runs-on: ubuntu-latest container: - image: 192.168.1.95:3000/timothy/ersatztv-ci:07048b8 + image: 192.168.1.95:3000/timothy/ersatztv-ci:4263cf7 credentials: username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} @@ -417,7 +417,7 @@ jobs: # v* tag builds. if: github.event_name == 'pull_request' || github.ref == 'refs/heads/main' container: - image: 192.168.1.95:3000/timothy/ersatztv-ci:07048b8 + image: 192.168.1.95:3000/timothy/ersatztv-ci:4263cf7 credentials: username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} @@ -496,12 +496,20 @@ jobs: build: name: Build & push image (amd64) - # `small` = the dedicated small-jobs runner lane (server-management#574). - # On PR runs this job only resolves its skip, but Gitea still dispatches it - # as a task — on the ubuntu-latest runners that skip queued behind long - # builds (observed 31 min). Real builds (main/tags) run on bumblebee, - # capped at 4 CPUs / 10g. - runs-on: small + # Moved back off `small` (server-management#639). This is the one HEAVY job that + # was still in that lane, and its 10g requirement was what pinned the lane's + # per-job cap at 10g — which in turn capped the lane at ONE slot on a 25 GiB + # host. Four jobs sharing one slot is what starved the git-only checks in act's + # setup phase (>10 min, no logs, then fail). With this job gone, `small` is + # git-only and can run wide and tiny on two hosts. + # + # The `ubuntu-latest` queueing that sent it to `small` in the first place + # (server-management#574: a PR-run skip stuck 31 min behind long builds) does not + # come back, because `needs: [test, migrations]` means this job cannot be + # dispatched until those two have already finished — by which point the lane it + # was queueing behind has drained. Real builds (main/tags) get the full + # ubuntu-latest allotment: 4 CPUs / 10g on ci-runner (.127). + runs-on: ubuntu-latest needs: [test, migrations] if: github.event_name != 'pull_request' steps: @@ -777,7 +785,7 @@ jobs: # 48 GiB at capacity 4 + a bumblebee overflow slot), which fixes the queue at the source. runs-on: ubuntu-latest container: - image: 192.168.1.95:3000/timothy/ersatztv-ci:07048b8 + image: 192.168.1.95:3000/timothy/ersatztv-ci:4263cf7 credentials: username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} @@ -872,7 +880,7 @@ jobs: # move to a lighter lane is a server-management capacity call (#604). runs-on: ubuntu-latest container: - image: 192.168.1.95:3000/timothy/ersatztv-ci:07048b8 + image: 192.168.1.95:3000/timothy/ersatztv-ci:4263cf7 credentials: username: ${{ secrets.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} diff --git a/docs/ci-cd.md b/docs/ci-cd.md index fbfe1819a..b56431a43 100644 --- a/docs/ci-cd.md +++ b/docs/ci-cd.md @@ -101,16 +101,30 @@ its superseded run, and image builds still serialize within their own ref. Do NO (History: originally one global group serializing ALL runs for the single runner — with three runners that starved the queue; changed 2026-07-11, server-management#574.) -Three runners serve the fork (server-management#570/#574): `ci-runner` (VM 127 pve4, -`ubuntu-latest`, 2 slots), `bumblebee-runner` (bumblebee, `ubuntu-latest`, 2 slots, -jobs capped `--cpus=4 --memory=10g` so CI can't starve prod media playback), and -`small-runner` (bumblebee, label **`small`**, 4 slots) — the small-jobs lane. The -`build` and `docs-reminder` jobs use `runs-on: small`: Gitea dispatches a job as a -runner task even when its `if` skips it, and those skip-tasks used to wait behind -long builds (observed 31 min) stalling every PR run. +**Four** runners serve the fork (server-management#570/#574/#639): -**Lane assignment (ersatztv#390).** The `ubuntu-latest` lane has **4 slots** (2 + 2) and the -`small` lane has 4. A 2026-07-17 audit of the Actions API found the `ubuntu-latest` lane +| Runner | Host | Label | Slots | Per-job cap | +|---|---|---|---|---| +| `ci-runner` | VM 127 (pve4) — no prod workload | `ubuntu-latest` | 4 | `--cpus=4 --memory=10g` | +| `bumblebee-runner` | bumblebee — prod media | `ubuntu-latest` | 2 | `--cpus=4 --memory=10g --cpu-shares=256` | +| `small-runner` | bumblebee — prod media | **`small`** | 2 | `--cpus=1 --memory=1g --cpu-shares=256` | +| `jazz-small-runner` | jazz — prod media (#633) | **`small`** | 2 | `--cpus=1 --memory=1g --cpu-shares=128` | + +The `small` lane exists because Gitea dispatches a job as a runner task **even when its +`if` skips it**, and those skip-tasks used to wait behind long builds (observed 31 min), +stalling every PR run. `--cpu-shares` below the default 1024 is what makes a runner on a +prod media host acceptable: under contention CI loses to the transcoders (ersatztv 1536 / +jellyfin), which are the reason those hosts exist. + +**`small` is git-only, and that is load-bearing (server-management#639).** Everything in +the lane is a checkout plus a `git diff`: `decisions-guard`, `ci-image-pin`, +`docs-reminder`. Nothing there runs a compiler or a `docker build`, which is why the lane +can be capped at 1 GiB per job. Route a heavy job here and it will OOM — give it +`ubuntu-latest`, or its own label on `ci-runner`, the only host with no prod workload. + +**Lane assignment (ersatztv#390).** *Slot counts below are as-of 2026-07-17; the table above is +current.* At the time, the `ubuntu-latest` lane had **4 slots** (2 + 2) and the +`small` lane 4. A 2026-07-17 audit of the Actions API found the `ubuntu-latest` lane saturated and the `small` lane idle — **queue wait exceeded every job's runtime**: | Job | Runtime | Queue wait | Lane | @@ -143,6 +157,26 @@ gone, so it is no longer a reason to keep the lane large. `api-docs` on an API-t Queue wait is still a dominant cost and capacity is server-management's boundary — tracked in **server-management#604**. The redundant triple-build behind those runtimes is **ersatztv#398**. +**The other failure mode: setup-phase starvation (server-management#639, 2026-07-20).** The +table above measures *queue wait* — time before a job is dispatched. A saturated lane also +produces a second, much more confusing symptom: a job that *is* dispatched, sits `in_progress` +for >10 minutes, writes **no log file at all** (`OpenLogs … .log.zst: file does not exist`), +and then fails — wedged in act's job-**setup** phase, before Checkout. Same-config siblings +that started 90s earlier finished in seconds; a concurrent job's log showed a normally-fast +compile taking a 7-minute gap between projects. This is the origin of the "`decisions.md` is a +known flake, just rerun it" folklore: the rerun succeeds only because it lands after load +clears, so the guard's logic gets blamed for a capacity problem. + +The fix was **not** more capacity for its own sake. `small` was stuck at one slot because it +still held two heavy jobs — `docker-build.yml`'s image build and `ci-image.yml`'s toolchain +buildx (the latter reads as lightweight because it is "docker-only", but it is the heaviest +thing that ran in the lane) — and their 10 GiB requirement set the lane's per-job cap, which +on a 25 GiB host permits exactly one slot. Moving both to `ubuntu-latest` made the lane +genuinely tiny, so it could widen to **4 slots across two hosts while committing less RAM to +CI than the single slot did**. `docker-build.yml`'s `build` does not re-create #574's +skip-task queueing, because `needs: [test, migrations]` means it cannot be dispatched until +the lane it would queue behind has already drained. + ### CI build memory: no persistent compiler servers (ersatztv#406) Roslyn's `VBCSCompiler` is a **persistent** compiler server — it outlives the `dotnet build` that @@ -533,7 +567,9 @@ ersatztv#299 seeded-media/scanner E2E follow-ups will need. `ci-image.yml` triggers on pushes touching `docker/ci/**`, `workflow_dispatch`, and a weekly Monday 05:00 UTC cron (base-image security updates; Gitea registers `schedule` only from `main`). It runs on -the `small` lane and, like `docker-build.yml`, needs BuildKit's inline `http = true` for the HTTP +`ubuntu-latest` — it was on `small` until server-management#639, where "docker-only" was found to be +a poor proxy for "small": this is a full buildx of the .NET toolchain image, the heaviest job in that +lane. Like `docker-build.yml`, it needs BuildKit's inline `http = true` for the HTTP registry. Renovate tracks the Dockerfile's image pins (`dockerfile` manager, see `renovate.json`). **Three container-specific gotchas** — worth knowing if you add a job or a step: diff --git a/docs/decisions.md b/docs/decisions.md index 492532cd7..68323d5e4 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -2440,3 +2440,29 @@ ErsatzTV. Full design: `docs/superpowers/specs/2026-07-20-qsv-native-decode-desi - **Native decode is Linux-only.** Guarded with `!OperatingSystem.IsWindows()` in the QSV builder — FFmpeg has no `vaapi` hwaccel on Windows (and Windows QSV capabilities are over-reported), so on Windows a QSV profile keeps QSV decode regardless of the flag. +## 2026-07-20 — `runs-on: small` means git-only; the two `docker build` jobs move to `ubuntu-latest` (server-management#639) + +- **The `small` lane is defined by what a job *does*, not by how long it usually takes.** Both jobs + removed from it here were justified as small on a runtime argument that only held in the common case: + `docker-build.yml`'s `build` is a 1-second skip on PR runs (but a real image build on main/tags), and + `ci-image.yml`'s `build` was reasoned about as "docker-only, no toolchain needed — it *builds* the + toolchain", which is true and yet describes the single heaviest job in the lane. The lane's per-job + memory cap is set by its worst member, not its median, so both of these forced `--memory=10g`. +- **That cap, not a capacity decision, is what pinned the lane at one slot.** 10 GiB per slot on a + 25 GiB host that also runs prod media permits exactly one — the sum-of-caps rule from #406/#604 + (6 slots × 10 GiB on a 25 GiB host produced load 340 and 21 GiB of swap). So "widen the lane" and + "keep the heavy jobs" were never simultaneously available; the earlier note in the runner config had + parked the widening indefinitely behind moving the lane to a different host. +- **Fixing the cap dominates fixing the capacity.** With both builds on `ubuntu-latest`, `small` is a + checkout plus a `git diff`, cappable at 1 GiB, so it widened from 1 slot to **4 across two hosts + while committing less RAM to CI than the single slot did**. A second runner was added on jazz at + `--cpu-shares=128` — CI on a prod media host is only acceptable while it loses every scheduling + contest to the transcoders. +- **The symptom this fixes is not queue wait.** A saturated lane also wedges *dispatched* jobs in act's + setup phase: >10 min `in_progress`, **no log file written at all**, then failure, before Checkout + runs. That produced the standing "`decisions.md` is a known flake, just rerun it" belief — the rerun + works only because it lands after load clears, so a capacity problem read as a bug in the guard. A + job that fails with zero log output is evidence about the runner, not about the job. +- **#574's skip-task queueing does not return** by moving `build` back to `ubuntu-latest`: + `needs: [test, migrations]` means it cannot be dispatched until the jobs it would have queued behind + have already finished.