diff --git a/.gitea/workflows/dependency-scan.yml b/.gitea/workflows/dependency-scan.yml index c2c68ab29..a971293e5 100644 --- a/.gitea/workflows/dependency-scan.yml +++ b/.gitea/workflows/dependency-scan.yml @@ -22,6 +22,17 @@ concurrency: group: ersatztv-depscan cancel-in-progress: true +# No persistent MSBuild/Roslyn servers (ersatztv#406). Workflow `env:` does not cross workflow +# files, so docker-build.yml's copy of these does not apply here and this has to be repeated. +# Smaller stakes than the build pipeline — `dotnet restore` + `dotnet list` are MSBuild-driven and +# never invoke csc, so this is lingering worker nodes (hundreds of MiB), not a 7.8 GB VBCSCompiler. +# Worth setting anyway: this runs unattended on a Monday 06:00 cron against the same host that runs +# prod media, and node reuse keeps workers alive ~15 min after the job. +env: + UseSharedCompilation: "false" + DOTNET_CLI_USE_MSBUILD_SERVER: "0" + MSBUILDDISABLENODEREUSE: "1" + jobs: scan: name: NuGet vulnerable packages diff --git a/.gitea/workflows/docker-build.yml b/.gitea/workflows/docker-build.yml index 981cdeb4c..1c308ac33 100644 --- a/.gitea/workflows/docker-build.yml +++ b/.gitea/workflows/docker-build.yml @@ -60,6 +60,26 @@ env: REGISTRY: 192.168.1.95:3000 IMAGE: 192.168.1.95:3000/timothy/ersatztv + # --- CI build memory (ersatztv#406, server-management#604) --- + # Roslyn's `VBCSCompiler` is a *persistent* compiler server: it outlives the `dotnet build` that + # started it and keeps its managed heap warm for the next one. Locally that is a real speedup. + # In CI it buys nothing — each job container is torn down at the end of the run, so there is + # never a "next build" to warm — while costing a lot: 7.8 GB RSS was measured live on bumblebee, + # the single largest consumer on a 25 GiB host that also runs prod media. Several of those, one + # per concurrent job container, is what drove the host to load 340 with 21 GiB swapped. + # + # These are MSBuild properties/switches, set here as environment variables so they apply to every + # dotnet invocation in every job (restore/build/test/format/api-docs) without touching each call + # site. MSBuild surfaces environment variables as properties, and `UseSharedCompilation` is only + # defaulted to true when empty, so setting it here wins. + # + # NOTE: this reaches the *runner-side* dotnet jobs only. The `build` job compiles inside + # `docker build`, where these do not propagate — the same switches are set as ENV in the + # Dockerfile's SDK stage (docker/Dockerfile) to cover it. + UseSharedCompilation: "false" # no persistent VBCSCompiler; csc runs per-project and exits + DOTNET_CLI_USE_MSBUILD_SERVER: "0" # no persistent MSBuild server process + MSBUILDDISABLENODEREUSE: "1" # MSBuild worker nodes exit with the build instead of lingering + jobs: test: name: Build & test (.NET) @@ -160,6 +180,70 @@ jobs: cat coverage/report/SummaryGithub.md >> "$GITHUB_STEP_SUMMARY" fi + # Memory of THIS job container, reported every run (ersatztv#406, server-management#604). + # #604 sizes the runners' per-job caps on these numbers, and until now they were inherited + # rather than measured: the 10g cap traces back to server-management#570 observing the image + # build peg 5.999/6 GiB, which is a different job entirely. + # + # ⚠️ READ THE BREAKDOWN, NOT JUST THE PEAK. `memory.peak` is the high-water mark of + # `memory.current`, which charges **page cache** to the cgroup as well as anonymous memory — + # it is NOT "peak RSS", and for a build job (NuGet/npm/obj/bin/coverage I/O) the cache + # dominates. Demonstrated on bumblebee: a container with anon=0 that merely reads an 800 MB + # file reports memory.peak=826 MiB, of which file=800 MiB. This matters because the naive + # reading inverts the decision: page cache is **reclaimed** under a tighter cap, not + # OOM-killed, so a large peak that is mostly `file` is NOT evidence that the cap must stay + # high. `anon` (+ a little kernel/sock) is the part that actually forces an OOM. + # + # The split below is read at end-of-job, so it is the *current* composition rather than the + # composition at the peak instant — indicative, not exact. Sizing a cap off one run is still + # wrong; take a few runs, and treat anon as the floor and peak as the (cache-inflated) + # ceiling. Refining this into a true peak-anon sample is ersatztv#412. + # + # Runs LAST on purpose: memory.peak read at step N reports the peak only up to N, so this + # sits after Coverage summary to include reportgenerator, the job's last real workload. + # cgroup v2 first, v1 fallback. + - name: Report peak container memory + # `always()` controls whether this step RUNS, not whether its failure fails the job — and + # `defaults.run.shell: bash` means `-e -o pipefail` is on, so a failed `cat`/redirect here + # would redden a green test job. `continue-on-error` is what actually makes it advisory, + # the same guarantee the Coverage summary step above uses. + if: always() + continue-on-error: true + run: | + mib() { echo "$(( ${1:-0} / 1048576 ))"; } + peak=""; src="" + for f in /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory/memory.max_usage_in_bytes; do + if [ -r "$f" ]; then peak=$(cat "$f" 2>/dev/null || echo ""); src="$f"; break; fi + done + if [ -z "$peak" ]; then + echo "No cgroup peak-memory file readable in this container -- skipping." + exit 0 + fi + anon=""; file="" + if [ -r /sys/fs/cgroup/memory.stat ]; then + anon=$(awk '/^anon /{print $2}' /sys/fs/cgroup/memory.stat 2>/dev/null || echo "") + file=$(awk '/^file /{print $2}' /sys/fs/cgroup/memory.stat 2>/dev/null || echo "") + fi + echo "::group::Container memory (ersatztv#406 / server-management#604)" + printf 'peak (incl. page cache): %s MiB [%s bytes, %s]\n' "$(mib "$peak")" "$peak" "$src" + if [ -n "$anon" ]; then + printf 'end-of-job anon (the part that OOMs): %s MiB\n' "$(mib "$anon")" + printf 'end-of-job file (page cache, reclaimable): %s MiB\n' "$(mib "${file:-0}")" + echo 'NOTE: peak counts reclaimable page cache. Size caps on anon, not on peak.' + else + echo 'NOTE: no memory.stat breakdown available; peak includes reclaimable page cache.' + fi + echo "::endgroup::" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + printf '**Container memory (test job):** peak %s MiB *(incl. reclaimable page cache)*' \ + "$(mib "$peak")" + [ -n "$anon" ] && printf ' · end-of-job anon %s MiB · file %s MiB' \ + "$(mib "$anon")" "$(mib "${file:-0}")" + printf '\n' + } >> "$GITHUB_STEP_SUMMARY" || true + fi + migrations: name: EF migration integrity (SQLite + MySql) runs-on: ubuntu-latest @@ -180,7 +264,36 @@ jobs: # No host-port binding: the job reaches this service as mysql:3306 on the shared # runner network. Publishing 3306 made concurrent runs collide ("port is already # allocated") whenever two migrations jobs overlapped. + # + # `--memory`/`--cpus` here because the runner's `container.options` (`--memory=10g`) + # applies to the JOB container ONLY, not to `services:` — verified by inspecting a live + # migrations job: the job container reported HostConfig.Memory=10737418240, its mysql + # service reported `mem=0 nanocpus=0`, i.e. unbounded. So every migrations run was adding + # an uncapped MySQL to an already-tight host (ersatztv#406, server-management#604). + # + # `--memory-swap=2g` is NOT redundant with `--memory=2g` — it is the point. Docker defaults + # an unset `--memory-swap` to *twice* `--memory`, so `--memory=2g` alone would grant 2g RAM + # **plus 2g of swap** (verified on bumblebee: `--memory=2g` alone → memory.max=2147483648 + # AND memory.swap.max=2147483648; with `--memory-swap=2g` → memory.swap.max=0). Setting it + # equal to --memory disables swap for this container. That matters more here than anywhere: + # swap thrash on this host is the whole reason this cap exists, and a swapping mysqld mid-DDL + # is precisely the pathology behind the known `Command Timeout expired` migrations flake. We + # want a loud OOM over silent swapping — an OOM is a clear signal to raise the cap. + # + # 2g is sized on measurement rather than inheritance, but honestly: a mysql:8.4 container + # with this exact env peaked at 543 MiB during init and settled at 481 MiB idle (probed on + # bumblebee 2026-07-17). That is init+idle, NOT the 787-migration replay, which grows caches + # idle never touches — so treat 2g as a measured floor with headroom, not a measured + # ceiling. The migrations job going green is what validates it. If this OOM-kills the + # service, raise it deliberately — do not remove the cap, and do not re-enable swap. + # + # `--cpus=2` is a ceiling, not a reservation, and is the one number here with no measurement + # behind it: 787 sequential DDL statements on one connection are ~1-core-bound, so 2 is + # judgement. Revisit if the apply step's tail latency grows. options: >- + --memory=2g + --memory-swap=2g + --cpus=2 --health-cmd="mysqladmin ping -h 127.0.0.1 -uroot -persatztv --silent" --health-interval=5s --health-timeout=5s @@ -402,7 +515,9 @@ jobs: trap 'docker rm -f "$NAME" >/dev/null 2>&1 || true' EXIT echo "Pulling ${IMG}" docker pull "$IMG" - docker run -d --name "$NAME" --memory 2g \ + # --memory-swap equal to --memory disables swap. Without it Docker defaults --memory-swap + # to 2x --memory, so `--memory 2g` alone silently grants 2g RAM + 2g swap (ersatztv#406). + docker run -d --name "$NAME" --memory 2g --memory-swap 2g \ -e ETV_CONFIG_FOLDER=/tmp/etv/config \ -e ETV_TRANSCODE_FOLDER=/tmp/etv/transcode \ "$IMG" diff --git a/docker/Dockerfile b/docker/Dockerfile index 24e95ec85..1cefcf67f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -56,6 +56,25 @@ COPY ErsatzTV.Infrastructure/*.csproj ./ErsatzTV.Infrastructure/ COPY ErsatzTV.Infrastructure.Sqlite/*.csproj ./ErsatzTV.Infrastructure.Sqlite/ COPY ErsatzTV.Infrastructure.MySql/*.csproj ./ErsatzTV.Infrastructure.MySql/ COPY ErsatzTV.Scanner/*.csproj ./ErsatzTV.Scanner/ + +# Disable the persistent Roslyn/MSBuild compiler servers for the in-image build (ersatztv#406, +# server-management#604). The workflow sets these as env for the runner-side dotnet jobs, but this +# stage compiles inside `docker build`, so the workflow's env does NOT reach it — the restore and +# the two publishes below would otherwise spin up their own VBCSCompiler and hold its heap. This is +# the `build` job that server-management#570 measured pegging 5.999/6 GiB, so it is the one that +# most needs this. +# +# Placed here rather than at the top of the stage on purpose: an ENV invalidates every layer below +# it, and the wget of the ~30MB openapi-generator jar above is deliberately ordered early to stay +# cached (ersatztv#190). Nothing between that wget and this line compiles, so this is the earliest +# point where the ENV is free. +# +# Build-stage only: the final image is FROM runtime-base and only COPY --from=build /app (files, +# not ENV), so none of this lands in the shipped image or affects runtime. +ENV UseSharedCompilation=false \ + DOTNET_CLI_USE_MSBUILD_SERVER=0 \ + MSBUILDDISABLENODEREUSE=1 + RUN dotnet restore -r linux-x64 ErsatzTV/ErsatzTV.csproj # copy everything else and build app diff --git a/docs/ci-cd.md b/docs/ci-cd.md index 532bb8666..8af1eea0b 100644 --- a/docs/ci-cd.md +++ b/docs/ci-cd.md @@ -99,20 +99,133 @@ saturated and the `small` lane idle — **queue wait exceeded every job's runtim | `test` | 354s | 1363s | ubuntu-latest | | `migrations` | 639s | 1428s | ubuntu-latest | | `functional-e2e` | 520s | 1447s | ubuntu-latest | -| `api-docs` | 5s | **1722s** | ubuntu-latest → now `small` | -| `format` | 37s | **1731s** | ubuntu-latest → now `small` | +| `api-docs` | 5s | **1722s** | ubuntu-latest → `small` → **reverted to ubuntu-latest** (#406) | +| `format` | 37s | **1731s** | ubuntu-latest → `small` → **reverted to ubuntu-latest** (#406) | | `docs-reminder` / `decisions-guard` | 10s | **5s** | small | -So `api-docs` and `format` — genuinely small jobs — moved to `small`, which also drops the -contended lane from 5 jobs to 3 and shortens the queue for the three heavy jobs. Both lanes run -the *identical* `runner-images:ubuntu-latest` base, so `small` is a label with spare capacity, not -a different capability. This move is only possible because those jobs now run in the **CI toolchain -image** (below) and no longer need the runner image to supply .NET/Node. Caveat: on an -API-touching PR `api-docs` does a full `dotnet build`, so it is not *always* small. +`api-docs` and `format` moved to `small` because the queue wait dwarfed their runtime. Both lanes +run the *identical* `runner-images:ubuntu-latest` base, so `small` was a label with spare capacity, +not a different capability — a move only possible because those jobs now run in the **CI toolchain +image** (below) and no longer need the runner image to supply .NET/Node. -Queue wait is still the dominant cost and capacity is server-management's boundary — tracked in +**Reverted 2026-07-17 (ersatztv#406 / server-management#604).** #390's own caveat — *"on an +API-touching PR `api-docs` does a full `dotnet build`, so it is not always small"* — turned out to +be the deciding factor, and "capacity 4 absorbs that" held only because **nothing enforces the sum** +of the lanes' per-job caps. Each job container is correctly capped (`--memory=10g`), but 6 slots × +10 GiB = **60 GiB on a 25 GiB host** that also runs prod media; on 2026-07-17 bumblebee hit load +340 with 21 GiB swapped. These are not small jobs — a live `docker stats` caught the `format` job +container at **3.95 GiB**, which the re-sized 2 GiB `small` lane would OOM-kill outright. #604 fixes +the queue at the source instead (`ubuntu-latest` grown to 5 slots: a 48 GiB ci-runner at capacity 4 +plus a bumblebee overflow slot), so the `small` lane can be reserved for genuinely-tiny shell jobs. + +Queue wait is still a dominant cost and capacity is server-management's boundary — tracked in **server-management#604**. The redundant triple-build behind those runtimes is **ersatztv#398**. +### CI build memory: no persistent compiler servers (ersatztv#406) + +Roslyn's `VBCSCompiler` is a **persistent** compiler server — it outlives the `dotnet build` that +started it and keeps its heap warm for the next one. Locally that is a genuine speedup; in CI it +buys nothing, because each job container is torn down at the end of the run and there is never a +"next build" to warm. It was measured at **7.8 GB RSS** on bumblebee — the single largest consumer +on the host, and the reason each job needed a 10 GiB cap in the first place. + +So the workflow's top-level `env:` disables the servers for every runner-side dotnet job: + +| Variable | Effect | +|---|---| +| `UseSharedCompilation=false` | no persistent `VBCSCompiler`; `csc` runs per project and exits | +| `DOTNET_CLI_USE_MSBUILD_SERVER=0` | no persistent MSBuild server process | +| `MSBUILDDISABLENODEREUSE=1` | MSBuild worker nodes exit with the build instead of lingering | + +These are MSBuild properties set as environment variables so they apply to every `dotnet` +invocation without touching each call site (MSBuild surfaces env vars as properties, and +`UseSharedCompilation` is only defaulted to `true` when empty, so the env var wins). + +**What this does and does not shrink.** It helps the jobs that *compile* — `test`, `migrations`, +`api-docs` on an API-touching PR, and the in-Docker `build`. It does **not** help `format`: `dotnet +format` loads Roslyn in-process via MSBuildWorkspace and never spawns `csc`, so its measured 3.95 +GiB is unaffected. Don't size the `small` lane expecting `format` to have shrunk — it hasn't. + +The same three are repeated in `dependency-scan.yml`; workflow `env:` does not cross workflow files. +That one is lower-stakes (restore/list are MSBuild-driven, so it's lingering worker nodes rather +than a 7.8 GB VBCSCompiler) but it runs unattended on a cron against the prod media host. + +**The workflow `env:` does not reach the `build` job's compilation**, which happens inside +`docker build` — the same three are set as `ENV` in the SDK stage of `docker/Dockerfile`. That is +the job server-management#570 measured pegging 5.999/6 GiB, so it is the one that most needs this. +Build-stage only; the final image is `FROM runtime-base`, so nothing lands in the shipped image. + +The `test` job's **last** step, **Report peak container memory** (`continue-on-error`, never fails a +build), prints the cgroup's `memory.peak` plus an `anon`/`file` breakdown to the log and the job +step summary. Read it off a recent run instead of re-deriving it by hand — but read it correctly: + +> ⚠️ **`memory.peak` is not "peak RSS".** It is the high-water mark of `memory.current`, which +> charges **page cache** to the cgroup as well as anonymous memory. Proven on bumblebee: a +> container with `anon=0` that merely reads an 800 MB file reports `memory.peak=826 MiB`, of which +> `file=800 MiB`. +> +> This matters because the naive reading **inverts the decision**: page cache is *reclaimed* under a +> tighter cap, not OOM-killed, so a big peak that is mostly `file` is **not** evidence that the cap +> must stay high. `anon` is the part that actually forces an OOM. **Size caps on `anon`, not on +> `peak`.** The `anon`/`file` split is read at end-of-job, so it is indicative, not the composition +> at the peak instant — sampling a true peak-anon is **ersatztv#412**. + +**First measurement (PR #411, `test` job):** `peak 8305 MiB`, cache share **unquantified** by that +run. Do not fill the gap with a guess in either direction — an independent probe (full solution +`dotnet build`, same CI image, shared compilation off) came back `peak 9457 MiB` / **`anon 7134 +MiB`** / `file 421 MiB`, i.e. **anon dominated and cache did not**. On that evidence a 6g cap looks +*unsafe* rather than safe, and #570's "6g proved too tight" reads as the rule rather than an +outlier. + +That also means **#406's premise — *"if disabling shared compilation brings peak RSS well under 6 +GiB, the whole budget loosens"* — is looking dead**: the 7134 MiB anon was measured *with* shared +compilation already off. Disabling the compiler servers remains right (no persistent 7.8 GB server +sits on the host between builds) but **do not bank a looser cap budget on it**. + +Treat all of the above as strong-but-not-final: it is one probe, not the `test` job, and there is +**no pre-change baseline from this instrument** (the 7.8 GB `VBCSCompiler` figure was measured +host-wide across concurrent jobs, not inside one job container). #412 covers the real A/B. +What *is* established: no persistent compiler server survives a build, and `migrations` is green +with mysql capped at 2g with swap off. + +### `services:` containers are capped explicitly (ersatztv#406) + +A runner's `container.options` (`--cpus=4 --memory=10g`) applies to the **job container only, not +to `services:`**. Verified on a live `migrations` job: the job container reported +`HostConfig.Memory=10737418240`; its `mysql:8.4` service reported `mem=0 nanocpus=0` — unbounded. +So `migrations` runs added an uncapped MySQL to an already-tight host. + +The mysql service now sets `--memory=2g --memory-swap=2g --cpus=2`. + +**`--memory-swap` is the part that matters, and it is easy to get wrong.** Docker defaults an unset +`--memory-swap` to **twice** `--memory`, so `--memory=2g` *alone* grants 2g RAM **plus 2g of swap**. +Verified on bumblebee: + +| Options | `memory.max` | `memory.swap.max` | +|---|---|---| +| `--memory=2g` | 2147483648 | **2147483648** ← 2 GiB of swap | +| `--memory=2g --memory-swap=2g` | 2147483648 | **0** ← swap disabled | + +Setting `--memory-swap` equal to `--memory` disables swap for the container. On this host that is +the whole point: swap thrash is what took prod down, and a swapping mysqld mid-DDL is precisely the +pathology behind the known `Command Timeout expired` migrations flake. **Prefer a loud OOM over +silent swapping** — an OOM is a clear signal to raise the cap; swapping just degrades everything. + +⚠️ **The same 2× applies to the runners' `container.options: --memory=10g`** — each job slot is +really 10 GiB RAM *plus* 10 GiB swap. The "6 slots × 10 GiB = 60 GiB on a 25 GiB host" framing +understates the promise by 2×, and it is a plausible direct mechanism for the incident's 21 GiB of +swap. Fixing that is server-management#604's call (reported there). + +**On the 2g figure, honestly:** a `mysql:8.4` container with this exact env peaked at **543 MiB** +during init and settled at 481 MiB idle (probed on bumblebee, 2026-07-17) — but that is init+idle, +**not** the 787-migration replay, which grows table/definition caches idle never touches. So 2g is a +measured *floor* plus headroom, not a measured ceiling; the `migrations` job going green is what +validates it. `--cpus=2` has no measurement behind it at all — 787 sequential DDL statements on one +connection are ~1-core-bound, so it is judgement; revisit if the apply step's tail latency grows. + +**Any new `services:` container needs its own explicit cap** — it will not inherit one, and it needs +`--memory-swap` set alongside `--memory` or it silently gets 2× in swap. + ### `test` job `dotnet restore` → strip the Scanner project ref (`sed -i '/Scanner/d'`, matching the diff --git a/docs/decisions.md b/docs/decisions.md index 8927699ea..e39aff7cb 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -87,6 +87,7 @@ in-file entries. - [2026-07-17 — Shuffle-source construction extracted to `ShuffleSourceBuilder`; per-family seam, not a god-factory (#380)](#2026-07-17--shuffle-source-construction-extracted-to-shufflesourcebuilder-per-family-seam-not-a-god-factory-380) - [2026-07-17 — Seasonal / date-conditional scheduling already exists (alternate schedules / playout templates); #73 closed as implemented](#2026-07-17--seasonal--date-conditional-scheduling-already-exists-alternate-schedules--playout-templates-73-closed-as-implemented) - [2026-07-17 — Auto-Tune DetailPanel member list = live search-index roll-up, not EF enumeration (#384)](#2026-07-17--auto-tune-detailpanel-member-list--live-search-index-roll-up-not-ef-enumeration-384) +- [2026-07-17 — No persistent compiler servers in CI; every `services:` container gets an explicit cap; #390's small-lane move reversed (#406)](#2026-07-17--no-persistent-compiler-servers-in-ci-every-services-container-gets-an-explicit-cap-390s-small-lane-move-reversed-406) --- @@ -1225,3 +1226,120 @@ the existing `PagedLibraryBrowseItemsResponseModel` / `LibraryBrowseItemResponse the cap — the same staleness bound the smart-collection path accepts. Read-only, catalog-read tier (no `[RequiresAuthentication]`), so a cold review sufficed. Sibling backend child #385 (write-path overrides + weights) and SPA child #386 remain open under the #383 milestone. + +--- + +## 2026-07-17 — No persistent compiler servers in CI; every `services:` container gets an explicit cap; #390's small-lane move reversed (#406) + +Three CI changes, all downstream of one incident: on 2026-07-17 bumblebee (the **prod media** Docker +host, 25 GiB / 12 cores) hit load **340** with **21 GiB swapped** and ~238 MiB free, taking prod +ersatztv/jellyfin down until reboot. Prod ersatztv itself was healthy at 168 MiB throughout — the +thrash was **CI-induced**, triggered by a burst of parallel merges to `main`. Infra sizing is +server-management#604's boundary; these three levers live in this repo and each does more than any +capacity knob. + +**1. No persistent compiler servers.** `VBCSCompiler` is a *persistent* Roslyn server: it outlives +the `dotnet build` that started it and holds its heap for the next one. Measured at **7.8 GB RSS** +live on bumblebee — the single largest consumer on the box, and the actual reason each job needed a +10 GiB cap. In CI it buys **nothing**: each job container is torn down at the end of the run, so +there is never a "next build" to warm. The workflow's top-level `env:` now sets +`UseSharedCompilation=false`, `DOTNET_CLI_USE_MSBUILD_SERVER=0`, `MSBUILDDISABLENODEREUSE=1` — +MSBuild properties set as env vars so they apply to every `dotnet` call without touching each call +site (MSBuild surfaces env vars as properties; `UseSharedCompilation` only defaults to `true` when +empty, so the env var wins). Verified locally: a default build leaves 1 `VBCSCompiler` alive, the +same build under these vars leaves **0**, and `ErsatzTV.sln` still builds clean (0 errors). + +**Why the Dockerfile also sets them** — and this is the part the issue's suggestion would have +missed: the workflow `env:` reaches the *runner-side* dotnet jobs only. The `build` job compiles +inside `docker build`, where it does not propagate, so the SDK stage of `docker/Dockerfile` sets the +same three as `ENV`. That is precisely the job server-management#570 measured pegging **5.999/6 +GiB** — the one that most needs it. Build-stage only; the final image is `FROM runtime-base`, so +nothing lands in the shipped image or affects runtime. + +**Trade-off accepted**: without the shared server each project's `csc` is a fresh process, which +costs some build time. Worth it — the memory spike is what takes prod down, and the cap sizing that +spike forces is what starves the lanes. + +**2. `services:` containers do not inherit the runner's cap.** A runner's `container.options` +(`--cpus=4 --memory=10g`) applies to the **job container only**. Verified by inspecting a live +`migrations` job: the job container reported `HostConfig.Memory=10737418240`, its `mysql:8.4` +service reported `mem=0 nanocpus=0` — **unbounded**. Every migrations run was adding an uncapped +MySQL to an already-tight host. Now `--memory=2g --memory-swap=2g --cpus=2`. + +**Why `--memory-swap` is not redundant** (cold-review catch, and the sharpest thing in this change): +Docker defaults an unset `--memory-swap` to **twice** `--memory`, so `--memory=2g` alone grants 2g +RAM **plus 2g of swap** — verified live: `--memory=2g` → `memory.max=2147483648` **and** +`memory.swap.max=2147483648`; `--memory=2g --memory-swap=2g` → `memory.swap.max=0`. Capping RAM +while silently permitting swap is close to the worst outcome **on the host whose swap thrash is the +entire reason for the cap**, and a swapping mysqld mid-DDL is exactly the pathology behind the known +`Command Timeout expired` migrations flake — i.e. the naive cap could have made that flake worse. +**Standing rule: prefer a loud OOM over silent swapping.** An OOM is an unambiguous "raise the cap" +signal; swapping just degrades everything and blames something else. + +**The same 2× applies to the runners' 10g job slots** — each is really 10 GiB RAM *plus* 10 GiB +swap, so the "60 GiB promise on a 25 GiB host" understates by 2× and is a plausible direct mechanism +for the incident's 21 GiB swapped. That is server-management#604's boundary; reported there. + +**On 2g, honestly**: 543 MiB is init+idle, **not** the 787-migration replay (which grows caches idle +never touches), so 2g is a measured *floor* plus headroom, not a measured ceiling — the `migrations` +job going green is what validates it. `--cpus=2` has no measurement behind it at all; 787 sequential +DDL statements on one connection are ~1-core-bound, so it is judgement. Recording that rather than +dressing a guess as measurement — the same failure this entry criticises below. + +**Standing rule: any new `services:` container needs its own explicit cap** — it will not inherit +one, and `--memory` without `--memory-swap` silently grants 2× in swap. + +**3. #390's `small`-lane move for `api-docs`/`format` reversed.** #390 moved them to dodge a ~29 min +`ubuntu-latest` queue. The queue was real, but the lane was the wrong fix, and **#390's own comment +flagged why**: *"on an API-touching PR this job does a full `dotnet build`, so it is not always a +'small' job; capacity 4 absorbs that."* "Capacity 4 absorbs that" held only because **nothing +enforces the sum** of the lanes' caps — 6 slots × 10 GiB on a 25 GiB host is a 60 GiB promise. These +are not small jobs: a live `docker stats` caught the `format` job container at **3.95 GiB**, which +#604's re-sized 2 GiB `small` lane would OOM-kill outright. #604 grows `ubuntu-latest` to 5 slots +(48 GiB ci-runner at capacity 4 + a bumblebee overflow slot) and fixes the queue at the source. This +one is **order-coupled with #604**: the small lane's caps can't tighten until it lands. + +**Measurement is now continuous, not a one-off.** The `test` job's **last** step reports the +cgroup's `memory.peak` plus an `anon`/`file` breakdown (`continue-on-error`, tolerates absence). +#604 sizes both runners' caps on that number, and until now it was *inherited* rather than measured +— the 10g cap traces back to #570 observing a different job entirely. Two things that look like +details but are the whole point: it must run **last** (`memory.peak` read at step N reports the peak +only up to N, so an earlier placement silently excludes the job's later workload), and +`continue-on-error` — not `if: always()` — is what makes it advisory (`always()` controls whether a +step *runs*, not whether its failure fails the job, and `defaults.run.shell: bash` means `-e` is on). + +**And then the instrument taught us the lesson twice, both times at our own expense.** The first +reading came back `peak 8305 MiB`. `memory.peak` is the high-water mark of `memory.current`, which +charges **page cache** as well as anonymous memory — proven: a container with `anon=0` that merely +reads an 800 MB file reports `memory.peak=826 MiB`, `file=800 MiB`. Page cache is *reclaimed* under +a tighter cap, not OOM-killed, so a large peak that is mostly `file` is **not** evidence a cap must +stay high. `anon` is what forces an OOM; size caps on it. The step now prints the split (end-of-job, +so indicative rather than peak-instant); a true peak-anon sample is **#412**. + +**Then we made the same mistake in the opposite direction.** Having established that peak +*overstates*, the docs (and a report to #604) leaned to "so this number is probably mostly cache." +That was a guess about *magnitude* dressed in a verified fact about *mechanism* — and an independent +probe killed it: a full solution build in the CI image with shared compilation off measured `peak +9457 MiB` / **`anon 7134 MiB`** / `file 421 MiB`. **Anon dominated.** So a 6g cap looks *unsafe*, +#570's "6g proved too tight" is the rule rather than an outlier, and **#406's premise ("if this +brings peak RSS well under 6 GiB, the whole budget loosens") is looking dead** — the 7134 MiB was +measured *with* shared compilation already off. The switches are still right (no persistent 7.8 GB +server between builds); the looser budget they were supposed to buy is not. + +**What is NOT established:** there is still no pre-change baseline from this instrument — the 7.8 GB +`VBCSCompiler` figure was measured host-wide across concurrent jobs, not inside one job container — +and the anon figure above is one probe, not the `test` job. #412 covers the real A/B. What *is* +established: no persistent compiler server survives a build, and `migrations` is green with mysql +capped at 2g with swap disabled. + +The lesson generalizes, and note it bit *this* change twice — once in the issue's premise and once +in our own instrument: this repo's CI perf work keeps stating numbers from plausibility rather than +measurement (see #390's "2–4min" apt-ffmpeg estimate; real 110s, and not load-bearing). Measuring +the wrong quantity precisely is the same failure wearing a lab coat. + +**What this does NOT shrink**: `format`. `dotnet format` loads Roslyn in-process via +MSBuildWorkspace and never spawns `csc`, so its measured 3.95 GiB is untouched by any of this. Do +not size the `small` lane expecting otherwise. + +**Not addressed here**: the 12–35 min queue waits (server-management#604) and the redundant +triple-build (#398). diff --git a/scripts/migration-smoke.sh b/scripts/migration-smoke.sh index bc12a9478..388c63065 100755 --- a/scripts/migration-smoke.sh +++ b/scripts/migration-smoke.sh @@ -99,7 +99,12 @@ echo "migration-smoke: rehearsing $IMAGE ($IMG_ID)" # pending migrations on startup; it logs "Applying database migrations" then "Done applying database # migrations", and on failure the host stops (default BackgroundServiceExceptionBehavior = StopHost), # so the container exits. We gate PASS on the "Done" line, FAIL on early exit / a migration exception. -docker run -d --name "$NAME" --memory 2g \ +# --memory-swap equal to --memory disables swap for this container. Without it Docker defaults +# --memory-swap to 2x --memory, so `--memory 2g` alone silently grants 2g RAM + 2g of swap +# (ersatztv#406). That matters here specifically: this smoke runs on the PROD host in the release +# path, so a runaway migration should die loudly against its cap rather than quietly swap out the +# box that is serving media. +docker run -d --name "$NAME" --memory 2g --memory-swap 2g \ -e ETV_CONFIG_FOLDER=/config \ -e ETV_TRANSCODE_FOLDER=/tmp/etv/transcode \ -v "$WORK/config:/config" \