.gitea/workflows/dependency-scan.yml

@@ -22,6 +22,17 @@ concurrency:
   group: ersatztv-depscan
   cancel-in-progress: true
 
+# No persistent MSBuild/Roslyn servers (ersatztv#406). Workflow `env:` does not cross workflow
+# files, so docker-build.yml's copy of these does not apply here and this has to be repeated.
+# Smaller stakes than the build pipeline — `dotnet restore` + `dotnet list` are MSBuild-driven and
+# never invoke csc, so this is lingering worker nodes (hundreds of MiB), not a 7.8 GB VBCSCompiler.
+# Worth setting anyway: this runs unattended on a Monday 06:00 cron against the same host that runs
+# prod media, and node reuse keeps workers alive ~15 min after the job.
+env:
+  UseSharedCompilation: "false"
+  DOTNET_CLI_USE_MSBUILD_SERVER: "0"
+  MSBUILDDISABLENODEREUSE: "1"
+
 jobs:
   scan:
     name: NuGet vulnerable packages

.gitea/workflows/docker-build.yml

@@ -60,6 +60,26 @@ env:
   REGISTRY: 192.168.1.95:3000
   IMAGE: 192.168.1.95:3000/timothy/ersatztv
 
+  # --- CI build memory (ersatztv#406, server-management#604) ---
+  # Roslyn's `VBCSCompiler` is a *persistent* compiler server: it outlives the `dotnet build` that
+  # started it and keeps its managed heap warm for the next one. Locally that is a real speedup.
+  # In CI it buys nothing — each job container is torn down at the end of the run, so there is
+  # never a "next build" to warm — while costing a lot: 7.8 GB RSS was measured live on bumblebee,
+  # the single largest consumer on a 25 GiB host that also runs prod media. Several of those, one
+  # per concurrent job container, is what drove the host to load 340 with 21 GiB swapped.
+  #
+  # These are MSBuild properties/switches, set here as environment variables so they apply to every
+  # dotnet invocation in every job (restore/build/test/format/api-docs) without touching each call
+  # site. MSBuild surfaces environment variables as properties, and `UseSharedCompilation` is only
+  # defaulted to true when empty, so setting it here wins.
+  #
+  # NOTE: this reaches the *runner-side* dotnet jobs only. The `build` job compiles inside
+  # `docker build`, where these do not propagate — the same switches are set as ENV in the
+  # Dockerfile's SDK stage (docker/Dockerfile) to cover it.
+  UseSharedCompilation: "false"   # no persistent VBCSCompiler; csc runs per-project and exits
+  DOTNET_CLI_USE_MSBUILD_SERVER: "0"  # no persistent MSBuild server process
+  MSBUILDDISABLENODEREUSE: "1"    # MSBuild worker nodes exit with the build instead of lingering
+
 jobs:
   test:
     name: Build & test (.NET)
@@ -160,6 +180,70 @@ jobs:
             cat coverage/report/SummaryGithub.md >> "$GITHUB_STEP_SUMMARY"
           fi
 
+      # Memory of THIS job container, reported every run (ersatztv#406, server-management#604).
+      # #604 sizes the runners' per-job caps on these numbers, and until now they were inherited
+      # rather than measured: the 10g cap traces back to server-management#570 observing the image
+      # build peg 5.999/6 GiB, which is a different job entirely.
+      #
+      # ⚠️ READ THE BREAKDOWN, NOT JUST THE PEAK. `memory.peak` is the high-water mark of
+      # `memory.current`, which charges **page cache** to the cgroup as well as anonymous memory —
+      # it is NOT "peak RSS", and for a build job (NuGet/npm/obj/bin/coverage I/O) the cache
+      # dominates. Demonstrated on bumblebee: a container with anon=0 that merely reads an 800 MB
+      # file reports memory.peak=826 MiB, of which file=800 MiB. This matters because the naive
+      # reading inverts the decision: page cache is **reclaimed** under a tighter cap, not
+      # OOM-killed, so a large peak that is mostly `file` is NOT evidence that the cap must stay
+      # high. `anon` (+ a little kernel/sock) is the part that actually forces an OOM.
+      #
+      # The split below is read at end-of-job, so it is the *current* composition rather than the
+      # composition at the peak instant — indicative, not exact. Sizing a cap off one run is still
+      # wrong; take a few runs, and treat anon as the floor and peak as the (cache-inflated)
+      # ceiling. Refining this into a true peak-anon sample is ersatztv#412.
+      #
+      # Runs LAST on purpose: memory.peak read at step N reports the peak only up to N, so this
+      # sits after Coverage summary to include reportgenerator, the job's last real workload.
+      # cgroup v2 first, v1 fallback.
+      - name: Report peak container memory
+        # `always()` controls whether this step RUNS, not whether its failure fails the job — and
+        # `defaults.run.shell: bash` means `-e -o pipefail` is on, so a failed `cat`/redirect here
+        # would redden a green test job. `continue-on-error` is what actually makes it advisory,
+        # the same guarantee the Coverage summary step above uses.
+        if: always()
+        continue-on-error: true
+        run: |
+          mib() { echo "$(( ${1:-0} / 1048576 ))"; }
+          peak=""; src=""
+          for f in /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory/memory.max_usage_in_bytes; do
+            if [ -r "$f" ]; then peak=$(cat "$f" 2>/dev/null || echo ""); src="$f"; break; fi
+          done
+          if [ -z "$peak" ]; then
+            echo "No cgroup peak-memory file readable in this container -- skipping."
+            exit 0
+          fi
+          anon=""; file=""
+          if [ -r /sys/fs/cgroup/memory.stat ]; then
+            anon=$(awk '/^anon /{print $2}' /sys/fs/cgroup/memory.stat 2>/dev/null || echo "")
+            file=$(awk '/^file /{print $2}' /sys/fs/cgroup/memory.stat 2>/dev/null || echo "")
+          fi
+          echo "::group::Container memory (ersatztv#406 / server-management#604)"
+          printf 'peak (incl. page cache): %s MiB  [%s bytes, %s]\n' "$(mib "$peak")" "$peak" "$src"
+          if [ -n "$anon" ]; then
+            printf 'end-of-job anon (the part that OOMs): %s MiB\n' "$(mib "$anon")"
+            printf 'end-of-job file (page cache, reclaimable): %s MiB\n' "$(mib "${file:-0}")"
+            echo 'NOTE: peak counts reclaimable page cache. Size caps on anon, not on peak.'
+          else
+            echo 'NOTE: no memory.stat breakdown available; peak includes reclaimable page cache.'
+          fi
+          echo "::endgroup::"
+          if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
+            {
+              printf '**Container memory (test job):** peak %s MiB *(incl. reclaimable page cache)*' \
+                "$(mib "$peak")"
+              [ -n "$anon" ] && printf ' · end-of-job anon %s MiB · file %s MiB' \
+                "$(mib "$anon")" "$(mib "${file:-0}")"
+              printf '\n'
+            } >> "$GITHUB_STEP_SUMMARY" || true
+          fi
+
   migrations:
     name: EF migration integrity (SQLite + MySql)
     runs-on: ubuntu-latest
@@ -180,7 +264,36 @@ jobs:
         # No host-port binding: the job reaches this service as mysql:3306 on the shared
         # runner network. Publishing 3306 made concurrent runs collide ("port is already
         # allocated") whenever two migrations jobs overlapped.
+        #
+        # `--memory`/`--cpus` here because the runner's `container.options` (`--memory=10g`)
+        # applies to the JOB container ONLY, not to `services:` — verified by inspecting a live
+        # migrations job: the job container reported HostConfig.Memory=10737418240, its mysql
+        # service reported `mem=0 nanocpus=0`, i.e. unbounded. So every migrations run was adding
+        # an uncapped MySQL to an already-tight host (ersatztv#406, server-management#604).
+        #
+        # `--memory-swap=2g` is NOT redundant with `--memory=2g` — it is the point. Docker defaults
+        # an unset `--memory-swap` to *twice* `--memory`, so `--memory=2g` alone would grant 2g RAM
+        # **plus 2g of swap** (verified on bumblebee: `--memory=2g` alone → memory.max=2147483648
+        # AND memory.swap.max=2147483648; with `--memory-swap=2g` → memory.swap.max=0). Setting it
+        # equal to --memory disables swap for this container. That matters more here than anywhere:
+        # swap thrash on this host is the whole reason this cap exists, and a swapping mysqld mid-DDL
+        # is precisely the pathology behind the known `Command Timeout expired` migrations flake. We
+        # want a loud OOM over silent swapping — an OOM is a clear signal to raise the cap.
+        #
+        # 2g is sized on measurement rather than inheritance, but honestly: a mysql:8.4 container
+        # with this exact env peaked at 543 MiB during init and settled at 481 MiB idle (probed on
+        # bumblebee 2026-07-17). That is init+idle, NOT the 787-migration replay, which grows caches
+        # idle never touches — so treat 2g as a measured floor with headroom, not a measured
+        # ceiling. The migrations job going green is what validates it. If this OOM-kills the
+        # service, raise it deliberately — do not remove the cap, and do not re-enable swap.
+        #
+        # `--cpus=2` is a ceiling, not a reservation, and is the one number here with no measurement
+        # behind it: 787 sequential DDL statements on one connection are ~1-core-bound, so 2 is
+        # judgement. Revisit if the apply step's tail latency grows.
         options: >-
+          --memory=2g
+          --memory-swap=2g
+          --cpus=2
           --health-cmd="mysqladmin ping -h 127.0.0.1 -uroot -persatztv --silent"
           --health-interval=5s
           --health-timeout=5s
@@ -402,7 +515,9 @@ jobs:
           trap 'docker rm -f "$NAME" >/dev/null 2>&1 || true' EXIT
           echo "Pulling ${IMG}"
           docker pull "$IMG"
-          docker run -d --name "$NAME" --memory 2g \
+          # --memory-swap equal to --memory disables swap. Without it Docker defaults --memory-swap
+          # to 2x --memory, so `--memory 2g` alone silently grants 2g RAM + 2g swap (ersatztv#406).
+          docker run -d --name "$NAME" --memory 2g --memory-swap 2g \
             -e ETV_CONFIG_FOLDER=/tmp/etv/config \
             -e ETV_TRANSCODE_FOLDER=/tmp/etv/transcode \
             "$IMG"

docker/Dockerfile

@@ -56,6 +56,25 @@ COPY ErsatzTV.Infrastructure/*.csproj ./ErsatzTV.Infrastructure/
 COPY ErsatzTV.Infrastructure.Sqlite/*.csproj ./ErsatzTV.Infrastructure.Sqlite/
 COPY ErsatzTV.Infrastructure.MySql/*.csproj ./ErsatzTV.Infrastructure.MySql/
 COPY ErsatzTV.Scanner/*.csproj ./ErsatzTV.Scanner/
+
+# Disable the persistent Roslyn/MSBuild compiler servers for the in-image build (ersatztv#406,
+# server-management#604). The workflow sets these as env for the runner-side dotnet jobs, but this
+# stage compiles inside `docker build`, so the workflow's env does NOT reach it — the restore and
+# the two publishes below would otherwise spin up their own VBCSCompiler and hold its heap. This is
+# the `build` job that server-management#570 measured pegging 5.999/6 GiB, so it is the one that
+# most needs this.
+#
+# Placed here rather than at the top of the stage on purpose: an ENV invalidates every layer below
+# it, and the wget of the ~30MB openapi-generator jar above is deliberately ordered early to stay
+# cached (ersatztv#190). Nothing between that wget and this line compiles, so this is the earliest
+# point where the ENV is free.
+#
+# Build-stage only: the final image is FROM runtime-base and only COPY --from=build /app (files,
+# not ENV), so none of this lands in the shipped image or affects runtime.
+ENV UseSharedCompilation=false \
+    DOTNET_CLI_USE_MSBUILD_SERVER=0 \
+    MSBUILDDISABLENODEREUSE=1
+
 RUN dotnet restore -r linux-x64 ErsatzTV/ErsatzTV.csproj
 
 # copy everything else and build app

docs/ci-cd.md

@@ -99,20 +99,133 @@ saturated and the `small` lane idle — **queue wait exceeded every job's runtim
 | `test` | 354s | 1363s | ubuntu-latest |
 | `migrations` | 639s | 1428s | ubuntu-latest |
 | `functional-e2e` | 520s | 1447s | ubuntu-latest |
-| `api-docs` | 5s | **1722s** | ubuntu-latest → now `small` |
-| `format` | 37s | **1731s** | ubuntu-latest → now `small` |
+| `api-docs` | 5s | **1722s** | ubuntu-latest → `small` → **reverted to ubuntu-latest** (#406) |
+| `format` | 37s | **1731s** | ubuntu-latest → `small` → **reverted to ubuntu-latest** (#406) |
 | `docs-reminder` / `decisions-guard` | 10s | **5s** | small |
 
-So `api-docs` and `format` — genuinely small jobs — moved to `small`, which also drops the
-contended lane from 5 jobs to 3 and shortens the queue for the three heavy jobs. Both lanes run
-the *identical* `runner-images:ubuntu-latest` base, so `small` is a label with spare capacity, not
-a different capability. This move is only possible because those jobs now run in the **CI toolchain
-image** (below) and no longer need the runner image to supply .NET/Node. Caveat: on an
-API-touching PR `api-docs` does a full `dotnet build`, so it is not *always* small.
+`api-docs` and `format` moved to `small` because the queue wait dwarfed their runtime. Both lanes
+run the *identical* `runner-images:ubuntu-latest` base, so `small` was a label with spare capacity,
+not a different capability — a move only possible because those jobs now run in the **CI toolchain
+image** (below) and no longer need the runner image to supply .NET/Node.
 
-Queue wait is still the dominant cost and capacity is server-management's boundary — tracked in
+**Reverted 2026-07-17 (ersatztv#406 / server-management#604).** #390's own caveat — *"on an
+API-touching PR `api-docs` does a full `dotnet build`, so it is not always small"* — turned out to
+be the deciding factor, and "capacity 4 absorbs that" held only because **nothing enforces the sum**
+of the lanes' per-job caps. Each job container is correctly capped (`--memory=10g`), but 6 slots ×
+10 GiB = **60 GiB on a 25 GiB host** that also runs prod media; on 2026-07-17 bumblebee hit load
+340 with 21 GiB swapped. These are not small jobs — a live `docker stats` caught the `format` job
+container at **3.95 GiB**, which the re-sized 2 GiB `small` lane would OOM-kill outright. #604 fixes
+the queue at the source instead (`ubuntu-latest` grown to 5 slots: a 48 GiB ci-runner at capacity 4
+plus a bumblebee overflow slot), so the `small` lane can be reserved for genuinely-tiny shell jobs.
+
+Queue wait is still a dominant cost and capacity is server-management's boundary — tracked in
 **server-management#604**. The redundant triple-build behind those runtimes is **ersatztv#398**.
 
+### CI build memory: no persistent compiler servers (ersatztv#406)
+
+Roslyn's `VBCSCompiler` is a **persistent** compiler server — it outlives the `dotnet build` that
+started it and keeps its heap warm for the next one. Locally that is a genuine speedup; in CI it
+buys nothing, because each job container is torn down at the end of the run and there is never a
+"next build" to warm. It was measured at **7.8 GB RSS** on bumblebee — the single largest consumer
+on the host, and the reason each job needed a 10 GiB cap in the first place.
+
+So the workflow's top-level `env:` disables the servers for every runner-side dotnet job:
+
+| Variable | Effect |
+|---|---|
+| `UseSharedCompilation=false` | no persistent `VBCSCompiler`; `csc` runs per project and exits |
+| `DOTNET_CLI_USE_MSBUILD_SERVER=0` | no persistent MSBuild server process |
+| `MSBUILDDISABLENODEREUSE=1` | MSBuild worker nodes exit with the build instead of lingering |
+
+These are MSBuild properties set as environment variables so they apply to every `dotnet`
+invocation without touching each call site (MSBuild surfaces env vars as properties, and
+`UseSharedCompilation` is only defaulted to `true` when empty, so the env var wins).
+
+**What this does and does not shrink.** It helps the jobs that *compile* — `test`, `migrations`,
+`api-docs` on an API-touching PR, and the in-Docker `build`. It does **not** help `format`: `dotnet
+format` loads Roslyn in-process via MSBuildWorkspace and never spawns `csc`, so its measured 3.95
+GiB is unaffected. Don't size the `small` lane expecting `format` to have shrunk — it hasn't.
+
+The same three are repeated in `dependency-scan.yml`; workflow `env:` does not cross workflow files.
+That one is lower-stakes (restore/list are MSBuild-driven, so it's lingering worker nodes rather
+than a 7.8 GB VBCSCompiler) but it runs unattended on a cron against the prod media host.
+
+**The workflow `env:` does not reach the `build` job's compilation**, which happens inside
+`docker build` — the same three are set as `ENV` in the SDK stage of `docker/Dockerfile`. That is
+the job server-management#570 measured pegging 5.999/6 GiB, so it is the one that most needs this.
+Build-stage only; the final image is `FROM runtime-base`, so nothing lands in the shipped image.
+
+The `test` job's **last** step, **Report peak container memory** (`continue-on-error`, never fails a
+build), prints the cgroup's `memory.peak` plus an `anon`/`file` breakdown to the log and the job
+step summary. Read it off a recent run instead of re-deriving it by hand — but read it correctly:
+
+> ⚠️ **`memory.peak` is not "peak RSS".** It is the high-water mark of `memory.current`, which
+> charges **page cache** to the cgroup as well as anonymous memory. Proven on bumblebee: a
+> container with `anon=0` that merely reads an 800 MB file reports `memory.peak=826 MiB`, of which
+> `file=800 MiB`.
+>
+> This matters because the naive reading **inverts the decision**: page cache is *reclaimed* under a
+> tighter cap, not OOM-killed, so a big peak that is mostly `file` is **not** evidence that the cap
+> must stay high. `anon` is the part that actually forces an OOM. **Size caps on `anon`, not on
+> `peak`.** The `anon`/`file` split is read at end-of-job, so it is indicative, not the composition
+> at the peak instant — sampling a true peak-anon is **ersatztv#412**.
+
+**First measurement (PR #411, `test` job):** `peak 8305 MiB`, cache share **unquantified** by that
+run. Do not fill the gap with a guess in either direction — an independent probe (full solution
+`dotnet build`, same CI image, shared compilation off) came back `peak 9457 MiB` / **`anon 7134
+MiB`** / `file 421 MiB`, i.e. **anon dominated and cache did not**. On that evidence a 6g cap looks
+*unsafe* rather than safe, and #570's "6g proved too tight" reads as the rule rather than an
+outlier.
+
+That also means **#406's premise — *"if disabling shared compilation brings peak RSS well under 6
+GiB, the whole budget loosens"* — is looking dead**: the 7134 MiB anon was measured *with* shared
+compilation already off. Disabling the compiler servers remains right (no persistent 7.8 GB server
+sits on the host between builds) but **do not bank a looser cap budget on it**.
+
+Treat all of the above as strong-but-not-final: it is one probe, not the `test` job, and there is
+**no pre-change baseline from this instrument** (the 7.8 GB `VBCSCompiler` figure was measured
+host-wide across concurrent jobs, not inside one job container). #412 covers the real A/B.
+What *is* established: no persistent compiler server survives a build, and `migrations` is green
+with mysql capped at 2g with swap off.
+
+### `services:` containers are capped explicitly (ersatztv#406)
+
+A runner's `container.options` (`--cpus=4 --memory=10g`) applies to the **job container only, not
+to `services:`**. Verified on a live `migrations` job: the job container reported
+`HostConfig.Memory=10737418240`; its `mysql:8.4` service reported `mem=0 nanocpus=0` — unbounded.
+So `migrations` runs added an uncapped MySQL to an already-tight host.
+
+The mysql service now sets `--memory=2g --memory-swap=2g --cpus=2`.
+
+**`--memory-swap` is the part that matters, and it is easy to get wrong.** Docker defaults an unset
+`--memory-swap` to **twice** `--memory`, so `--memory=2g` *alone* grants 2g RAM **plus 2g of swap**.
+Verified on bumblebee:
+
+| Options | `memory.max` | `memory.swap.max` |
+|---|---|---|
+| `--memory=2g` | 2147483648 | **2147483648** ← 2 GiB of swap |
+| `--memory=2g --memory-swap=2g` | 2147483648 | **0** ← swap disabled |
+
+Setting `--memory-swap` equal to `--memory` disables swap for the container. On this host that is
+the whole point: swap thrash is what took prod down, and a swapping mysqld mid-DDL is precisely the
+pathology behind the known `Command Timeout expired` migrations flake. **Prefer a loud OOM over
+silent swapping** — an OOM is a clear signal to raise the cap; swapping just degrades everything.
+
+⚠️ **The same 2× applies to the runners' `container.options: --memory=10g`** — each job slot is
+really 10 GiB RAM *plus* 10 GiB swap. The "6 slots × 10 GiB = 60 GiB on a 25 GiB host" framing
+understates the promise by 2×, and it is a plausible direct mechanism for the incident's 21 GiB of
+swap. Fixing that is server-management#604's call (reported there).
+
+**On the 2g figure, honestly:** a `mysql:8.4` container with this exact env peaked at **543 MiB**
+during init and settled at 481 MiB idle (probed on bumblebee, 2026-07-17) — but that is init+idle,
+**not** the 787-migration replay, which grows table/definition caches idle never touches. So 2g is a
+measured *floor* plus headroom, not a measured ceiling; the `migrations` job going green is what
+validates it. `--cpus=2` has no measurement behind it at all — 787 sequential DDL statements on one
+connection are ~1-core-bound, so it is judgement; revisit if the apply step's tail latency grows.
+
+**Any new `services:` container needs its own explicit cap** — it will not inherit one, and it needs
+`--memory-swap` set alongside `--memory` or it silently gets 2× in swap.
+
 ### `test` job
 
 `dotnet restore` → strip the Scanner project ref (`sed -i '/Scanner/d'`, matching the

docs/decisions.md

@@ -87,6 +87,7 @@ in-file entries.
 - [2026-07-17 — Shuffle-source construction extracted to `ShuffleSourceBuilder`; per-family seam, not a god-factory (#380)](#2026-07-17--shuffle-source-construction-extracted-to-shufflesourcebuilder-per-family-seam-not-a-god-factory-380)
 - [2026-07-17 — Seasonal / date-conditional scheduling already exists (alternate schedules / playout templates); #73 closed as implemented](#2026-07-17--seasonal--date-conditional-scheduling-already-exists-alternate-schedules--playout-templates-73-closed-as-implemented)
 - [2026-07-17 — Auto-Tune DetailPanel member list = live search-index roll-up, not EF enumeration (#384)](#2026-07-17--auto-tune-detailpanel-member-list--live-search-index-roll-up-not-ef-enumeration-384)
+- [2026-07-17 — No persistent compiler servers in CI; every `services:` container gets an explicit cap; #390's small-lane move reversed (#406)](#2026-07-17--no-persistent-compiler-servers-in-ci-every-services-container-gets-an-explicit-cap-390s-small-lane-move-reversed-406)
 
 ---
 
@@ -1225,3 +1226,120 @@ the existing `PagedLibraryBrowseItemsResponseModel` / `LibraryBrowseItemResponse
 the cap — the same staleness bound the smart-collection path accepts. Read-only, catalog-read tier (no
 `[RequiresAuthentication]`), so a cold review sufficed. Sibling backend child #385 (write-path overrides +
 weights) and SPA child #386 remain open under the #383 milestone.
+
+---
+
+## 2026-07-17 — No persistent compiler servers in CI; every `services:` container gets an explicit cap; #390's small-lane move reversed (#406)
+
+Three CI changes, all downstream of one incident: on 2026-07-17 bumblebee (the **prod media** Docker
+host, 25 GiB / 12 cores) hit load **340** with **21 GiB swapped** and ~238 MiB free, taking prod
+ersatztv/jellyfin down until reboot. Prod ersatztv itself was healthy at 168 MiB throughout — the
+thrash was **CI-induced**, triggered by a burst of parallel merges to `main`. Infra sizing is
+server-management#604's boundary; these three levers live in this repo and each does more than any
+capacity knob.
+
+**1. No persistent compiler servers.** `VBCSCompiler` is a *persistent* Roslyn server: it outlives
+the `dotnet build` that started it and holds its heap for the next one. Measured at **7.8 GB RSS**
+live on bumblebee — the single largest consumer on the box, and the actual reason each job needed a
+10 GiB cap. In CI it buys **nothing**: each job container is torn down at the end of the run, so
+there is never a "next build" to warm. The workflow's top-level `env:` now sets
+`UseSharedCompilation=false`, `DOTNET_CLI_USE_MSBUILD_SERVER=0`, `MSBUILDDISABLENODEREUSE=1` —
+MSBuild properties set as env vars so they apply to every `dotnet` call without touching each call
+site (MSBuild surfaces env vars as properties; `UseSharedCompilation` only defaults to `true` when
+empty, so the env var wins). Verified locally: a default build leaves 1 `VBCSCompiler` alive, the
+same build under these vars leaves **0**, and `ErsatzTV.sln` still builds clean (0 errors).
+
+**Why the Dockerfile also sets them** — and this is the part the issue's suggestion would have
+missed: the workflow `env:` reaches the *runner-side* dotnet jobs only. The `build` job compiles
+inside `docker build`, where it does not propagate, so the SDK stage of `docker/Dockerfile` sets the
+same three as `ENV`. That is precisely the job server-management#570 measured pegging **5.999/6
+GiB** — the one that most needs it. Build-stage only; the final image is `FROM runtime-base`, so
+nothing lands in the shipped image or affects runtime.
+
+**Trade-off accepted**: without the shared server each project's `csc` is a fresh process, which
+costs some build time. Worth it — the memory spike is what takes prod down, and the cap sizing that
+spike forces is what starves the lanes.
+
+**2. `services:` containers do not inherit the runner's cap.** A runner's `container.options`
+(`--cpus=4 --memory=10g`) applies to the **job container only**. Verified by inspecting a live
+`migrations` job: the job container reported `HostConfig.Memory=10737418240`, its `mysql:8.4`
+service reported `mem=0 nanocpus=0` — **unbounded**. Every migrations run was adding an uncapped
+MySQL to an already-tight host. Now `--memory=2g --memory-swap=2g --cpus=2`.
+
+**Why `--memory-swap` is not redundant** (cold-review catch, and the sharpest thing in this change):
+Docker defaults an unset `--memory-swap` to **twice** `--memory`, so `--memory=2g` alone grants 2g
+RAM **plus 2g of swap** — verified live: `--memory=2g` → `memory.max=2147483648` **and**
+`memory.swap.max=2147483648`; `--memory=2g --memory-swap=2g` → `memory.swap.max=0`. Capping RAM
+while silently permitting swap is close to the worst outcome **on the host whose swap thrash is the
+entire reason for the cap**, and a swapping mysqld mid-DDL is exactly the pathology behind the known
+`Command Timeout expired` migrations flake — i.e. the naive cap could have made that flake worse.
+**Standing rule: prefer a loud OOM over silent swapping.** An OOM is an unambiguous "raise the cap"
+signal; swapping just degrades everything and blames something else.
+
+**The same 2× applies to the runners' 10g job slots** — each is really 10 GiB RAM *plus* 10 GiB
+swap, so the "60 GiB promise on a 25 GiB host" understates by 2× and is a plausible direct mechanism
+for the incident's 21 GiB swapped. That is server-management#604's boundary; reported there.
+
+**On 2g, honestly**: 543 MiB is init+idle, **not** the 787-migration replay (which grows caches idle
+never touches), so 2g is a measured *floor* plus headroom, not a measured ceiling — the `migrations`
+job going green is what validates it. `--cpus=2` has no measurement behind it at all; 787 sequential
+DDL statements on one connection are ~1-core-bound, so it is judgement. Recording that rather than
+dressing a guess as measurement — the same failure this entry criticises below.
+
+**Standing rule: any new `services:` container needs its own explicit cap** — it will not inherit
+one, and `--memory` without `--memory-swap` silently grants 2× in swap.
+
+**3. #390's `small`-lane move for `api-docs`/`format` reversed.** #390 moved them to dodge a ~29 min
+`ubuntu-latest` queue. The queue was real, but the lane was the wrong fix, and **#390's own comment
+flagged why**: *"on an API-touching PR this job does a full `dotnet build`, so it is not always a
+'small' job; capacity 4 absorbs that."* "Capacity 4 absorbs that" held only because **nothing
+enforces the sum** of the lanes' caps — 6 slots × 10 GiB on a 25 GiB host is a 60 GiB promise. These
+are not small jobs: a live `docker stats` caught the `format` job container at **3.95 GiB**, which
+#604's re-sized 2 GiB `small` lane would OOM-kill outright. #604 grows `ubuntu-latest` to 5 slots
+(48 GiB ci-runner at capacity 4 + a bumblebee overflow slot) and fixes the queue at the source. This
+one is **order-coupled with #604**: the small lane's caps can't tighten until it lands.
+
+**Measurement is now continuous, not a one-off.** The `test` job's **last** step reports the
+cgroup's `memory.peak` plus an `anon`/`file` breakdown (`continue-on-error`, tolerates absence).
+#604 sizes both runners' caps on that number, and until now it was *inherited* rather than measured
+— the 10g cap traces back to #570 observing a different job entirely. Two things that look like
+details but are the whole point: it must run **last** (`memory.peak` read at step N reports the peak
+only up to N, so an earlier placement silently excludes the job's later workload), and
+`continue-on-error` — not `if: always()` — is what makes it advisory (`always()` controls whether a
+step *runs*, not whether its failure fails the job, and `defaults.run.shell: bash` means `-e` is on).
+
+**And then the instrument taught us the lesson twice, both times at our own expense.** The first
+reading came back `peak 8305 MiB`. `memory.peak` is the high-water mark of `memory.current`, which
+charges **page cache** as well as anonymous memory — proven: a container with `anon=0` that merely
+reads an 800 MB file reports `memory.peak=826 MiB`, `file=800 MiB`. Page cache is *reclaimed* under
+a tighter cap, not OOM-killed, so a large peak that is mostly `file` is **not** evidence a cap must
+stay high. `anon` is what forces an OOM; size caps on it. The step now prints the split (end-of-job,
+so indicative rather than peak-instant); a true peak-anon sample is **#412**.
+
+**Then we made the same mistake in the opposite direction.** Having established that peak
+*overstates*, the docs (and a report to #604) leaned to "so this number is probably mostly cache."
+That was a guess about *magnitude* dressed in a verified fact about *mechanism* — and an independent
+probe killed it: a full solution build in the CI image with shared compilation off measured `peak
+9457 MiB` / **`anon 7134 MiB`** / `file 421 MiB`. **Anon dominated.** So a 6g cap looks *unsafe*,
+#570's "6g proved too tight" is the rule rather than an outlier, and **#406's premise ("if this
+brings peak RSS well under 6 GiB, the whole budget loosens") is looking dead** — the 7134 MiB was
+measured *with* shared compilation already off. The switches are still right (no persistent 7.8 GB
+server between builds); the looser budget they were supposed to buy is not.
+
+**What is NOT established:** there is still no pre-change baseline from this instrument — the 7.8 GB
+`VBCSCompiler` figure was measured host-wide across concurrent jobs, not inside one job container —
+and the anon figure above is one probe, not the `test` job. #412 covers the real A/B. What *is*
+established: no persistent compiler server survives a build, and `migrations` is green with mysql
+capped at 2g with swap disabled.
+
+The lesson generalizes, and note it bit *this* change twice — once in the issue's premise and once
+in our own instrument: this repo's CI perf work keeps stating numbers from plausibility rather than
+measurement (see #390's "2–4min" apt-ffmpeg estimate; real 110s, and not load-bearing). Measuring
+the wrong quantity precisely is the same failure wearing a lab coat.
+
+**What this does NOT shrink**: `format`. `dotnet format` loads Roslyn in-process via
+MSBuildWorkspace and never spawns `csc`, so its measured 3.95 GiB is untouched by any of this. Do
+not size the `small` lane expecting otherwise.
+
+**Not addressed here**: the 12–35 min queue waits (server-management#604) and the redundant
+triple-build (#398).

scripts/migration-smoke.sh

@@ -99,7 +99,12 @@ echo "migration-smoke: rehearsing $IMAGE ($IMG_ID)"
 # pending migrations on startup; it logs "Applying database migrations" then "Done applying database
 # migrations", and on failure the host stops (default BackgroundServiceExceptionBehavior = StopHost),
 # so the container exits. We gate PASS on the "Done" line, FAIL on early exit / a migration exception.
-docker run -d --name "$NAME" --memory 2g \
+# --memory-swap equal to --memory disables swap for this container. Without it Docker defaults
+# --memory-swap to 2x --memory, so `--memory 2g` alone silently grants 2g RAM + 2g of swap
+# (ersatztv#406). That matters here specifically: this smoke runs on the PROD host in the release
+# path, so a runaway migration should die loudly against its cap rather than quietly swap out the
+# box that is serving media.
+docker run -d --name "$NAME" --memory 2g --memory-swap 2g \
   -e ETV_CONFIG_FOLDER=/config \
   -e ETV_TRANSCODE_FOLDER=/tmp/etv/transcode \
   -v "$WORK/config:/config" \