ci(412): sample true peak-anon in the test job, not cache-inflated memory.peak #475

Merged
timothy merged 1 commits from ci/412-peak-anon-sampler into main 2026-07-19 21:09:30 +02:00
4 changed files with 223 additions and 82 deletions
+26 -61
View File
@@ -183,6 +183,16 @@ jobs:
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
run: sed -i '/Scanner/d' ErsatzTV/ErsatzTV.csproj
# Start the true peak-anon sampler just before the memory-heavy dotnet Build/Test/Coverage so
# its high-water mark spans them (SPA build/test above are comparatively light). Paired with the
# "Report peak container memory" step below. continue-on-error + a fail-open script => this
# instrumentation never reddens a build. Why anon and not memory.peak: ersatztv#412 /
# scripts/ci-peak-anon.sh header / docs/ci-cd.md "CI build memory".
- name: Start peak-anon sampler (ersatztv#412)
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
continue-on-error: true
run: scripts/ci-peak-anon.sh start
- name: Build
if: steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true'
run: dotnet build --configuration Release --no-restore
@@ -223,71 +233,26 @@ jobs:
cat coverage/report/SummaryGithub.md >> "$GITHUB_STEP_SUMMARY"
fi
# Memory of THIS job container, reported every run (ersatztv#406, server-management#604).
# #604 sizes the runners' per-job caps on these numbers, and until now they were inherited
# rather than measured: the 10g cap traces back to server-management#570 observing the image
# build peg 5.999/6 GiB, which is a different job entirely.
# Memory of THIS job container, reported every run (ersatztv#406/#412, server-management#604).
# #604 sizes the runners' per-job caps on these numbers. The headline is the TRUE PEAK ANON
# sampled by the "Start peak-anon sampler" step above — NOT `memory.peak`, which is the
# high-water mark of memory.current and charges reclaimable page cache to the cgroup (a build
# job does heavy NuGet/npm/obj/bin/coverage I/O, so cache can dominate the peak). Page cache is
# reclaimed under a tighter cap, not OOM-killed, so sizing a cap off `memory.peak` inverts the
# decision. peak anon is the OOM-forcing number. Full rationale + the bumblebee demo:
# scripts/ci-peak-anon.sh header and docs/ci-cd.md "CI build memory".
#
# ⚠️ READ THE BREAKDOWN, NOT JUST THE PEAK. `memory.peak` is the high-water mark of
# `memory.current`, which charges **page cache** to the cgroup as well as anonymous memory —
# it is NOT "peak RSS", and for a build job (NuGet/npm/obj/bin/coverage I/O) the cache
# dominates. Demonstrated on bumblebee: a container with anon=0 that merely reads an 800 MB
# file reports memory.peak=826 MiB, of which file=800 MiB. This matters because the naive
# reading inverts the decision: page cache is **reclaimed** under a tighter cap, not
# OOM-killed, so a large peak that is mostly `file` is NOT evidence that the cap must stay
# high. `anon` (+ a little kernel/sock) is the part that actually forces an OOM.
#
# The split below is read at end-of-job, so it is the *current* composition rather than the
# composition at the peak instant — indicative, not exact. Sizing a cap off one run is still
# wrong; take a few runs, and treat anon as the floor and peak as the (cache-inflated)
# ceiling. Refining this into a true peak-anon sample is ersatztv#412.
#
# Runs LAST on purpose: memory.peak read at step N reports the peak only up to N, so this
# sits after Coverage summary to include reportgenerator, the job's last real workload.
# cgroup v2 first, v1 fallback.
#
# Skipped on docs-only runs (ersatztv#416): nothing ran, so there is nothing to measure.
# Runs LAST on purpose (after Coverage summary / reportgenerator, the job's last real workload)
# and stops the sampler. `always()` so a failed Build/Test still gets a peak reading; the split
# is read here (end-of-job = composition then, not at the peak instant — that is exactly why the
# sampler exists). Skipped on docs-only/already-validated runs (nothing ran to measure).
- name: Report peak container memory
# `always()` controls whether this step RUNS, not whether its failure fails the job — and
# `defaults.run.shell: bash` means `-e -o pipefail` is on, so a failed `cat`/redirect here
# would redden a green test job. `continue-on-error` is what actually makes it advisory,
# the same guarantee the Coverage summary step above uses.
# `always()` controls whether this step RUNS, not whether its failure fails the job. With
# `defaults.run.shell: bash` (`-e -o pipefail`) a stray non-zero here would redden a green
# test job, so `continue-on-error` makes it advisory — the same guarantee Coverage summary uses.
if: ${{ always() && steps.detect.outputs.docs_only != 'true' && steps.revalidate.outputs.skip != 'true' }}
continue-on-error: true
run: |
mib() { echo "$(( ${1:-0} / 1048576 ))"; }
peak=""; src=""
for f in /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory/memory.max_usage_in_bytes; do
if [ -r "$f" ]; then peak=$(cat "$f" 2>/dev/null || echo ""); src="$f"; break; fi
done
if [ -z "$peak" ]; then
echo "No cgroup peak-memory file readable in this container -- skipping."
exit 0
fi
anon=""; file=""
if [ -r /sys/fs/cgroup/memory.stat ]; then
anon=$(awk '/^anon /{print $2}' /sys/fs/cgroup/memory.stat 2>/dev/null || echo "")
file=$(awk '/^file /{print $2}' /sys/fs/cgroup/memory.stat 2>/dev/null || echo "")
fi
echo "::group::Container memory (ersatztv#406 / server-management#604)"
printf 'peak (incl. page cache): %s MiB [%s bytes, %s]\n' "$(mib "$peak")" "$peak" "$src"
if [ -n "$anon" ]; then
printf 'end-of-job anon (the part that OOMs): %s MiB\n' "$(mib "$anon")"
printf 'end-of-job file (page cache, reclaimable): %s MiB\n' "$(mib "${file:-0}")"
echo 'NOTE: peak counts reclaimable page cache. Size caps on anon, not on peak.'
else
echo 'NOTE: no memory.stat breakdown available; peak includes reclaimable page cache.'
fi
echo "::endgroup::"
if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
{
printf '**Container memory (test job):** peak %s MiB *(incl. reclaimable page cache)*' \
"$(mib "$peak")"
[ -n "$anon" ] && printf ' · end-of-job anon %s MiB · file %s MiB' \
"$(mib "$anon")" "$(mib "${file:-0}")"
printf '\n'
} >> "$GITHUB_STEP_SUMMARY" || true
fi
run: scripts/ci-peak-anon.sh report
migrations:
name: EF migration integrity (SQLite + MySql)
+47 -21
View File
@@ -164,9 +164,16 @@ than a 7.8 GB VBCSCompiler) but it runs unattended on a cron against the prod me
the job server-management#570 measured pegging 5.999/6 GiB, so it is the one that most needs this.
Build-stage only; the final image is `FROM runtime-base`, so nothing lands in the shipped image.
The `test` job's **last** step, **Report peak container memory** (`continue-on-error`, never fails a
build), prints the cgroup's `memory.peak` plus an `anon`/`file` breakdown to the log and the job
step summary. Read it off a recent run instead of re-deriving it by hand — but read it correctly:
The `test` job samples its own memory and reports it every run. Two steps (`continue-on-error`, so
they never fail a build), driven by `scripts/ci-peak-anon.sh` (ersatztv#412):
- **Start peak-anon sampler** (before the dotnet Build/Test/Coverage steps) launches a detached
background poller that tracks the high-water mark of the cgroup's **anon** memory every 2 s.
- **Report peak container memory** (the job's last step) stops the sampler and prints the sampled
**peak anon** — the headline number — alongside `memory.peak` and the end-of-job `anon`/`file`
split, to the log and the job step summary.
Read the **peak anon** off a recent run to size a cap — not `memory.peak`, and here is why:
> ⚠️ **`memory.peak` is not "peak RSS".** It is the high-water mark of `memory.current`, which
> charges **page cache** to the cgroup as well as anonymous memory. Proven on bumblebee: a
@@ -175,27 +182,46 @@ step summary. Read it off a recent run instead of re-deriving it by hand — but
>
> This matters because the naive reading **inverts the decision**: page cache is *reclaimed* under a
> tighter cap, not OOM-killed, so a big peak that is mostly `file` is **not** evidence that the cap
> must stay high. `anon` is the part that actually forces an OOM. **Size caps on `anon`, not on
> `peak`.** The `anon`/`file` split is read at end-of-job, so it is indicative, not the composition
> at the peak instant — sampling a true peak-anon is **ersatztv#412**.
> must stay high. `anon` is the part that actually forces an OOM. **Size caps on peak `anon`, not on
> `peak`.**
>
> Why a *sampler* and not just the end-of-job split: the kernel exposes `memory.peak` (peak of
> anon+cache) but has **no peak-anon counter**, and the end-of-job `anon` is the composition *then*,
> not at the peak instant — a job that peaks mid-`dotnet test` and then frees reports a misleadingly
> low `anon`. The 2 s background sampler catches the true peak-anon instant; `memory.peak` and the
> end-of-job split stay in the report as a cache-inflated ceiling and a reference. (Before #412 the
> instrument printed only `memory.peak` + the end-of-job split — see #411.)
**First measurement (PR #411, `test` job):** `peak 8305 MiB`, cache share **unquantified** by that
run. Do not fill the gap with a guess in either direction — an independent probe (full solution
`dotnet build`, same CI image, shared compilation off) came back `peak 9457 MiB` / **`anon 7134
MiB`** / `file 421 MiB`, i.e. **anon dominated and cache did not**. On that evidence a 6g cap looks
*unsafe* rather than safe, and #570's "6g proved too tight" reads as the rule rather than an
outlier.
**The compiler-server A/B (ersatztv#412).** Measured on bumblebee in the CI toolchain image,
swap-off (`--memory-swap == --memory`), full-solution `dotnet build --no-incremental`, peak anon
sampled by *this* instrument, servers shut down between arms (n=2 each, interleaved):
That also means **#406's premise — *"if disabling shared compilation brings peak RSS well under 6
GiB, the whole budget loosens"* — is looking dead**: the 7134 MiB anon was measured *with* shared
compilation already off. Disabling the compiler servers remains right (no persistent 7.8 GB server
sits on the host between builds) but **do not bank a looser cap budget on it**.
| Arm | shared-compilation env | peak anon (2 runs) | resident after build |
|---|---|---|---|
| **OFF** (the CI config) | disabled | **5818 / 5854 MiB** (~5.84 GiB, tight) | none |
| **ON** (dotnet defaults) | enabled | **6305 / 7604 MiB** (~7.0 GiB, noisy) | ~3 GiB `VBCSCompiler` |
Treat all of the above as strong-but-not-final: it is one probe, not the `test` job, and there is
**no pre-change baseline from this instrument** (the 7.8 GB `VBCSCompiler` figure was measured
host-wide across concurrent jobs, not inside one job container). #412 covers the real A/B.
What *is* established: no persistent compiler server survives a build, and `migrations` is green
with mysql capped at 2g with swap off.
Two things are solid: OFF is **consistently ~5.84 GiB** and ON is **always higher** (mean delta
~1.1 GiB, up to ~1.8 GiB), so disabling the servers is worth it; and ON leaves a ~3 GiB
`VBCSCompiler` **resident** after the build — the host-between-jobs cost #406 removed. Don't read a
*precise* delta into the ON peak: it is noisy because a parallel build's peak depends on how many
`csc`/project compilations overlap at the peak instant.
**#406's premise — *"if disabling shared compilation brings peak RSS well under 6 GiB, the whole
budget loosens"* — is NOT supported.** OFF sits at ~5.84 GiB for the *build phase alone* — right at
the 6 GiB line, not well under it — and the `test` job adds `dotnet test` + coverlet +
reportgenerator on top. Disabling the compiler servers stays right (consistent reduction, no 3 GiB
resident server) but **do not bank a looser cap budget on it**: size the cap off the **live
test-job peak anon this instrument now reports** (build + test + coverage), not off this build-only
A/B.
> The earlier PR #411 probe (`peak 9457 / anon 7134 / file 421 MiB`) read *higher* than these
> swap-off sampled numbers. Swap settings and read-method (end-of-job snapshot vs sampled peak) move
> these figures by >1 GiB (#406), so treat the committed instrument's sampled peak-anon as
> authoritative and that probe as superseded.
What *is* established: no persistent compiler server survives a build, `migrations` is green with
mysql capped at 2g swap-off, and the `test` job now self-reports a true peak-anon every run.
### `services:` containers are capped explicitly (ersatztv#406)
+36
View File
@@ -101,6 +101,7 @@ in-file entries.
- [2026-07-18 — Never-scanned `LastScan` surfaces as null at the API boundary, not the 0001-01-01 MinValue sentinel (#409)](#2026-07-18--never-scanned-lastscan-surfaces-as-null-at-the-api-boundary-not-the-0001-01-01-minvalue-sentinel-409)
- [2026-07-19 — WeightedShuffle SPA: weights edited on the multi-collection, order offered only on classic MultiCollection schedule items; fair-share is a reset not a mode (#404)](#2026-07-19--weightedshuffle-spa-weights-edited-on-the-multi-collection-order-offered-only-on-classic-multicollection-schedule-items-fair-share-is-a-reset-not-a-mode-404)
- [2026-07-19 — Health-check results are TTL-cached; `?refresh=true` forces a fresh run (#431)](#2026-07-19--health-check-results-are-ttl-cached-refreshtrue-forces-a-fresh-run-431)
- [2026-07-19 — CI `test` job reports a sampled true peak-anon, not cache-inflated `memory.peak` (#412)](#2026-07-19--ci-test-job-reports-a-sampled-true-peak-anon-not-cache-inflated-memorypeak-412)
---
@@ -2045,3 +2046,38 @@ The blocking `format` CI job (and the matching Husky pre-commit hook) verify cha
- **Lane left on `ubuntu-latest`.** The job is now seconds-long and low-memory, so it could move to a
lighter lane, but that re-touches the per-lane memory-cap accounting (#406/#604) and is a
server-management capacity call — deliberately out of scope here.
## 2026-07-19 — CI `test` job reports a sampled true peak-anon, not cache-inflated `memory.peak` (#412)
**Decision.** The `test` job's memory instrument (added in #411) now reports a **sampled high-water
mark of the cgroup's `anon` memory** as the headline figure, produced by `scripts/ci-peak-anon.sh`
(a `start` step before the dotnet Build/Test/Coverage, a `report` step last). `memory.peak` and the
end-of-job `anon`/`file` split stay in the output as a cache-inflated ceiling and a reference.
**Why not just `memory.peak`.** `memory.peak` is the high-water mark of `memory.current`, which
charges reclaimable **page cache** to the cgroup alongside anon. A build does heavy
NuGet/npm/obj/bin/coverage I/O, so cache can dominate the peak — and page cache is *reclaimed* under
a tighter cap, not OOM-killed. Sizing a per-job cap (server-management#604) off `memory.peak`
therefore **inverts the decision**: a big, mostly-`file` peak reads like "the cap must stay high"
when it isn't. The OOM-forcing quantity is peak **anon**. The kernel exposes `memory.peak` but has
**no peak-anon counter**, and the end-of-job `anon` is the composition *then*, not at the peak
instant (a job that peaks mid-`dotnet test` then frees reports a misleadingly low `anon`) — so it
must be **sampled**. Details + the sampler's robustness rationale: `scripts/ci-peak-anon.sh` header
and `docs/ci-cd.md` → "CI build memory".
**Implementation note (do not "simplify" back to `memory.peak`).** The sampler is a detached
`nohup` poller that survives step-boundary re-execs (reparents to the container's PID 1) and is
reaped at container teardown; a TERM trap + `sleep & wait` stops it at once on `report`. Both steps
are `continue-on-error` with a fail-open script, so the instrument can never redden a green build.
Validated on bumblebee: it catches a transient 2.5 GiB anon spike that the end-of-job snapshot
reports as 0.
**Compiler-server A/B verdict (refines the #406 entry's "premise looking dead" read).** Measured
in the CI image, swap-off, sampled peak-anon, n=2 interleaved: **OFF (the CI config) ≈ 5.84 GiB,
consistent; ON (defaults) 6.3–7.6 GiB, always higher, + a ~3 GiB resident `VBCSCompiler`.**
Disabling the servers is worth it (consistent reduction, no resident server), but OFF sits *right at
6 GiB for the build phase alone* and the `test` job adds test + coverage on top — so #406's premise
("disabling brings peak *well under* 6 GiB → the budget loosens") is **not supported**. Size the cap
off the live test-job peak-anon this instrument now reports, not off the build-only A/B. The older
#411 probe (`anon 7134 MiB`) read higher than these swap-off sampled numbers and is superseded
(swap/read-method move the figure >1 GiB).
+114
View File
@@ -0,0 +1,114 @@
#!/usr/bin/env bash
# scripts/ci-peak-anon.sh — true peak-anon memory sampler for the CI `test` job (ersatztv#412).
#
# WHY. server-management#604 sizes the runners' per-job memory caps on what CI actually uses. The
# obvious number, cgroup v2 `memory.peak`, is the WRONG one: it is the high-water mark of
# `memory.current`, which charges reclaimable **page cache** to the cgroup alongside anonymous
# memory. A build job does heavy NuGet/npm/obj/bin/coverage I/O, so cache can dominate the peak —
# and page cache is *reclaimed* under a tighter cap, not OOM-killed. Sizing a cap off `memory.peak`
# therefore INVERTS the decision (ersatztv#406/#411): a big, mostly-`file` peak reads like "the cap
# must stay high" when it is nothing of the sort. The number that actually forces an OOM is peak
# **anon**. The kernel exposes `memory.peak` (peak of anon+cache) but has no peak-anon counter, so
# we must sample it.
#
# HOW. `start` launches a detached background poller that tracks the high-water mark of the cgroup's
# `anon` (from `memory.stat`) into a file; `report` (the job's last step) stops it and prints the
# sampled peak-anon alongside `memory.peak` (kept as the cache-inflated ceiling) and the end-of-job
# split (kept for reference — but note it is the composition *then*, not at the peak instant, which
# is exactly why the sampler exists: a job that peaks mid-`dotnet test` then frees reports a
# misleadingly low end-of-job anon).
#
# Robustness (validated on bumblebee, ersatztv#412):
# - The detached sampler survives step-boundary re-execs (reparents to the container's PID 1) and
# is reaped at container teardown, so it cannot outlive the job.
# - A TERM trap + `sleep & wait` makes `report`'s `kill` stop it at once (not after the sleep).
# - Fail-OPEN: if the cgroup files are unreadable it disables sampling and returns 0 — this runs
# `continue-on-error` in the workflow and must never redden a green build.
set -uo pipefail
OUT="${RUNNER_TEMP:-/tmp}/etv-peak-anon"
PIDF="${RUNNER_TEMP:-/tmp}/etv-peak-anon.pid"
STAT="${ETV_CGROUP_MEMSTAT:-/sys/fs/cgroup/memory.stat}" # ETV_CGROUP_MEMSTAT overrides for tests
PEAKFILES=("${ETV_CGROUP_MEMPEAK:-/sys/fs/cgroup/memory.peak}" /sys/fs/cgroup/memory/memory.max_usage_in_bytes)
mib() { echo "$(( ${1:-0} / 1048576 ))"; }
anon_bytes() { awk '/^anon /{print $2}' "$STAT" 2>/dev/null; }
# Internal: the sampler loop itself (run detached via `start`).
sample_loop() {
trap 'exit 0' TERM INT
local max=0 a
while :; do
a=$(anon_bytes)
# Atomic update (write-temp + rename) so a concurrent `report` read never catches a truncated
# $OUT mid-write and mistakes real data for "not sampled" (cold-review nit, ersatztv#412).
if [ -n "$a" ] && [ "$a" -gt "$max" ]; then max=$a; echo "$max" > "$OUT.tmp" && mv -f "$OUT.tmp" "$OUT"; fi
sleep "${ETV_PEAK_ANON_INTERVAL:-2}" & wait $!
done
}
start() {
if [ ! -r "$STAT" ] || [ -z "$(anon_bytes)" ]; then
echo "peak-anon: cgroup 'anon' not readable at $STAT -- sampling disabled (non-fatal)."
return 0
fi
echo 0 > "$OUT"
# Re-exec THIS script in sampler mode (absolute path, so a later CWD change can't strand it).
local self
self="$(cd "$(dirname "$0")" && pwd)/$(basename "$0")"
nohup "$self" __sample >/dev/null 2>&1 &
echo $! > "$PIDF"
echo "peak-anon: sampler started (pid $(cat "$PIDF"), ${ETV_PEAK_ANON_INTERVAL:-2}s interval)"
}
report() {
# Stop the sampler (best-effort; container teardown reaps it regardless).
[ -r "$PIDF" ] && kill "$(cat "$PIDF")" 2>/dev/null || true
local peakanon peak src f anon file
peakanon=$(cat "$OUT" 2>/dev/null || echo "")
peak=""; src=""
for f in "${PEAKFILES[@]}"; do
if [ -r "$f" ]; then peak=$(cat "$f" 2>/dev/null || echo ""); src="$f"; break; fi
done
anon=""; file=""
if [ -r "$STAT" ]; then
anon=$(anon_bytes)
file=$(awk '/^file /{print $2}' "$STAT" 2>/dev/null || echo "")
fi
echo "::group::Container memory (ersatztv#406/#412, server-management#604)"
if [ -n "$peakanon" ] && [ "$peakanon" != "0" ]; then
printf 'PEAK ANON: %s MiB [%s bytes, sampled] <- size caps on THIS (the part that forces an OOM)\n' \
"$(mib "$peakanon")" "$peakanon"
else
echo "PEAK ANON: not sampled (sampler disabled or no data) -- use peak + end-of-job split below."
fi
[ -n "$peak" ] && printf 'peak incl. page cache (cache-inflated ceiling, do NOT size on this): %s MiB [%s]\n' \
"$(mib "$peak")" "$src"
[ -n "$anon" ] && printf 'end-of-job anon: %s MiB end-of-job file (reclaimable cache): %s MiB\n' \
"$(mib "$anon")" "$(mib "${file:-0}")"
echo "::endgroup::"
if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
{
printf '**Container memory (test job):** '
if [ -n "$peakanon" ] && [ "$peakanon" != "0" ]; then
printf 'peak anon **%s MiB** *(size caps on this)*' "$(mib "$peakanon")"
else
printf 'peak anon *(not sampled)*'
fi
[ -n "$peak" ] && printf ' · peak incl. cache %s MiB *(inflated ceiling)*' "$(mib "$peak")"
[ -n "$anon" ] && printf ' · end-of-job anon %s MiB · file %s MiB' "$(mib "$anon")" "$(mib "${file:-0}")"
printf '\n'
} >> "$GITHUB_STEP_SUMMARY" || true
fi
}
case "${1:-}" in
start) start ;;
report) report ;;
__sample) sample_loop ;; # internal: detached sampler process
*) echo "usage: $0 {start|report}" >&2; exit 2 ;;
esac