ersatztv/scripts/ci-peak-anon.sh

#!/usr/bin/env bash
# scripts/ci-peak-anon.sh — true peak-anon memory sampler for the CI `test` job (ersatztv#412).
#
# WHY. server-management#604 sizes the runners' per-job memory caps on what CI actually uses. The
# obvious number, cgroup v2 `memory.peak`, is the WRONG one: it is the high-water mark of
# `memory.current`, which charges reclaimable **page cache** to the cgroup alongside anonymous
# memory. A build job does heavy NuGet/npm/obj/bin/coverage I/O, so cache can dominate the peak —
# and page cache is *reclaimed* under a tighter cap, not OOM-killed. Sizing a cap off `memory.peak`
# therefore INVERTS the decision (ersatztv#406/#411): a big, mostly-`file` peak reads like "the cap
# must stay high" when it is nothing of the sort. The number that actually forces an OOM is peak
# **anon**. The kernel exposes `memory.peak` (peak of anon+cache) but has no peak-anon counter, so
# we must sample it.
#
# HOW. `start` launches a detached background poller that tracks the high-water mark of the cgroup's
# `anon` (from `memory.stat`) into a file; `report` (the job's last step) stops it and prints the
# sampled peak-anon alongside `memory.peak` (kept as the cache-inflated ceiling) and the end-of-job
# split (kept for reference — but note it is the composition *then*, not at the peak instant, which
# is exactly why the sampler exists: a job that peaks mid-`dotnet test` then frees reports a
# misleadingly low end-of-job anon).
#
# Robustness (validated on bumblebee, ersatztv#412):
#   - The detached sampler survives step-boundary re-execs (reparents to the container's PID 1) and
#     is reaped at container teardown, so it cannot outlive the job.
#   - A TERM trap + `sleep & wait` makes `report`'s `kill` stop it at once (not after the sleep).
#   - Fail-OPEN: if the cgroup files are unreadable it disables sampling and returns 0 — this runs
#     `continue-on-error` in the workflow and must never redden a green build.
set -uo pipefail

OUT="${RUNNER_TEMP:-/tmp}/etv-peak-anon"
PIDF="${RUNNER_TEMP:-/tmp}/etv-peak-anon.pid"
STAT="${ETV_CGROUP_MEMSTAT:-/sys/fs/cgroup/memory.stat}"      # ETV_CGROUP_MEMSTAT overrides for tests
PEAKFILES=("${ETV_CGROUP_MEMPEAK:-/sys/fs/cgroup/memory.peak}" /sys/fs/cgroup/memory/memory.max_usage_in_bytes)

mib() { echo "$(( ${1:-0} / 1048576 ))"; }

anon_bytes() { awk '/^anon /{print $2}' "$STAT" 2>/dev/null; }

# Internal: the sampler loop itself (run detached via `start`).
sample_loop() {
  trap 'exit 0' TERM INT
  local max=0 a
  while :; do
    a=$(anon_bytes)
    # Atomic update (write-temp + rename) so a concurrent `report` read never catches a truncated
    # $OUT mid-write and mistakes real data for "not sampled" (cold-review nit, ersatztv#412).
    if [ -n "$a" ] && [ "$a" -gt "$max" ]; then max=$a; echo "$max" > "$OUT.tmp" && mv -f "$OUT.tmp" "$OUT"; fi
    sleep "${ETV_PEAK_ANON_INTERVAL:-2}" & wait $!
  done
}

start() {
  if [ ! -r "$STAT" ] || [ -z "$(anon_bytes)" ]; then
    echo "peak-anon: cgroup 'anon' not readable at $STAT -- sampling disabled (non-fatal)."
    return 0
  fi
  echo 0 > "$OUT"
  # Re-exec THIS script in sampler mode (absolute path, so a later CWD change can't strand it).
  local self
  self="$(cd "$(dirname "$0")" && pwd)/$(basename "$0")"
  nohup "$self" __sample >/dev/null 2>&1 &
  echo $! > "$PIDF"
  echo "peak-anon: sampler started (pid $(cat "$PIDF"), ${ETV_PEAK_ANON_INTERVAL:-2}s interval)"
}

report() {
  # Stop the sampler (best-effort; container teardown reaps it regardless).
  [ -r "$PIDF" ] && kill "$(cat "$PIDF")" 2>/dev/null || true

  local peakanon peak src f anon file
  peakanon=$(cat "$OUT" 2>/dev/null || echo "")
  peak=""; src=""
  for f in "${PEAKFILES[@]}"; do
    if [ -r "$f" ]; then peak=$(cat "$f" 2>/dev/null || echo ""); src="$f"; break; fi
  done
  anon=""; file=""
  if [ -r "$STAT" ]; then
    anon=$(anon_bytes)
    file=$(awk '/^file /{print $2}' "$STAT" 2>/dev/null || echo "")
  fi

  echo "::group::Container memory (ersatztv#406/#412, server-management#604)"
  if [ -n "$peakanon" ] && [ "$peakanon" != "0" ]; then
    printf 'PEAK ANON: %s MiB  [%s bytes, sampled] <- size caps on THIS (the part that forces an OOM)\n' \
      "$(mib "$peakanon")" "$peakanon"
  else
    echo "PEAK ANON: not sampled (sampler disabled or no data) -- use peak + end-of-job split below."
  fi
  [ -n "$peak" ] && printf 'peak incl. page cache (cache-inflated ceiling, do NOT size on this): %s MiB  [%s]\n' \
    "$(mib "$peak")" "$src"
  [ -n "$anon" ] && printf 'end-of-job anon: %s MiB   end-of-job file (reclaimable cache): %s MiB\n' \
    "$(mib "$anon")" "$(mib "${file:-0}")"
  echo "::endgroup::"

  if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
    {
      printf '**Container memory (test job):** '
      if [ -n "$peakanon" ] && [ "$peakanon" != "0" ]; then
        printf 'peak anon **%s MiB** *(size caps on this)*' "$(mib "$peakanon")"
      else
        printf 'peak anon *(not sampled)*'
      fi
      [ -n "$peak" ] && printf ' · peak incl. cache %s MiB *(inflated ceiling)*' "$(mib "$peak")"
      [ -n "$anon" ] && printf ' · end-of-job anon %s MiB · file %s MiB' "$(mib "$anon")" "$(mib "${file:-0}")"
      printf '\n'
    } >> "$GITHUB_STEP_SUMMARY" || true
  fi
}

case "${1:-}" in
  start)    start ;;
  report)   report ;;
  __sample) sample_loop ;;   # internal: detached sampler process
  *) echo "usage: $0 {start|report}" >&2; exit 2 ;;
esac