fix(539): WorkAheadSlots.Release never publishes a negative count #546

Merged
timothy merged 1 commits from fix/539-workaheadslots-hardening into main 2026-07-21 21:20:43 +02:00
5 changed files with 115 additions and 20 deletions
@@ -759,9 +759,15 @@ public class HlsSessionWorker : IHlsSessionWorker
// do nothing
}
if (ownsWorkAheadSlot)
if (ownsWorkAheadSlot && !_workAheadSlots.Release())
{
_workAheadSlots.Release();
// Release() reports false only when the pool was already empty, i.e. this slot was
// released more than once. Nothing reaches here in a correct program, but if a
// future second release site breaks the ownership contract this is the one in-band
// signal that the unthrottled-transcode budget is inflated (ersatztv#536/#539 §3).
_logger.LogWarning(
"Released a work-ahead slot that was not held for channel {Channel} - the unthrottled-transcode budget may be inflated",
_channelNumber);
}
}
@@ -53,38 +53,49 @@ public sealed class WorkAheadSlots
}
/// <summary>
/// Gets the number of releases that were not matched by a successful acquire. Always zero in a
/// correct program; a non-zero value means the ownership contract was broken somewhere.
/// Gets the number of releases that were not matched by a successful acquire. Non-zero always
/// means the ownership contract was broken somewhere (no false positives), so the value is a
/// reliable "something is wrong" signal — but it can UNDER-count and zero does not prove
/// correctness. It only increments when a release finds the pool already empty; an over-release
/// that happens while the count is positive — e.g. one cancelling out a coexisting leak —
/// decrements a real-looking slot and is never recorded, so the two bugs hide each other. This
/// is inherent to a single counter; exact accounting would need per-owner tokens (ersatztv#539 §2).
/// </summary>
public int UnbalancedReleases => Volatile.Read(ref _unbalancedReleases);
/// <summary>
/// Returns a slot claimed by <see cref="TryAcquire" />. Only ever called by the owner of that slot.
/// </summary>
/// <returns>
/// <c>true</c> when a held slot was returned; <c>false</c> when the pool was already empty, i.e.
/// the release was unbalanced (also counted in <see cref="UnbalancedReleases" />). Callers should
/// log the <c>false</c> case: it is the only in-band signal that the budget contract was broken.
/// </returns>
/// <remarks>
/// Ownership is a discipline, not a token — the same call-once contract as `EntityLocker` (#231).
/// The one failure this does defend against is an unbalanced release driving the count negative:
/// this pool is process-wide and lives for the life of the app, so a count of -1 would silently
/// and permanently admit one extra unthrottled transcode, re-opening the #529 QSV pool
/// exhaustion with nothing in the logs to find it by. Clamping keeps the resource guarantee
/// intact and records the breakage in <see cref="UnbalancedReleases" /> instead of throwing —
/// the single caller releases from a `finally`, where a throw would swallow the real exception.
/// The one failure this defends against is an unbalanced release inflating the budget: this pool
/// is process-wide and lives for the life of the app, so a leaked slot would silently and
/// permanently admit one extra unthrottled transcode, re-opening the #529 QSV pool exhaustion.
/// It clamps at zero rather than throwing — the single caller releases from a `finally`, where a
/// throw would swallow the real exception. Unlike a decrement-first-then-clamp shape, this never
/// publishes a negative count even transiently, so a concurrent <see cref="TryAcquire" /> can
/// never read the pool as having phantom room and over-admit (ersatztv#539 §1); and it records
/// the unbalanced release synchronously here, rather than blaming a later, innocent release.
/// </remarks>
public void Release()
public bool Release()
{
if (Interlocked.Decrement(ref _count) >= 0)
{
return;
}
Interlocked.Increment(ref _unbalancedReleases);
while (true)
{
int current = Volatile.Read(ref _count);
if (current >= 0 || Interlocked.CompareExchange(ref _count, 0, current) == current)
if (current <= 0)
{
return;
Interlocked.Increment(ref _unbalancedReleases);
return false;
}
if (Interlocked.CompareExchange(ref _count, current - 1, current) == current)
{
return true;
}
}
}
@@ -151,6 +151,69 @@ public class WorkAheadSlotsTests
slots.UnbalancedReleases.ShouldBe(0);
}
// §1 (ersatztv#539): an unbalanced Release() must never publish a NEGATIVE count, even
// transiently. The pre-#539 shape decremented FIRST (0 -> -1) and clamped afterwards, so a
// concurrent TryAcquire(limit) could read the -1, see "-1 < limit" as phantom room, and admit a
// holder the budget doesn't have (the second acquirer then reads 0 and admits another). Because
// TryAcquire is the sole, CAS-guarded increment path, a count that is provably never negative is
// exactly what forecloses that over-admit. Here one thread hammers unbalanced releases on an
// empty pool while readers sample the count; none may ever observe a value below zero.
//
// NEGATIVE CONTROL (per the class-level ersatztv#231/#250 lesson) — revert Release() to its
// pre-#539 decrement-first body to prove this test is non-vacuous:
//
// if (Interlocked.Decrement(ref _count) >= 0) return true;
// Interlocked.Increment(ref _unbalancedReleases);
// while (true) { int c = Volatile.Read(ref _count);
// if (c >= 0 || Interlocked.CompareExchange(ref _count, 0, c) == c) return false; }
//
// — and this test must FAIL (sawNegative > 0): the readers catch the transient -1. Do NOT stub
// `if (true)`: that leaves values assigned-but-never-read, and CS0219 under warnings-as-errors
// fails the build silently so `dotnet test --no-build` runs the STALE dll and the control falsely
// passes. Always grep the build output for `error CS` first.
[Test]
public void Release_Unbalanced_NeverPublishesNegativeCount()
{
const int releases = 2_000_000;
const int readers = 4;
var slots = new WorkAheadSlots();
var sawNegative = 0;
var done = false;
var readerThreads = new Thread[readers];
for (var i = 0; i < readers; i++)
{
readerThreads[i] = new Thread(() =>
{
while (!Volatile.Read(ref done))
{
if (slots.Count < 0)
{
Interlocked.Increment(ref sawNegative);
}
}
});
readerThreads[i].Start();
}
// every release finds the pool empty, so every one is unbalanced
for (var r = 0; r < releases; r++)
{
slots.Release();
}
Volatile.Write(ref done, true);
foreach (Thread reader in readerThreads)
{
reader.Join();
}
sawNegative.ShouldBe(0);
slots.Count.ShouldBe(0);
slots.UnbalancedReleases.ShouldBe(releases);
}
// The release path is the fiddly half: a slot freed by its owner must become available again,
// and a burst of acquire/release cycles must not drift the count in either direction.
[Test]
+14
View File
@@ -3413,3 +3413,17 @@ not a failure, because the common reason for one is that another session is legi
tool would have disabled itself after one use. It uses `npm ci`, which installs strictly from the
lockfile and never writes it. And it asserts the tree is clean at exit, reporting loudly if not:
leaving the shared tree dirty is the one outcome that would make this script a net negative.
## 2026-07-21 — `WorkAheadSlots.Release()` clamps before decrementing and reports unbalance in-band (#539)
`key: ffmpeg.work-ahead-slot-release-never-negative` · `status: active` · `since: 2026-07-21` · `supersedes: none` · `superseded-by: none`
**Rule:** `Release()` reads the count and compare-exchanges `current - 1` only when `current > 0`; a release against an empty pool records an unbalanced release and returns `false` **without ever writing a negative value**. It never decrements first and clamps afterward. The single caller (`HlsSessionWorker.Transcode`'s `finally`) logs a warning on the `false` return.
**Signals:** work-ahead slot release, unbalanced release, negative slot count, over-admit at limit 1, phantom work-ahead room, UnbalancedReleases counter · paths: `ErsatzTV.Application/Streaming/WorkAheadSlots.cs`, `HlsSessionWorker.Transcode`, `ErsatzTV.Core.Tests/Streaming/WorkAheadSlotsTests.cs` · issues: #539, #536, #529, #231, #250
**Mechanics:** `bool Release()` CAS loop guarded by `current <= 0`; call site `if (ownsWorkAheadSlot && !_workAheadSlots.Release()) _logger.LogWarning(...)`
**Why never-negative, given the release is already unreachable-when-unbalanced.** Today `HlsSessionWorker` has exactly one release site, guarded by `ownsWorkAheadSlot`, so an unbalanced release cannot happen — these were three **Low** findings from the #536 re-review, filed against the day someone adds a second release site. The pre-#539 shape decremented first (`0 → −1`) and clamped afterward, leaving a reachable interleaving where a concurrent `TryAcquire(limit)` reads the `−1`, sees phantom room, and admits a holder the budget doesn't have (a second acquirer then reads `0` and admits another) — two unthrottled transcodes at limit 1, re-opening the #529 QSV pool exhaustion. Because `TryAcquire` is the sole, CAS-guarded increment path, a count proven never-negative is exactly what forecloses that over-admit. Clamping *before* the decrement also records the breakage **synchronously on the offending thread**, instead of the decrement-first shape where the blame lands on a later, innocent release.
**`Release()` returns `bool` so the breach is visible in the logs, not only to tests.** #536's motivation was "a silently inflated budget with nothing in the logs to find it by," yet `UnbalancedReleases` was reachable only from a debugger or a test. `WorkAheadSlots` has no logger by design (it is a leaf primitive), so rather than plumb one in, `Release()` returns `false` on an unbalanced release and the caller — which already holds an `ILogger` — logs the warning. This is the one in-band signal that the ownership contract broke.
**`UnbalancedReleases` can under-count, and that is documented rather than fixed.** It only increments when a release finds the pool already empty. An over-release while the count is positive — e.g. one cancelling out a coexisting leak — decrements a real-looking slot and is never recorded, so the two bugs hide each other. There are no false positives (non-zero still means the contract broke), but zero does not prove correctness. Exact accounting would need per-owner tokens, which the #536 "ownership is a discipline, not a token" decision deliberately avoids; the docstring now states the limitation instead.
**Negative control (inherited from #231/#250).** A dedicated test hammers unbalanced releases on an empty pool while reader threads sample the count; none may ever observe a value below zero. Reinstating the pre-#539 decrement-first body makes it fail (`sawNegative > 0` — the readers catch the transient `−1`); verified. As with the #536 tests, break the primitive by reverting the real body, **not** `if (true)` (CS0219 under warnings-as-errors leaves `--no-build` running a stale, still-fixed dll).
+1
View File
@@ -64,6 +64,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
| `ffmpeg.qsv-extra-hw-frames-floor` | a QSV upload never emits `extra_hw_frames` below `FFmpegState.MinimumQsvExtraHardwareFrames` (64); a stored `0` or negative value is treated as "no pool configured" rather than honored literally, because with no headroom any unthrottled read exhausts the pool and the transcode writes nothing at all. | 2026-07-21 | [link](../decisions.md#2026-07-21--qsv-hardware-frame-headroom-is-a-floor-not-an-operator-preference-529) |
| `ffmpeg.remote-image-fetcher-bounded` | remote graphics-engine images are fetched through `IRemoteImageFetcher` with a pooled `HttpClientFactory` client, a body-covering deadline, a wire-transfer size cap, and a decoder-enforced `DecoderOptions.MaxFrames` bound re-verified post-decode — never cached, re-fetched per element init. | 2026-07-20 | [link](../decisions.md#2026-07-20--remote-graphics-engine-images-are-fetched-through-a-bounded-pooled-iremoteimagefetcher-re-fetched-per-element-init-not-cached-511) |
| `ffmpeg.work-ahead-slot-atomic` | `workAheadSegmenterLimit` is enforced by a single compare-exchange claim on a shared `WorkAheadSlots` pool taken by the *caller* of `Transcode`, which then passes ownership in and gets the release in `Transcode`'s `finally` — never a `Volatile.Read` compare in one place and an `Interlocked.Increment` in another. | 2026-07-21 | [link](../decisions.md#2026-07-21--work-ahead-slots-are-claimed-atomically-by-the-caller-released-by-the-transcode-it-hands-them-to-536) |
| `ffmpeg.work-ahead-slot-release-never-negative` | `Release()` reads the count and compare-exchanges `current - 1` only when `current > 0`; a release against an empty pool records an unbalanced release and returns `false` **without ever writing a negative value**. It never decrements first and clamps afterward. The single caller (`HlsSessionWorker.Transcode`'s `finally`) logs a warning on the `false` return. | 2026-07-21 | [link](../decisions.md#2026-07-21--workaheadslotsrelease-clamps-before-decrementing-and-reports-unbalance-in-band-539) |
| `graphics.channel-logo-caching` | An external `http(s)` channel-logo URL is fetched, decode-budget-validated, and stored in the image cache under a content-hash name at SAVE time — becoming byte-identical to an uploaded logo — so the render path never fetches a logo over HTTP; a bad URL fails the save with a 422 (BaseError → ValidationProblemDetails). | 2026-07-21 | [link](../decisions.md#2026-07-21--external-channel-logo-urls-are-downloaded-and-cached-at-save-time-the-render-path-never-fetches-a-logo-525) |
| `iptv.base-url` | An optional advertised base URL (`iptv.base_url`) is resolved centrally via a pure Core helper (`AdvertisedBaseUrl`) inside the two IPTV generation handlers (M3U + XMLTV); unset/malformed values fall back byte-identical to the request-derived host, and it's a new `iptv` settings group distinct from `ETV_BASE_URL` and out of scope for HDHomeRun. | 2026-07-16 | [link](../decisions.md#2026-07-16--optional-advertised-iptv-base-url-iptvbase_url-resolved-centrally-in-the-two-generators-340) |
| `iptv.logo-drives-bug-preset` | One uploaded channel logo drives both the listing logo and the on-screen bug via a shared, seeded `ChannelLogo`-sourced watermark preset (`Channel Bug`), not new per-channel schema. | 2026-07-20 | [link](../decisions.md#2026-07-20--one-logo-drives-the-bug-via-a-shared-channellogo-preset-not-new-schema-67) |