diff --git a/ErsatzTV.Application/Streaming/HlsSessionWorker.cs b/ErsatzTV.Application/Streaming/HlsSessionWorker.cs index c5acce6d4..f172aad99 100644 --- a/ErsatzTV.Application/Streaming/HlsSessionWorker.cs +++ b/ErsatzTV.Application/Streaming/HlsSessionWorker.cs @@ -759,9 +759,15 @@ public class HlsSessionWorker : IHlsSessionWorker // do nothing } - if (ownsWorkAheadSlot) + if (ownsWorkAheadSlot && !_workAheadSlots.Release()) { - _workAheadSlots.Release(); + // Release() reports false only when the pool was already empty, i.e. this slot was + // released more than once. Nothing reaches here in a correct program, but if a + // future second release site breaks the ownership contract this is the one in-band + // signal that the unthrottled-transcode budget is inflated (ersatztv#536/#539 §3). + _logger.LogWarning( + "Released a work-ahead slot that was not held for channel {Channel} - the unthrottled-transcode budget may be inflated", + _channelNumber); } } diff --git a/ErsatzTV.Application/Streaming/WorkAheadSlots.cs b/ErsatzTV.Application/Streaming/WorkAheadSlots.cs index cbddb355b..e35d2db9b 100644 --- a/ErsatzTV.Application/Streaming/WorkAheadSlots.cs +++ b/ErsatzTV.Application/Streaming/WorkAheadSlots.cs @@ -53,38 +53,49 @@ public sealed class WorkAheadSlots } /// - /// Gets the number of releases that were not matched by a successful acquire. Always zero in a - /// correct program; a non-zero value means the ownership contract was broken somewhere. + /// Gets the number of releases that were not matched by a successful acquire. Non-zero always + /// means the ownership contract was broken somewhere (no false positives), so the value is a + /// reliable "something is wrong" signal — but it can UNDER-count and zero does not prove + /// correctness. It only increments when a release finds the pool already empty; an over-release + /// that happens while the count is positive — e.g. one cancelling out a coexisting leak — + /// decrements a real-looking slot and is never recorded, so the two bugs hide each other. This + /// is inherent to a single counter; exact accounting would need per-owner tokens (ersatztv#539 §2). /// public int UnbalancedReleases => Volatile.Read(ref _unbalancedReleases); /// /// Returns a slot claimed by . Only ever called by the owner of that slot. /// + /// + /// true when a held slot was returned; false when the pool was already empty, i.e. + /// the release was unbalanced (also counted in ). Callers should + /// log the false case: it is the only in-band signal that the budget contract was broken. + /// /// /// Ownership is a discipline, not a token — the same call-once contract as `EntityLocker` (#231). - /// The one failure this does defend against is an unbalanced release driving the count negative: - /// this pool is process-wide and lives for the life of the app, so a count of -1 would silently - /// and permanently admit one extra unthrottled transcode, re-opening the #529 QSV pool - /// exhaustion with nothing in the logs to find it by. Clamping keeps the resource guarantee - /// intact and records the breakage in instead of throwing — - /// the single caller releases from a `finally`, where a throw would swallow the real exception. + /// The one failure this defends against is an unbalanced release inflating the budget: this pool + /// is process-wide and lives for the life of the app, so a leaked slot would silently and + /// permanently admit one extra unthrottled transcode, re-opening the #529 QSV pool exhaustion. + /// It clamps at zero rather than throwing — the single caller releases from a `finally`, where a + /// throw would swallow the real exception. Unlike a decrement-first-then-clamp shape, this never + /// publishes a negative count even transiently, so a concurrent can + /// never read the pool as having phantom room and over-admit (ersatztv#539 §1); and it records + /// the unbalanced release synchronously here, rather than blaming a later, innocent release. /// - public void Release() + public bool Release() { - if (Interlocked.Decrement(ref _count) >= 0) - { - return; - } - - Interlocked.Increment(ref _unbalancedReleases); - while (true) { int current = Volatile.Read(ref _count); - if (current >= 0 || Interlocked.CompareExchange(ref _count, 0, current) == current) + if (current <= 0) { - return; + Interlocked.Increment(ref _unbalancedReleases); + return false; + } + + if (Interlocked.CompareExchange(ref _count, current - 1, current) == current) + { + return true; } } } diff --git a/ErsatzTV.Core.Tests/Streaming/WorkAheadSlotsTests.cs b/ErsatzTV.Core.Tests/Streaming/WorkAheadSlotsTests.cs index eda97f1d6..880f388c3 100644 --- a/ErsatzTV.Core.Tests/Streaming/WorkAheadSlotsTests.cs +++ b/ErsatzTV.Core.Tests/Streaming/WorkAheadSlotsTests.cs @@ -151,6 +151,69 @@ public class WorkAheadSlotsTests slots.UnbalancedReleases.ShouldBe(0); } + // §1 (ersatztv#539): an unbalanced Release() must never publish a NEGATIVE count, even + // transiently. The pre-#539 shape decremented FIRST (0 -> -1) and clamped afterwards, so a + // concurrent TryAcquire(limit) could read the -1, see "-1 < limit" as phantom room, and admit a + // holder the budget doesn't have (the second acquirer then reads 0 and admits another). Because + // TryAcquire is the sole, CAS-guarded increment path, a count that is provably never negative is + // exactly what forecloses that over-admit. Here one thread hammers unbalanced releases on an + // empty pool while readers sample the count; none may ever observe a value below zero. + // + // NEGATIVE CONTROL (per the class-level ersatztv#231/#250 lesson) — revert Release() to its + // pre-#539 decrement-first body to prove this test is non-vacuous: + // + // if (Interlocked.Decrement(ref _count) >= 0) return true; + // Interlocked.Increment(ref _unbalancedReleases); + // while (true) { int c = Volatile.Read(ref _count); + // if (c >= 0 || Interlocked.CompareExchange(ref _count, 0, c) == c) return false; } + // + // — and this test must FAIL (sawNegative > 0): the readers catch the transient -1. Do NOT stub + // `if (true)`: that leaves values assigned-but-never-read, and CS0219 under warnings-as-errors + // fails the build silently so `dotnet test --no-build` runs the STALE dll and the control falsely + // passes. Always grep the build output for `error CS` first. + [Test] + public void Release_Unbalanced_NeverPublishesNegativeCount() + { + const int releases = 2_000_000; + const int readers = 4; + + var slots = new WorkAheadSlots(); + var sawNegative = 0; + var done = false; + + var readerThreads = new Thread[readers]; + for (var i = 0; i < readers; i++) + { + readerThreads[i] = new Thread(() => + { + while (!Volatile.Read(ref done)) + { + if (slots.Count < 0) + { + Interlocked.Increment(ref sawNegative); + } + } + }); + readerThreads[i].Start(); + } + + // every release finds the pool empty, so every one is unbalanced + for (var r = 0; r < releases; r++) + { + slots.Release(); + } + + Volatile.Write(ref done, true); + foreach (Thread reader in readerThreads) + { + reader.Join(); + } + + sawNegative.ShouldBe(0); + slots.Count.ShouldBe(0); + slots.UnbalancedReleases.ShouldBe(releases); + } + // The release path is the fiddly half: a slot freed by its owner must become available again, // and a burst of acquire/release cycles must not drift the count in either direction. [Test] diff --git a/docs/decisions.md b/docs/decisions.md index c10974276..e41012497 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -3413,3 +3413,17 @@ not a failure, because the common reason for one is that another session is legi tool would have disabled itself after one use. It uses `npm ci`, which installs strictly from the lockfile and never writes it. And it asserts the tree is clean at exit, reporting loudly if not: leaving the shared tree dirty is the one outcome that would make this script a net negative. +## 2026-07-21 — `WorkAheadSlots.Release()` clamps before decrementing and reports unbalance in-band (#539) + +`key: ffmpeg.work-ahead-slot-release-never-negative` · `status: active` · `since: 2026-07-21` · `supersedes: none` · `superseded-by: none` +**Rule:** `Release()` reads the count and compare-exchanges `current - 1` only when `current > 0`; a release against an empty pool records an unbalanced release and returns `false` **without ever writing a negative value**. It never decrements first and clamps afterward. The single caller (`HlsSessionWorker.Transcode`'s `finally`) logs a warning on the `false` return. +**Signals:** work-ahead slot release, unbalanced release, negative slot count, over-admit at limit 1, phantom work-ahead room, UnbalancedReleases counter · paths: `ErsatzTV.Application/Streaming/WorkAheadSlots.cs`, `HlsSessionWorker.Transcode`, `ErsatzTV.Core.Tests/Streaming/WorkAheadSlotsTests.cs` · issues: #539, #536, #529, #231, #250 +**Mechanics:** `bool Release()` CAS loop guarded by `current <= 0`; call site `if (ownsWorkAheadSlot && !_workAheadSlots.Release()) _logger.LogWarning(...)` + +**Why never-negative, given the release is already unreachable-when-unbalanced.** Today `HlsSessionWorker` has exactly one release site, guarded by `ownsWorkAheadSlot`, so an unbalanced release cannot happen — these were three **Low** findings from the #536 re-review, filed against the day someone adds a second release site. The pre-#539 shape decremented first (`0 → −1`) and clamped afterward, leaving a reachable interleaving where a concurrent `TryAcquire(limit)` reads the `−1`, sees phantom room, and admits a holder the budget doesn't have (a second acquirer then reads `0` and admits another) — two unthrottled transcodes at limit 1, re-opening the #529 QSV pool exhaustion. Because `TryAcquire` is the sole, CAS-guarded increment path, a count proven never-negative is exactly what forecloses that over-admit. Clamping *before* the decrement also records the breakage **synchronously on the offending thread**, instead of the decrement-first shape where the blame lands on a later, innocent release. + +**`Release()` returns `bool` so the breach is visible in the logs, not only to tests.** #536's motivation was "a silently inflated budget with nothing in the logs to find it by," yet `UnbalancedReleases` was reachable only from a debugger or a test. `WorkAheadSlots` has no logger by design (it is a leaf primitive), so rather than plumb one in, `Release()` returns `false` on an unbalanced release and the caller — which already holds an `ILogger` — logs the warning. This is the one in-band signal that the ownership contract broke. + +**`UnbalancedReleases` can under-count, and that is documented rather than fixed.** It only increments when a release finds the pool already empty. An over-release while the count is positive — e.g. one cancelling out a coexisting leak — decrements a real-looking slot and is never recorded, so the two bugs hide each other. There are no false positives (non-zero still means the contract broke), but zero does not prove correctness. Exact accounting would need per-owner tokens, which the #536 "ownership is a discipline, not a token" decision deliberately avoids; the docstring now states the limitation instead. + +**Negative control (inherited from #231/#250).** A dedicated test hammers unbalanced releases on an empty pool while reader threads sample the count; none may ever observe a value below zero. Reinstating the pre-#539 decrement-first body makes it fail (`sawNegative > 0` — the readers catch the transient `−1`); verified. As with the #536 tests, break the primitive by reverting the real body, **not** `if (true)` (CS0219 under warnings-as-errors leaves `--no-build` running a stale, still-fixed dll). diff --git a/docs/decisions/README.md b/docs/decisions/README.md index a9bba2e48..1abb8ce3a 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -64,6 +64,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera | `ffmpeg.qsv-extra-hw-frames-floor` | a QSV upload never emits `extra_hw_frames` below `FFmpegState.MinimumQsvExtraHardwareFrames` (64); a stored `0` or negative value is treated as "no pool configured" rather than honored literally, because with no headroom any unthrottled read exhausts the pool and the transcode writes nothing at all. | 2026-07-21 | [link](../decisions.md#2026-07-21--qsv-hardware-frame-headroom-is-a-floor-not-an-operator-preference-529) | | `ffmpeg.remote-image-fetcher-bounded` | remote graphics-engine images are fetched through `IRemoteImageFetcher` with a pooled `HttpClientFactory` client, a body-covering deadline, a wire-transfer size cap, and a decoder-enforced `DecoderOptions.MaxFrames` bound re-verified post-decode — never cached, re-fetched per element init. | 2026-07-20 | [link](../decisions.md#2026-07-20--remote-graphics-engine-images-are-fetched-through-a-bounded-pooled-iremoteimagefetcher-re-fetched-per-element-init-not-cached-511) | | `ffmpeg.work-ahead-slot-atomic` | `workAheadSegmenterLimit` is enforced by a single compare-exchange claim on a shared `WorkAheadSlots` pool taken by the *caller* of `Transcode`, which then passes ownership in and gets the release in `Transcode`'s `finally` — never a `Volatile.Read` compare in one place and an `Interlocked.Increment` in another. | 2026-07-21 | [link](../decisions.md#2026-07-21--work-ahead-slots-are-claimed-atomically-by-the-caller-released-by-the-transcode-it-hands-them-to-536) | +| `ffmpeg.work-ahead-slot-release-never-negative` | `Release()` reads the count and compare-exchanges `current - 1` only when `current > 0`; a release against an empty pool records an unbalanced release and returns `false` **without ever writing a negative value**. It never decrements first and clamps afterward. The single caller (`HlsSessionWorker.Transcode`'s `finally`) logs a warning on the `false` return. | 2026-07-21 | [link](../decisions.md#2026-07-21--workaheadslotsrelease-clamps-before-decrementing-and-reports-unbalance-in-band-539) | | `graphics.channel-logo-caching` | An external `http(s)` channel-logo URL is fetched, decode-budget-validated, and stored in the image cache under a content-hash name at SAVE time — becoming byte-identical to an uploaded logo — so the render path never fetches a logo over HTTP; a bad URL fails the save with a 422 (BaseError → ValidationProblemDetails). | 2026-07-21 | [link](../decisions.md#2026-07-21--external-channel-logo-urls-are-downloaded-and-cached-at-save-time-the-render-path-never-fetches-a-logo-525) | | `iptv.base-url` | An optional advertised base URL (`iptv.base_url`) is resolved centrally via a pure Core helper (`AdvertisedBaseUrl`) inside the two IPTV generation handlers (M3U + XMLTV); unset/malformed values fall back byte-identical to the request-derived host, and it's a new `iptv` settings group distinct from `ETV_BASE_URL` and out of scope for HDHomeRun. | 2026-07-16 | [link](../decisions.md#2026-07-16--optional-advertised-iptv-base-url-iptvbase_url-resolved-centrally-in-the-two-generators-340) | | `iptv.logo-drives-bug-preset` | One uploaded channel logo drives both the listing logo and the on-screen bug via a shared, seeded `ChannelLogo`-sourced watermark preset (`Channel Bug`), not new per-channel schema. | 2026-07-20 | [link](../decisions.md#2026-07-20--one-logo-drives-the-bug-via-a-shared-channellogo-preset-not-new-schema-67) |