diff --git a/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs b/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs index 53f4114e9..cb47e9cee 100644 --- a/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs +++ b/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs @@ -593,7 +593,97 @@ public class PipelineBuilderBaseTests command.ShouldNotContain("-readrate_initial_burst"); } - private string BuildRealtimeCommand(IFFmpegCapabilities capabilities, bool stillImage = false) + [Test] + public void Realtime_Input_Should_Catch_Up_When_Option_Is_Supported() + { + string command = BuildRealtimeCommand(new CatchupCapableFFmpegCapabilities()); + + // -readrate paces an input off its furthest-behind stream, so a sparse stream sharing the + // input pins throughput below realtime; catchup lets it recover (ersatztv#726). anchor on + // the input path so this can't be satisfied by some other input carrying the option + // this overlaps Bitmap_Subtitle_Burn_In_... by design: that one pins the #726 MECHANISM on a + // bitmap pipeline, this one pins the plain no-subtitle shape plus the uniqueness guard below + command.ShouldContain("-readrate 1.05 -readrate_initial_burst 8 -readrate_catchup 6.0 -i /tmp/whatever.mkv"); + Regex.Matches(command, Regex.Escape("-readrate_catchup 6.0")).Count.ShouldBe(1); + } + + [Test] + public void Realtime_Input_Should_Not_Catch_Up_A_Still_Image() + { + // mirrors the burst's still-image exclusion (ersatztv#350): the video input takes no readrate + // at all, so catchup would only reach the separate audio input and run it ahead of a graph + // that the realtime filter is already pacing. pinned so the divergence can't reappear silently + string command = BuildRealtimeCommand(new CatchupCapableFFmpegCapabilities(), stillImage: true); + + // the positive anchor keeps this from passing vacuously if the helper ever stops + // producing a realtime audio input at all + command.ShouldContain("-readrate 1.05"); + command.ShouldNotContain("-readrate_catchup"); + } + + [Test] + public void Realtime_Input_Should_Not_Catch_Up_When_Option_Is_Unsupported() + { + // an older binary silently keeps today's behavior rather than failing to start + string command = BuildRealtimeCommand(new BurstCapableFFmpegCapabilities()); + + // the positive anchor keeps this from passing vacuously if the helper ever stops + // producing a realtime input at all + command.ShouldContain("-readrate 1.05"); + command.ShouldNotContain("-readrate_catchup"); + } + + [Test] + public void Concat_Should_Never_Catch_Up() + { + // concat reads already-written segments from the running segmenter at a flat 1.0; it has no + // sparse stream to lag on, and letting it catch up would gallop through the segments + var concatInputFile = new ConcatInputFile("http://localhost:8080/ffmpeg/concat/1", new FrameSize(1920, 1080)); + + var builder = new SoftwarePipelineBuilder( + new CatchupCapableFFmpegCapabilities(), + HardwareAccelerationMode.None, + None, + None, + None, + None, + concatInputFile, + Option.None, + "", + "", + _logger); + + FFmpegPipeline result = builder.Concat(concatInputFile, FFmpegState.Concat(false, "Some Channel")); + + string command = PrintCommand(None, None, None, concatInputFile, None, result); + + command.ShouldContain("-readrate 1.0"); + command.ShouldNotContain("-readrate_catchup"); + } + + [Test] + public void Bitmap_Subtitle_Burn_In_Should_Catch_Up_On_The_Shared_Video_Input() + { + // THE #726 regression test. an embedded bitmap subtitle is read through the SAME -i as the + // video (SubtitleInputFile carries the video's path and resolves to a stream specifier on + // that input), and being sparse it drags that input's pacing down to ~0.53x realtime. + // this must be built on a BITMAP subtitle: a text subtitle is fetched by the libass filter + // outside the demuxer, so the same assertions would pass vacuously while the bug is present. + string command = BuildRealtimeCommand(new CatchupCapableFFmpegCapabilities(), imageSubtitle: true); + + // the mechanism itself: subtitle stream 2 resolves onto input 0 -- the VIDEO's input -- so it + // is read through the throttled demuxer that catchup is being applied to. if the subtitle + // ever moves to an input of its own this label changes and the test fails, which is the point + command.ShouldContain("[0:0][0:2]overlay"); + + // ...so the catchup has to be on that input + command.ShouldContain("-readrate 1.05 -readrate_initial_burst 8 -readrate_catchup 6.0 -i /tmp/whatever.mkv"); + } + + private string BuildRealtimeCommand( + IFFmpegCapabilities capabilities, + bool stillImage = false, + bool imageSubtitle = false) { var videoInputFile = new VideoInputFile( "/tmp/whatever.mkv", @@ -676,13 +766,22 @@ public class PipelineBuilderBaseTests AudioFilter.None, Option.None)); + // an embedded bitmap subtitle carries the VIDEO's path, which is how it ends up sharing the + // video's single throttled -i rather than getting one of its own (ersatztv#726) + Option subtitleInputFile = imageSubtitle + ? new SubtitleInputFile( + "/tmp/whatever.mkv", + new List { new(2, "dvdsub", StreamKind.Subtitle) }, + SubtitleMethod.Burn) + : Option.None; + var builder = new SoftwarePipelineBuilder( capabilities, HardwareAccelerationMode.None, videoInputFile, audioInputFile, None, - None, + subtitleInputFile, None, Option.None, "", @@ -735,4 +834,19 @@ public class PipelineBuilderBaseTests new System.Collections.Generic.HashSet(), new System.Collections.Generic.HashSet { FFmpegKnownOption.ReadrateInitialBurst.Name }, new System.Collections.Generic.HashSet()); + + // a binary new enough for -readrate_catchup also has -readrate_initial_burst, so this models a + // real ffmpeg rather than an impossible catchup-without-burst one + public class CatchupCapableFFmpegCapabilities() : FFmpegCapabilities( + string.Empty, + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet + { + FFmpegKnownOption.ReadrateInitialBurst.Name, + FFmpegKnownOption.ReadrateCatchup.Name + }, + new System.Collections.Generic.HashSet()); } diff --git a/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownOption.cs b/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownOption.cs index c4a66cb89..101a94c74 100644 --- a/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownOption.cs +++ b/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownOption.cs @@ -13,8 +13,15 @@ public record FFmpegKnownOption // ffmpeg 6.1+; lets a readrate-throttled input read flat out for an initial window public static FFmpegKnownOption ReadrateInitialBurst => new("readrate_initial_burst"); + // ffmpeg 8.0+ (added 2025-02-15 in 6232f416b, first released in 8.0); lets a readrate-throttled + // input read faster than its readrate *while it is behind*, so a sparse stream sharing that + // input cannot pin throughput below realtime (ersatztv#726). verified present in 8.1.2, the + // pinned base image — note this is NEWER than 7.1, so it is detected at runtime, never assumed + public static FFmpegKnownOption ReadrateCatchup => new("readrate_catchup"); + public static IList AllOptions => [ - ReadrateInitialBurst.Name + ReadrateInitialBurst.Name, + ReadrateCatchup.Name ]; } diff --git a/ErsatzTV.FFmpeg/InputOption/ReadrateInputOption.cs b/ErsatzTV.FFmpeg/InputOption/ReadrateInputOption.cs index a7f4e9163..801d79c99 100644 --- a/ErsatzTV.FFmpeg/InputOption/ReadrateInputOption.cs +++ b/ErsatzTV.FFmpeg/InputOption/ReadrateInputOption.cs @@ -3,10 +3,11 @@ using ErsatzTV.FFmpeg.Environment; namespace ErsatzTV.FFmpeg.InputOption; -public class ReadrateInputOption(double readRate, Option initialBurstSeconds) : IInputOption +public class ReadrateInputOption(double readRate, Option initialBurstSeconds, Option catchupReadRate) + : IInputOption { public ReadrateInputOption(double readRate) - : this(readRate, Option.None) + : this(readRate, Option.None, Option.None) { } @@ -30,6 +31,17 @@ public class ReadrateInputOption(double readRate, Option initialBurstSecond result.Add(burst.ToString(CultureInfo.InvariantCulture)); } + // -readrate paces the WHOLE input off its furthest-behind stream, so one sparse stream + // (an embedded PGS/DVD bitmap subtitle feeding the overlay) drags the video down with it + // and output collapses to ~0.53x realtime. catchup lets a lagging input read faster until + // it is level again; it is a ceiling that only applies WHILE behind, never a target, so + // caught-up input still paces at readRate and cannot race ahead (ersatztv#726) + foreach (double catchup in catchupReadRate) + { + result.Add("-readrate_catchup"); + result.Add(catchup.ToString("0.0####", CultureInfo.InvariantCulture)); + } + return result.ToArray(); } diff --git a/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs b/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs index 76c9e60a2..6faf837ee 100644 --- a/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs +++ b/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs @@ -22,6 +22,14 @@ public abstract class PipelineBuilderBase : IPipelineBuilder // an operator who raises that setting above 2 gets less of the benefit (ersatztv#350) private const int InitialBurstSeconds = OutputFormatHls.SegmentSeconds * 2; + // how fast a LAGGING realtime input may read until it is level again. measured on the #726 + // repro (embedded dvd_subtitle -> overlay, QSV encode): 1.05 alone sustains 0.53x, catchup 2.0 + // reaches 0.711x, and 6.0 restores the full 1.067x that the same pipeline achieves with no + // subtitle at all. 20.0 also measures 1.067x — i.e. the value is not a throughput dial above + // the point where the input catches up, so 6.0 is chosen as the smallest measured-sufficient + // ceiling rather than the largest that works (ersatztv#726) + private const double CatchupReadRate = 6.0; + private readonly Option _audioInputFile; private readonly Option _concatInputFile; private readonly IFFmpegCapabilities _ffmpegCapabilities; @@ -871,8 +879,26 @@ public abstract class PipelineBuilderBase : IPipelineBuilder ? InitialBurstSeconds : Option.None; - _audioInputFile.Iter(a => a.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds))); - videoInputFile.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds)); + // -readrate paces an input off its furthest-behind stream. an embedded bitmap subtitle is + // read through the SAME -i as the video (its SubtitleInputFile carries the video's path and + // resolves to a stream specifier on that input), and being sparse it falls further behind + // every second, dragging video throughput to ~0.53x — well under the 1.0x a live client + // consumes at. catchup lets the lagging input recover instead of pinning the whole process. + // applied to every realtime input, not just subtitle pipelines: it is inert unless an input + // is actually behind, and any sparse stream can cause this (ersatztv#726). + // + // a still image is excluded for the SAME reason the burst above excludes it: its video input + // takes no readrate at all, so this would reach only the separate audio input and let it run + // ahead of the video, which is exactly what #350 declined. for a non-still-image item both + // inputs carry identical options, so the symmetry is preserved there. and an image-based + // subtitle always rides the video path, so this shape cannot suffer the starvation anyway + Option catchupReadRate = + !isStillImage && _ffmpegCapabilities.HasOption(FFmpegKnownOption.ReadrateCatchup) + ? CatchupReadRate + : Option.None; + + _audioInputFile.Iter(a => a.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds, catchupReadRate))); + videoInputFile.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds, catchupReadRate)); } protected static void SetStillImageLoop( diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 8e4dbbb1d..f1934f784 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -83,6 +83,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera | `ffmpeg.qsv-decode-encode-split` | QSV decode is decoupled from QSV encode via a single `FFmpegProfile.QsvPreferNativeDecoder` bool (default ON, Linux-only), so a QSV encode profile can decode with the more tolerant native VA-API decoder instead of the QSV decoder, mirroring Jellyfin's hybrid decode/encode toggle instead of a general decode-family enum. | 2026-07-20 | [link](records/ffmpeg/qsv-decode-encode-split.md) | | `ffmpeg.qsv-extra-hw-frames-floor` | a QSV upload never emits `extra_hw_frames` below `FFmpegState.MinimumQsvExtraHardwareFrames` (64); a stored `0` or negative value is treated as "no pool configured" rather than honored literally, because with no headroom any unthrottled read exhausts the pool and the transcode writes nothing at all. | 2026-07-21 | [link](records/ffmpeg/qsv-extra-hw-frames-floor.md) | | `ffmpeg.qsv-hdr-tonemap-opencl` | the QSV pipeline never emits `vpp_qsv=tonemap=1`, which is a SILENT no-op on pre-Gen11 Intel graphics; HDR is tonemapped on the GPU via `hwupload=derive_device=vaapi` → `scale_vaapi` → `hwmap=derive_device=opencl` → `tonemap_opencl` when a VA-API device exists, the frames are still in software, and `tonemap_opencl` is available, and by the software `TonemapFilter` otherwise. The scale runs BEFORE the tonemap, and any hardware filter on the path forces the output to be re-tagged bt709. | 2026-07-26 | [link](records/ffmpeg/qsv-hdr-tonemap-opencl.md) | +| `ffmpeg.readrate-catchup-sparse-streams` | a realtime video/audio input also gets `-readrate_catchup` (6.0) when the binary supports it — but NOT a still-image input (mirroring the #350 exclusion) and NOT a concat input, which keep at most bare `-readrate` (a still image's video input takes none at all). Reason: `-readrate` paces the whole input off its furthest-behind stream, so a sparse stream sharing that input (an embedded PGS/DVD bitmap subtitle feeding the overlay) otherwise pins output at ~0.53x realtime. Catchup is a ceiling that applies only WHILE an input is behind, never a target, so it does not let a caught-up input race ahead. | 2026-08-04 | [link](records/ffmpeg/readrate-catchup-sparse-streams.md) | | `ffmpeg.remote-image-fetcher-bounded` | remote graphics-engine images are fetched through `IRemoteImageFetcher` with a pooled `HttpClientFactory` client, a body-covering deadline, a wire-transfer size cap, and a decoder-enforced `DecoderOptions.MaxFrames` bound re-verified post-decode — never cached, re-fetched per element init. | 2026-07-20 | [link](records/ffmpeg/remote-image-fetcher-bounded.md) | | `ffmpeg.watermark-resolution-unified` | Every watermark `WatermarkSelector` resolves goes through one shared `ResolveWatermark` — the playout-item, channel and global precedence levels AND the deco path, for all three `ChannelWatermarkImageSource` values. An unresolvable watermark (missing file, un-migrated external URL, or no logo artwork) resolves to no on-screen bug plus a warning, never a dead path or a URL handed downstream; the one deliberate exception is a playout-item `Custom` with a blank image, which still falls THROUGH to channel/global. The generated-initials fallback is therefore off everywhere, including the deco path where it demonstrably rendered. Watermarks built OUTSIDE the selector (the song-progress overlay, #653) are not covered and remain unchecked. | 2026-07-26 | [link](records/ffmpeg/watermark-resolution-unified.md) | | `ffmpeg.work-ahead-slot-atomic` | `workAheadSegmenterLimit` is enforced by a single compare-exchange claim on a shared `WorkAheadSlots` pool taken by the *caller* of `Transcode`, which then passes ownership in and gets the release in `Transcode`'s `finally` — never a `Volatile.Read` compare in one place and an `Interlocked.Increment` in another. | 2026-07-21 | [link](records/ffmpeg/work-ahead-slot-atomic.md) | diff --git a/docs/decisions/records/ffmpeg/hls-cold-start-burst.md b/docs/decisions/records/ffmpeg/hls-cold-start-burst.md index f870c9165..dfbd984c5 100644 --- a/docs/decisions/records/ffmpeg/hls-cold-start-burst.md +++ b/docs/decisions/records/ffmpeg/hls-cold-start-burst.md @@ -58,7 +58,8 @@ mechanics: '`SetRealtimeInput` readrate-burst option; `FFmpegKnownOption.HasOpti session" flag through `FFmpegState`; that complexity was not judged worth a bounded peak. - **Still images are excluded.** Their video input is paced by the realtime *filter* and takes no readrate at all, so a burst would only run the audio input ahead of the video for songs and offline - filler, with no cold-start gain to show for it. + filler, with no cold-start gain to show for it. (`-readrate_catchup` mirrors this exclusion for the + same reason — `ffmpeg.readrate-catchup-sparse-streams`, #726.) - **Non-HLS realtime outputs (`TransportStream`, HLS-Direct) burst too**, since `FFmpegPlaybackSettingsCalculator` makes them unconditionally realtime. That is untested by the benchmark, which was segmenter-only; it is kept because the same first-read throttle delays those diff --git a/docs/decisions/records/ffmpeg/readrate-catchup-sparse-streams.md b/docs/decisions/records/ffmpeg/readrate-catchup-sparse-streams.md new file mode 100644 index 000000000..53a09e870 --- /dev/null +++ b/docs/decisions/records/ffmpeg/readrate-catchup-sparse-streams.md @@ -0,0 +1,71 @@ +--- +key: ffmpeg.readrate-catchup-sparse-streams +title: 2026-08-04 — a realtime input gets `-readrate_catchup`, because `-readrate` paces off its furthest-behind stream (#726) +status: active +since: '2026-08-04' +supersedes: none +superseded-by: none +rule: 'a realtime video/audio input also gets `-readrate_catchup` (6.0) when the binary supports it — but NOT a still-image input (mirroring the #350 exclusion) and NOT a concat input, which keep at most bare `-readrate` (a still image''s video input takes none at all). Reason: `-readrate` paces the whole input off its furthest-behind stream, so a sparse stream sharing that input (an embedded PGS/DVD bitmap subtitle feeding the overlay) otherwise pins output at ~0.53x realtime. Catchup is a ceiling that applies only WHILE an input is behind, never a target, so it does not let a caught-up input race ahead.' +signals: 'readrate, readrate_catchup, sparse stream, bitmap subtitle, PGS, DVD subtitle, dvdsub, pgssub, overlay burn-in, Live TV buffering/stalling, "Resumed reading at pts N with rate R after a lag of Ns" · paths: `PipelineBuilderBase.SetRealtimeInput`, `ReadrateInputOption`, `FFmpegKnownOption` · issues: #726, #350, #529' +mechanics: '`PipelineBuilderBase.CatchupReadRate` (6.0); `ReadrateInputOption` catchup arg; `FFmpegKnownOption.ReadrateCatchup` capability gate' +--- + +- **`-readrate` throttles an input, not a stream, and it paces off whichever stream is furthest + behind.** An embedded bitmap subtitle is read through the *same* `-i` as the video — + `SubtitleInputFile` carries the video's path and `ComplexFilter` resolves it to a stream specifier + on that input, and `CommandGenerator` never emits a second `-i` for it. Being sparse, the subtitle + stream falls further behind every second and drags the video down with it. FFmpeg says so itself at + `-loglevel warning`: `[sist#0:3/dvd_subtitle] Resumed reading at pts 10.400 with rate 6.000 after a + lag of 0.922s`, repeating with the lag growing 0.9→3.8 s while `pts` stays pinned (no new packet). +- **Measured on prod (QSV, `-threads 1`, `dvd_subtitle`→overlay), 45 s steady-state window after a + 6 s settle:** + + | variant | throughput | + |---|---| + | `-readrate 1.05` (baseline) | **0.533x** (×3 runs) | + | `+ -readrate_catchup 2.0` | 0.711x | + | `+ -readrate_catchup 6.0` | **1.067x** (×2 runs) | + | `+ -readrate_catchup 20.0` | 1.067x | + | no subtitle overlay (control) | 1.067x | + + A live client consumes at 1.0x, so 0.53x drains its buffer until it stalls — the reported symptom. +- **`20.0` measuring the same as `6.0` is why 6.0 was chosen** — above the catch-up point the value + is not a throughput dial, so there is nothing to buy by going higher. It is **not** evidence about + allocation: that is a steady-state throughput number, not a count of frames in flight. +- **Why this does not reopen `ffmpeg.qsv-extra-hw-frames-floor` (#529).** Not because catchup is + brief (a permanently GPU-bound channel lags forever, so 6x is a standing licence), and **not** + because read rate is allocation-irrelevant — #529 measured that it is not (at `extra_hw_frames=0`, + `1.05` without a burst exits 0 while `1.05`+burst hits ENOMEM). Read rate changes how fast frames + enter the graph, not how deep its queues are, and #529 showed that only bites when the pool has + **no headroom**. The 64-frame floor now guarantees headroom, so the load-bearing measurement is + row 5 of that truth table — **no `-readrate` at all with 64 frames → 14 segments, exit 0** — and a + 6x ceiling is strictly less aggressive than no throttle. Reinforcing it, `-readrate_initial_burst 8` + has read *flat out* at the start of every playout item since #350, so an unbounded read here is not + new. A 240 s QSV soak (64 frames, 60 segment boundaries) adds 1.043x sustained with **zero** + `Cannot allocate memory` — but it stayed largely caught-up, so it corroborates rather than proves; + the argument above is what carries the decision. +- **Not QSV-specific:** reproduces on libx264 too (0.533x → 1.067x), as expected for an input-pacing + option upstream of any encoder or filter choice. +- **Raising the base `-readrate` is not an alternative, and was measured:** 2.0→0.62x, 3.0→0.80x, + 4.0→0.80x, 6.0→0.89x. It asymptotes *below* realtime, because the rate ceiling was never the + binding constraint. Recorded so it is not re-proposed. +- **Catchup does NOT subsume the #350 burst; they fix orthogonal metrics.** Measured + time-to-first-segment: `-readrate` alone 3.71 s, `+burst` **0.72 s**, `+catchup` alone **3.65 s**, + both 0.67 s. Catchup buys nothing at cold start (no accumulated lag at t=0 to recover) and the + burst buys nothing for throughput (the 0.533x baseline already had it), so removing the burst on + the theory that catchup replaces it would regress tune-in ~5x. +- **Applied to realtime video/audio inputs generally, not only subtitle pipelines** — it is inert + unless an input is behind, and any sparse stream can trigger this, so gating it on "has a bitmap + subtitle" would fix the site instead of the boundary. Two deliberate exclusions, both test-pinned: + `ConcatInputFile` (reads already-written segments at a flat 1.0, nothing sparse to lag on) and + **still images**, mirroring #350 — their video input takes no readrate at all, so catchup would + reach only the separate audio input and break the pacing symmetry #350 declined to break. An + image-based subtitle always rides the *video* path, so that shape cannot starve this way anyway. +- **Capability-gated via `FFmpegKnownOption.HasOption`**, the same fail-safe posture as + `-readrate_initial_burst`: detection parses `ffmpeg -h long`, so a binary without the option + silently keeps today's behavior instead of failing to start. + +**Accepted residual:** the affected population is items carrying an embedded bitmap subtitle matching +the channel's subtitle mode — 3,182 of 24,646 media versions (12.9%) on prod. It is a property of the +*item*, not the channel, which is why the stall presented as random: a channel plays one episode fine +and stalls on the next.