ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs

@@ -593,7 +593,97 @@ public class PipelineBuilderBaseTests
         command.ShouldNotContain("-readrate_initial_burst");
     }
 
-    private string BuildRealtimeCommand(IFFmpegCapabilities capabilities, bool stillImage = false)
+    [Test]
+    public void Realtime_Input_Should_Catch_Up_When_Option_Is_Supported()
+    {
+        string command = BuildRealtimeCommand(new CatchupCapableFFmpegCapabilities());
+
+        // -readrate paces an input off its furthest-behind stream, so a sparse stream sharing the
+        // input pins throughput below realtime; catchup lets it recover (ersatztv#726). anchor on
+        // the input path so this can't be satisfied by some other input carrying the option
+        // this overlaps Bitmap_Subtitle_Burn_In_... by design: that one pins the #726 MECHANISM on a
+        // bitmap pipeline, this one pins the plain no-subtitle shape plus the uniqueness guard below
+        command.ShouldContain("-readrate 1.05 -readrate_initial_burst 8 -readrate_catchup 6.0 -i /tmp/whatever.mkv");
+        Regex.Matches(command, Regex.Escape("-readrate_catchup 6.0")).Count.ShouldBe(1);
+    }
+
+    [Test]
+    public void Realtime_Input_Should_Not_Catch_Up_A_Still_Image()
+    {
+        // mirrors the burst's still-image exclusion (ersatztv#350): the video input takes no readrate
+        // at all, so catchup would only reach the separate audio input and run it ahead of a graph
+        // that the realtime filter is already pacing. pinned so the divergence can't reappear silently
+        string command = BuildRealtimeCommand(new CatchupCapableFFmpegCapabilities(), stillImage: true);
+
+        // the positive anchor keeps this from passing vacuously if the helper ever stops
+        // producing a realtime audio input at all
+        command.ShouldContain("-readrate 1.05");
+        command.ShouldNotContain("-readrate_catchup");
+    }
+
+    [Test]
+    public void Realtime_Input_Should_Not_Catch_Up_When_Option_Is_Unsupported()
+    {
+        // an older binary silently keeps today's behavior rather than failing to start
+        string command = BuildRealtimeCommand(new BurstCapableFFmpegCapabilities());
+
+        // the positive anchor keeps this from passing vacuously if the helper ever stops
+        // producing a realtime input at all
+        command.ShouldContain("-readrate 1.05");
+        command.ShouldNotContain("-readrate_catchup");
+    }
+
+    [Test]
+    public void Concat_Should_Never_Catch_Up()
+    {
+        // concat reads already-written segments from the running segmenter at a flat 1.0; it has no
+        // sparse stream to lag on, and letting it catch up would gallop through the segments
+        var concatInputFile = new ConcatInputFile("http://localhost:8080/ffmpeg/concat/1", new FrameSize(1920, 1080));
+
+        var builder = new SoftwarePipelineBuilder(
+            new CatchupCapableFFmpegCapabilities(),
+            HardwareAccelerationMode.None,
+            None,
+            None,
+            None,
+            None,
+            concatInputFile,
+            Option<GraphicsEngineInput>.None,
+            "",
+            "",
+            _logger);
+
+        FFmpegPipeline result = builder.Concat(concatInputFile, FFmpegState.Concat(false, "Some Channel"));
+
+        string command = PrintCommand(None, None, None, concatInputFile, None, result);
+
+        command.ShouldContain("-readrate 1.0");
+        command.ShouldNotContain("-readrate_catchup");
+    }
+
+    [Test]
+    public void Bitmap_Subtitle_Burn_In_Should_Catch_Up_On_The_Shared_Video_Input()
+    {
+        // THE #726 regression test. an embedded bitmap subtitle is read through the SAME -i as the
+        // video (SubtitleInputFile carries the video's path and resolves to a stream specifier on
+        // that input), and being sparse it drags that input's pacing down to ~0.53x realtime.
+        // this must be built on a BITMAP subtitle: a text subtitle is fetched by the libass filter
+        // outside the demuxer, so the same assertions would pass vacuously while the bug is present.
+        string command = BuildRealtimeCommand(new CatchupCapableFFmpegCapabilities(), imageSubtitle: true);
+
+        // the mechanism itself: subtitle stream 2 resolves onto input 0 -- the VIDEO's input -- so it
+        // is read through the throttled demuxer that catchup is being applied to. if the subtitle
+        // ever moves to an input of its own this label changes and the test fails, which is the point
+        command.ShouldContain("[0:0][0:2]overlay");
+
+        // ...so the catchup has to be on that input
+        command.ShouldContain("-readrate 1.05 -readrate_initial_burst 8 -readrate_catchup 6.0 -i /tmp/whatever.mkv");
+    }
+
+    private string BuildRealtimeCommand(
+        IFFmpegCapabilities capabilities,
+        bool stillImage = false,
+        bool imageSubtitle = false)
     {
         var videoInputFile = new VideoInputFile(
             "/tmp/whatever.mkv",
@@ -676,13 +766,22 @@ public class PipelineBuilderBaseTests
                 AudioFilter.None,
                 Option<double>.None));
 
+        // an embedded bitmap subtitle carries the VIDEO's path, which is how it ends up sharing the
+        // video's single throttled -i rather than getting one of its own (ersatztv#726)
+        Option<SubtitleInputFile> subtitleInputFile = imageSubtitle
+            ? new SubtitleInputFile(
+                "/tmp/whatever.mkv",
+                new List<MediaStream> { new(2, "dvdsub", StreamKind.Subtitle) },
+                SubtitleMethod.Burn)
+            : Option<SubtitleInputFile>.None;
+
         var builder = new SoftwarePipelineBuilder(
             capabilities,
             HardwareAccelerationMode.None,
             videoInputFile,
             audioInputFile,
             None,
-            None,
+            subtitleInputFile,
             None,
             Option<GraphicsEngineInput>.None,
             "",
@@ -735,4 +834,19 @@ public class PipelineBuilderBaseTests
         new System.Collections.Generic.HashSet<string>(),
         new System.Collections.Generic.HashSet<string> { FFmpegKnownOption.ReadrateInitialBurst.Name },
         new System.Collections.Generic.HashSet<string>());
+
+    // a binary new enough for -readrate_catchup also has -readrate_initial_burst, so this models a
+    // real ffmpeg rather than an impossible catchup-without-burst one
+    public class CatchupCapableFFmpegCapabilities() : FFmpegCapabilities(
+        string.Empty,
+        new System.Collections.Generic.HashSet<string>(),
+        new System.Collections.Generic.HashSet<string>(),
+        new System.Collections.Generic.HashSet<string>(),
+        new System.Collections.Generic.HashSet<string>(),
+        new System.Collections.Generic.HashSet<string>
+        {
+            FFmpegKnownOption.ReadrateInitialBurst.Name,
+            FFmpegKnownOption.ReadrateCatchup.Name
+        },
+        new System.Collections.Generic.HashSet<string>());
 }

ErsatzTV.FFmpeg/Capabilities/FFmpegKnownOption.cs

@@ -13,8 +13,15 @@ public record FFmpegKnownOption
     // ffmpeg 6.1+; lets a readrate-throttled input read flat out for an initial window
     public static FFmpegKnownOption ReadrateInitialBurst => new("readrate_initial_burst");
 
+    // ffmpeg 8.0+ (added 2025-02-15 in 6232f416b, first released in 8.0); lets a readrate-throttled
+    // input read faster than its readrate *while it is behind*, so a sparse stream sharing that
+    // input cannot pin throughput below realtime (ersatztv#726). verified present in 8.1.2, the
+    // pinned base image — note this is NEWER than 7.1, so it is detected at runtime, never assumed
+    public static FFmpegKnownOption ReadrateCatchup => new("readrate_catchup");
+
     public static IList<string> AllOptions =>
     [
-        ReadrateInitialBurst.Name
+        ReadrateInitialBurst.Name,
+        ReadrateCatchup.Name
     ];
 }

ErsatzTV.FFmpeg/InputOption/ReadrateInputOption.cs

@@ -3,10 +3,11 @@ using ErsatzTV.FFmpeg.Environment;
 
 namespace ErsatzTV.FFmpeg.InputOption;
 
-public class ReadrateInputOption(double readRate, Option<int> initialBurstSeconds) : IInputOption
+public class ReadrateInputOption(double readRate, Option<int> initialBurstSeconds, Option<double> catchupReadRate)
+    : IInputOption
 {
     public ReadrateInputOption(double readRate)
-        : this(readRate, Option<int>.None)
+        : this(readRate, Option<int>.None, Option<double>.None)
     {
     }
 
@@ -30,6 +31,17 @@ public class ReadrateInputOption(double readRate, Option<int> initialBurstSecond
             result.Add(burst.ToString(CultureInfo.InvariantCulture));
         }
 
+        // -readrate paces the WHOLE input off its furthest-behind stream, so one sparse stream
+        // (an embedded PGS/DVD bitmap subtitle feeding the overlay) drags the video down with it
+        // and output collapses to ~0.53x realtime. catchup lets a lagging input read faster until
+        // it is level again; it is a ceiling that only applies WHILE behind, never a target, so
+        // caught-up input still paces at readRate and cannot race ahead (ersatztv#726)
+        foreach (double catchup in catchupReadRate)
+        {
+            result.Add("-readrate_catchup");
+            result.Add(catchup.ToString("0.0####", CultureInfo.InvariantCulture));
+        }
+
         return result.ToArray();
     }
 

ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs

@@ -22,6 +22,14 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
     // an operator who raises that setting above 2 gets less of the benefit (ersatztv#350)
     private const int InitialBurstSeconds = OutputFormatHls.SegmentSeconds * 2;
 
+    // how fast a LAGGING realtime input may read until it is level again. measured on the #726
+    // repro (embedded dvd_subtitle -> overlay, QSV encode): 1.05 alone sustains 0.53x, catchup 2.0
+    // reaches 0.711x, and 6.0 restores the full 1.067x that the same pipeline achieves with no
+    // subtitle at all. 20.0 also measures 1.067x — i.e. the value is not a throughput dial above
+    // the point where the input catches up, so 6.0 is chosen as the smallest measured-sufficient
+    // ceiling rather than the largest that works (ersatztv#726)
+    private const double CatchupReadRate = 6.0;
+
     private readonly Option<AudioInputFile> _audioInputFile;
     private readonly Option<ConcatInputFile> _concatInputFile;
     private readonly IFFmpegCapabilities _ffmpegCapabilities;
@@ -871,8 +879,26 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
                 ? InitialBurstSeconds
                 : Option<int>.None;
 
-        _audioInputFile.Iter(a => a.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds)));
-        videoInputFile.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds));
+        // -readrate paces an input off its furthest-behind stream. an embedded bitmap subtitle is
+        // read through the SAME -i as the video (its SubtitleInputFile carries the video's path and
+        // resolves to a stream specifier on that input), and being sparse it falls further behind
+        // every second, dragging video throughput to ~0.53x — well under the 1.0x a live client
+        // consumes at. catchup lets the lagging input recover instead of pinning the whole process.
+        // applied to every realtime input, not just subtitle pipelines: it is inert unless an input
+        // is actually behind, and any sparse stream can cause this (ersatztv#726).
+        //
+        // a still image is excluded for the SAME reason the burst above excludes it: its video input
+        // takes no readrate at all, so this would reach only the separate audio input and let it run
+        // ahead of the video, which is exactly what #350 declined. for a non-still-image item both
+        // inputs carry identical options, so the symmetry is preserved there. and an image-based
+        // subtitle always rides the video path, so this shape cannot suffer the starvation anyway
+        Option<double> catchupReadRate =
+            !isStillImage && _ffmpegCapabilities.HasOption(FFmpegKnownOption.ReadrateCatchup)
+                ? CatchupReadRate
+                : Option<double>.None;
+
+        _audioInputFile.Iter(a => a.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds, catchupReadRate)));
+        videoInputFile.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds, catchupReadRate));
     }
 
     protected static void SetStillImageLoop(

docs/decisions/README.md

@@ -83,6 +83,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera
 | `ffmpeg.qsv-decode-encode-split` | QSV decode is decoupled from QSV encode via a single `FFmpegProfile.QsvPreferNativeDecoder` bool (default ON, Linux-only), so a QSV encode profile can decode with the more tolerant native VA-API decoder instead of the QSV decoder, mirroring Jellyfin's hybrid decode/encode toggle instead of a general decode-family enum. | 2026-07-20 | [link](records/ffmpeg/qsv-decode-encode-split.md) |
 | `ffmpeg.qsv-extra-hw-frames-floor` | a QSV upload never emits `extra_hw_frames` below `FFmpegState.MinimumQsvExtraHardwareFrames` (64); a stored `0` or negative value is treated as "no pool configured" rather than honored literally, because with no headroom any unthrottled read exhausts the pool and the transcode writes nothing at all. | 2026-07-21 | [link](records/ffmpeg/qsv-extra-hw-frames-floor.md) |
 | `ffmpeg.qsv-hdr-tonemap-opencl` | the QSV pipeline never emits `vpp_qsv=tonemap=1`, which is a SILENT no-op on pre-Gen11 Intel graphics; HDR is tonemapped on the GPU via `hwupload=derive_device=vaapi` → `scale_vaapi` → `hwmap=derive_device=opencl` → `tonemap_opencl` when a VA-API device exists, the frames are still in software, and `tonemap_opencl` is available, and by the software `TonemapFilter` otherwise. The scale runs BEFORE the tonemap, and any hardware filter on the path forces the output to be re-tagged bt709. | 2026-07-26 | [link](records/ffmpeg/qsv-hdr-tonemap-opencl.md) |
+| `ffmpeg.readrate-catchup-sparse-streams` | a realtime video/audio input also gets `-readrate_catchup` (6.0) when the binary supports it — but NOT a still-image input (mirroring the #350 exclusion) and NOT a concat input, which keep at most bare `-readrate` (a still image's video input takes none at all). Reason: `-readrate` paces the whole input off its furthest-behind stream, so a sparse stream sharing that input (an embedded PGS/DVD bitmap subtitle feeding the overlay) otherwise pins output at ~0.53x realtime. Catchup is a ceiling that applies only WHILE an input is behind, never a target, so it does not let a caught-up input race ahead. | 2026-08-04 | [link](records/ffmpeg/readrate-catchup-sparse-streams.md) |
 | `ffmpeg.remote-image-fetcher-bounded` | remote graphics-engine images are fetched through `IRemoteImageFetcher` with a pooled `HttpClientFactory` client, a body-covering deadline, a wire-transfer size cap, and a decoder-enforced `DecoderOptions.MaxFrames` bound re-verified post-decode — never cached, re-fetched per element init. | 2026-07-20 | [link](records/ffmpeg/remote-image-fetcher-bounded.md) |
 | `ffmpeg.watermark-resolution-unified` | Every watermark `WatermarkSelector` resolves goes through one shared `ResolveWatermark` — the playout-item, channel and global precedence levels AND the deco path, for all three `ChannelWatermarkImageSource` values. An unresolvable watermark (missing file, un-migrated external URL, or no logo artwork) resolves to no on-screen bug plus a warning, never a dead path or a URL handed downstream; the one deliberate exception is a playout-item `Custom` with a blank image, which still falls THROUGH to channel/global. The generated-initials fallback is therefore off everywhere, including the deco path where it demonstrably rendered. Watermarks built OUTSIDE the selector (the song-progress overlay, #653) are not covered and remain unchecked. | 2026-07-26 | [link](records/ffmpeg/watermark-resolution-unified.md) |
 | `ffmpeg.work-ahead-slot-atomic` | `workAheadSegmenterLimit` is enforced by a single compare-exchange claim on a shared `WorkAheadSlots` pool taken by the *caller* of `Transcode`, which then passes ownership in and gets the release in `Transcode`'s `finally` — never a `Volatile.Read` compare in one place and an `Interlocked.Increment` in another. | 2026-07-21 | [link](records/ffmpeg/work-ahead-slot-atomic.md) |

docs/decisions/records/ffmpeg/hls-cold-start-burst.md

@@ -58,7 +58,8 @@ mechanics: '`SetRealtimeInput` readrate-burst option; `FFmpegKnownOption.HasOpti
   session" flag through `FFmpegState`; that complexity was not judged worth a bounded peak.
 - **Still images are excluded.** Their video input is paced by the realtime *filter* and takes no
   readrate at all, so a burst would only run the audio input ahead of the video for songs and offline
-  filler, with no cold-start gain to show for it.
+  filler, with no cold-start gain to show for it. (`-readrate_catchup` mirrors this exclusion for the
+  same reason — `ffmpeg.readrate-catchup-sparse-streams`, #726.)
 - **Non-HLS realtime outputs (`TransportStream`, HLS-Direct) burst too**, since
   `FFmpegPlaybackSettingsCalculator` makes them unconditionally realtime. That is untested by the
   benchmark, which was segmenter-only; it is kept because the same first-read throttle delays those

docs/decisions/records/ffmpeg/readrate-catchup-sparse-streams.md

@@ -0,0 +1,71 @@
+---
+key: ffmpeg.readrate-catchup-sparse-streams
+title: 2026-08-04 — a realtime input gets `-readrate_catchup`, because `-readrate` paces off its furthest-behind stream (#726)
+status: active
+since: '2026-08-04'
+supersedes: none
+superseded-by: none
+rule: 'a realtime video/audio input also gets `-readrate_catchup` (6.0) when the binary supports it — but NOT a still-image input (mirroring the #350 exclusion) and NOT a concat input, which keep at most bare `-readrate` (a still image''s video input takes none at all). Reason: `-readrate` paces the whole input off its furthest-behind stream, so a sparse stream sharing that input (an embedded PGS/DVD bitmap subtitle feeding the overlay) otherwise pins output at ~0.53x realtime. Catchup is a ceiling that applies only WHILE an input is behind, never a target, so it does not let a caught-up input race ahead.'
+signals: 'readrate, readrate_catchup, sparse stream, bitmap subtitle, PGS, DVD subtitle, dvdsub, pgssub, overlay burn-in, Live TV buffering/stalling, "Resumed reading at pts N with rate R after a lag of Ns" · paths: `PipelineBuilderBase.SetRealtimeInput`, `ReadrateInputOption`, `FFmpegKnownOption` · issues: #726, #350, #529'
+mechanics: '`PipelineBuilderBase.CatchupReadRate` (6.0); `ReadrateInputOption` catchup arg; `FFmpegKnownOption.ReadrateCatchup` capability gate'
+---
+
+- **`-readrate` throttles an input, not a stream, and it paces off whichever stream is furthest
+  behind.** An embedded bitmap subtitle is read through the *same* `-i` as the video —
+  `SubtitleInputFile` carries the video's path and `ComplexFilter` resolves it to a stream specifier
+  on that input, and `CommandGenerator` never emits a second `-i` for it. Being sparse, the subtitle
+  stream falls further behind every second and drags the video down with it. FFmpeg says so itself at
+  `-loglevel warning`: `[sist#0:3/dvd_subtitle] Resumed reading at pts 10.400 with rate 6.000 after a
+  lag of 0.922s`, repeating with the lag growing 0.9→3.8 s while `pts` stays pinned (no new packet).
+- **Measured on prod (QSV, `-threads 1`, `dvd_subtitle`→overlay), 45 s steady-state window after a
+  6 s settle:**
+
+  | variant | throughput |
+  |---|---|
+  | `-readrate 1.05` (baseline) | **0.533x** (×3 runs) |
+  | `+ -readrate_catchup 2.0` | 0.711x |
+  | `+ -readrate_catchup 6.0` | **1.067x** (×2 runs) |
+  | `+ -readrate_catchup 20.0` | 1.067x |
+  | no subtitle overlay (control) | 1.067x |
+
+  A live client consumes at 1.0x, so 0.53x drains its buffer until it stalls — the reported symptom.
+- **`20.0` measuring the same as `6.0` is why 6.0 was chosen** — above the catch-up point the value
+  is not a throughput dial, so there is nothing to buy by going higher. It is **not** evidence about
+  allocation: that is a steady-state throughput number, not a count of frames in flight.
+- **Why this does not reopen `ffmpeg.qsv-extra-hw-frames-floor` (#529).** Not because catchup is
+  brief (a permanently GPU-bound channel lags forever, so 6x is a standing licence), and **not**
+  because read rate is allocation-irrelevant — #529 measured that it is not (at `extra_hw_frames=0`,
+  `1.05` without a burst exits 0 while `1.05`+burst hits ENOMEM). Read rate changes how fast frames
+  enter the graph, not how deep its queues are, and #529 showed that only bites when the pool has
+  **no headroom**. The 64-frame floor now guarantees headroom, so the load-bearing measurement is
+  row 5 of that truth table — **no `-readrate` at all with 64 frames → 14 segments, exit 0** — and a
+  6x ceiling is strictly less aggressive than no throttle. Reinforcing it, `-readrate_initial_burst 8`
+  has read *flat out* at the start of every playout item since #350, so an unbounded read here is not
+  new. A 240 s QSV soak (64 frames, 60 segment boundaries) adds 1.043x sustained with **zero**
+  `Cannot allocate memory` — but it stayed largely caught-up, so it corroborates rather than proves;
+  the argument above is what carries the decision.
+- **Not QSV-specific:** reproduces on libx264 too (0.533x → 1.067x), as expected for an input-pacing
+  option upstream of any encoder or filter choice.
+- **Raising the base `-readrate` is not an alternative, and was measured:** 2.0→0.62x, 3.0→0.80x,
+  4.0→0.80x, 6.0→0.89x. It asymptotes *below* realtime, because the rate ceiling was never the
+  binding constraint. Recorded so it is not re-proposed.
+- **Catchup does NOT subsume the #350 burst; they fix orthogonal metrics.** Measured
+  time-to-first-segment: `-readrate` alone 3.71 s, `+burst` **0.72 s**, `+catchup` alone **3.65 s**,
+  both 0.67 s. Catchup buys nothing at cold start (no accumulated lag at t=0 to recover) and the
+  burst buys nothing for throughput (the 0.533x baseline already had it), so removing the burst on
+  the theory that catchup replaces it would regress tune-in ~5x.
+- **Applied to realtime video/audio inputs generally, not only subtitle pipelines** — it is inert
+  unless an input is behind, and any sparse stream can trigger this, so gating it on "has a bitmap
+  subtitle" would fix the site instead of the boundary. Two deliberate exclusions, both test-pinned:
+  `ConcatInputFile` (reads already-written segments at a flat 1.0, nothing sparse to lag on) and
+  **still images**, mirroring #350 — their video input takes no readrate at all, so catchup would
+  reach only the separate audio input and break the pacing symmetry #350 declined to break. An
+  image-based subtitle always rides the *video* path, so that shape cannot starve this way anyway.
+- **Capability-gated via `FFmpegKnownOption.HasOption`**, the same fail-safe posture as
+  `-readrate_initial_burst`: detection parses `ffmpeg -h long`, so a binary without the option
+  silently keeps today's behavior instead of failing to start.
+
+**Accepted residual:** the affected population is items carrying an embedded bitmap subtitle matching
+the channel's subtitle mode — 3,182 of 24,646 media versions (12.9%) on prod. It is a property of the
+*item*, not the channel, which is why the stall presented as random: a channel plays one episode fine
+and stalls on the next.