From 6c6ccfa94b093c7ae7a1393ee877818726f06aa7 Mon Sep 17 00:00:00 2001 From: Jason Dove <1695733+jasongdove@users.noreply.github.com> Date: Mon, 28 Jul 2025 16:19:20 +0000 Subject: [PATCH] fix seeking with text subtitles (#2214) --- CHANGELOG.md | 1 + .../Queries/GetSeekTextSubtitleProcess.cs | 6 ++ .../GetSeekTextSubtitleProcessHandler.cs | 47 +++++++++++++ .../Streaming/SeekTextSubtitleProcess.cs | 5 ++ .../FFmpeg/FFmpegLibraryProcessService.cs | 44 ++++++++++++ .../FFmpeg/IFFmpegProcessService.cs | 2 + .../InputOption/CopyTimestampInputOption.cs | 2 +- .../OutputFormat/OutputFormatAss.cs | 13 ++++ .../OutputFormat/OutputFormatSrt.cs | 13 ++++ .../OutputFormat/OutputFormatWebVtt.cs | 13 ++++ ErsatzTV.FFmpeg/Pipeline/IPipelineBuilder.cs | 1 + .../Pipeline/NvidiaPipelineBuilder.cs | 2 - .../Pipeline/PipelineBuilderBase.cs | 34 ++++++++-- .../Pipeline/QsvPipelineBuilder.cs | 2 - .../Pipeline/SoftwarePipelineBuilder.cs | 2 - .../Pipeline/VaapiPipelineBuilder.cs | 2 - ErsatzTV/Controllers/InternalController.cs | 68 +++++++++++++++---- 17 files changed, 228 insertions(+), 29 deletions(-) create mode 100644 ErsatzTV.Application/Streaming/Queries/GetSeekTextSubtitleProcess.cs create mode 100644 ErsatzTV.Application/Streaming/Queries/GetSeekTextSubtitleProcessHandler.cs create mode 100644 ErsatzTV.Application/Streaming/SeekTextSubtitleProcess.cs create mode 100644 ErsatzTV.FFmpeg/OutputFormat/OutputFormatAss.cs create mode 100644 ErsatzTV.FFmpeg/OutputFormat/OutputFormatSrt.cs create mode 100644 ErsatzTV.FFmpeg/OutputFormat/OutputFormatWebVtt.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 66fe2509c..067c29f62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Fix app startup with MySql/MariaDB - YAML playout: fix `pad_to_next` always running over time +- Fix playback with text subtitles when seeking into content, i.e. when first joining a channel ### Changed - Always tell ffmpeg to stop encoding with a specific duration diff --git a/ErsatzTV.Application/Streaming/Queries/GetSeekTextSubtitleProcess.cs b/ErsatzTV.Application/Streaming/Queries/GetSeekTextSubtitleProcess.cs new file mode 100644 index 000000000..0c8f050e8 --- /dev/null +++ b/ErsatzTV.Application/Streaming/Queries/GetSeekTextSubtitleProcess.cs @@ -0,0 +1,6 @@ +using ErsatzTV.Core; + +namespace ErsatzTV.Application.Streaming; + +public record GetSeekTextSubtitleProcess(string SubtitlePath, TimeSpan Seek) + : IRequest>; diff --git a/ErsatzTV.Application/Streaming/Queries/GetSeekTextSubtitleProcessHandler.cs b/ErsatzTV.Application/Streaming/Queries/GetSeekTextSubtitleProcessHandler.cs new file mode 100644 index 000000000..fd6ddf5bf --- /dev/null +++ b/ErsatzTV.Application/Streaming/Queries/GetSeekTextSubtitleProcessHandler.cs @@ -0,0 +1,47 @@ +using CliWrap; +using ErsatzTV.Core; +using ErsatzTV.Core.Domain; +using ErsatzTV.Core.Interfaces.FFmpeg; +using ErsatzTV.Infrastructure.Data; +using ErsatzTV.Infrastructure.Extensions; +using Microsoft.EntityFrameworkCore; + +namespace ErsatzTV.Application.Streaming; + +public class GetSeekTextSubtitleProcessHandler( + IDbContextFactory dbContextFactory, + IFFmpegProcessService ffmpegProcessService) + : IRequestHandler> +{ + public async Task> Handle( + GetSeekTextSubtitleProcess request, + CancellationToken cancellationToken) + { + await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken); + Validation validation = await Validate(dbContext); + return await validation.Match( + ffmpegPath => GetProcess(request, ffmpegPath), + error => Task.FromResult>(error.Join())); + } + + private async Task> GetProcess( + GetSeekTextSubtitleProcess request, + string ffmpegPath) + { + Command process = await ffmpegProcessService.SeekTextSubtitle( + ffmpegPath, + request.SubtitlePath, + request.Seek); + + return new SeekTextSubtitleProcess(process); + } + + private static async Task> Validate(TvContext dbContext) => + await FFmpegPathMustExist(dbContext); + + private static Task> FFmpegPathMustExist(TvContext dbContext) => + dbContext.ConfigElements.GetValue(ConfigElementKey.FFmpegPath) + .FilterT(File.Exists) + .Map(maybePath => maybePath.ToValidation("FFmpeg path does not exist on filesystem")); +} diff --git a/ErsatzTV.Application/Streaming/SeekTextSubtitleProcess.cs b/ErsatzTV.Application/Streaming/SeekTextSubtitleProcess.cs new file mode 100644 index 000000000..06b2fdcd4 --- /dev/null +++ b/ErsatzTV.Application/Streaming/SeekTextSubtitleProcess.cs @@ -0,0 +1,5 @@ +using CliWrap; + +namespace ErsatzTV.Application.Streaming; + +public record SeekTextSubtitleProcess(Command Process); diff --git a/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs b/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs index 69cb061b2..6ab04e592 100644 --- a/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs +++ b/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs @@ -141,6 +141,11 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService { // proxy to avoid dealing with escaping subtitle.Path = $"http://localhost:{Settings.StreamingPort}/media/subtitle/{subtitle.Id}"; + + foreach (TimeSpan seek in playbackSettings.StreamSeek) + { + subtitle.Path += $"?seekToMs={(int)seek.TotalMilliseconds}"; + } } } @@ -920,6 +925,45 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService watermarkWidthPercent, cancellationToken); + public async Task SeekTextSubtitle(string ffmpegPath, string inputFile, TimeSpan seek) + { + var videoInputFile = new VideoInputFile( + inputFile, + new List + { + new( + 0, + string.Empty, + string.Empty, + None, + ColorParams.Default, + FrameSize.Unknown, + string.Empty, + string.Empty, + None, + true, + ScanKind.Progressive) + }); + + IPipelineBuilder pipelineBuilder = await _pipelineBuilderFactory.GetBuilder( + HardwareAccelerationMode.None, + videoInputFile, + None, + None, + None, + Option.None, + Option.None, + None, + None, + FileSystemLayout.FFmpegReportsFolder, + FileSystemLayout.FontsCacheFolder, + ffmpegPath); + + FFmpegPipeline pipeline = pipelineBuilder.Seek(inputFile, seek); + + return GetCommand(ffmpegPath, videoInputFile, None, None, None, pipeline, false); + } + private static Option GetWatermarkInputFile( Option watermarkOptions, Option> maybeFadePoints) diff --git a/ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegProcessService.cs b/ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegProcessService.cs index 2b11016b9..1405186c5 100644 --- a/ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegProcessService.cs +++ b/ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegProcessService.cs @@ -90,4 +90,6 @@ public interface IFFmpegProcessService int verticalMarginPercent, int watermarkWidthPercent, CancellationToken cancellationToken); + + Task SeekTextSubtitle(string ffmpegPath, string inputFile, TimeSpan seek); } diff --git a/ErsatzTV.FFmpeg/InputOption/CopyTimestampInputOption.cs b/ErsatzTV.FFmpeg/InputOption/CopyTimestampInputOption.cs index 205325177..28ef67d19 100644 --- a/ErsatzTV.FFmpeg/InputOption/CopyTimestampInputOption.cs +++ b/ErsatzTV.FFmpeg/InputOption/CopyTimestampInputOption.cs @@ -7,7 +7,7 @@ public class CopyTimestampInputOption : IInputOption public EnvironmentVariable[] EnvironmentVariables => Array.Empty(); public string[] GlobalOptions => Array.Empty(); - public string[] InputOptions(InputFile inputFile) => new[] { "-copyts" }; + public string[] InputOptions(InputFile inputFile) => []; //new[] { "-copyts" }; public string[] FilterOptions => Array.Empty(); public string[] OutputOptions => Array.Empty(); diff --git a/ErsatzTV.FFmpeg/OutputFormat/OutputFormatAss.cs b/ErsatzTV.FFmpeg/OutputFormat/OutputFormatAss.cs new file mode 100644 index 000000000..95d14b4c2 --- /dev/null +++ b/ErsatzTV.FFmpeg/OutputFormat/OutputFormatAss.cs @@ -0,0 +1,13 @@ +using ErsatzTV.FFmpeg.Environment; + +namespace ErsatzTV.FFmpeg.OutputFormat; + +public class OutputFormatAss : IPipelineStep +{ + public EnvironmentVariable[] EnvironmentVariables => []; + public string[] GlobalOptions => []; + public string[] InputOptions(InputFile inputFile) => []; + public string[] FilterOptions => []; + public string[] OutputOptions => ["-f", "ass"]; + public FrameState NextState(FrameState currentState) => currentState; +} diff --git a/ErsatzTV.FFmpeg/OutputFormat/OutputFormatSrt.cs b/ErsatzTV.FFmpeg/OutputFormat/OutputFormatSrt.cs new file mode 100644 index 000000000..292b6cc81 --- /dev/null +++ b/ErsatzTV.FFmpeg/OutputFormat/OutputFormatSrt.cs @@ -0,0 +1,13 @@ +using ErsatzTV.FFmpeg.Environment; + +namespace ErsatzTV.FFmpeg.OutputFormat; + +public class OutputFormatSrt : IPipelineStep +{ + public EnvironmentVariable[] EnvironmentVariables => []; + public string[] GlobalOptions => []; + public string[] InputOptions(InputFile inputFile) => []; + public string[] FilterOptions => []; + public string[] OutputOptions => ["-f", "srt"]; + public FrameState NextState(FrameState currentState) => currentState; +} diff --git a/ErsatzTV.FFmpeg/OutputFormat/OutputFormatWebVtt.cs b/ErsatzTV.FFmpeg/OutputFormat/OutputFormatWebVtt.cs new file mode 100644 index 000000000..ceb0e2718 --- /dev/null +++ b/ErsatzTV.FFmpeg/OutputFormat/OutputFormatWebVtt.cs @@ -0,0 +1,13 @@ +using ErsatzTV.FFmpeg.Environment; + +namespace ErsatzTV.FFmpeg.OutputFormat; + +public class OutputFormatWebVtt : IPipelineStep +{ + public EnvironmentVariable[] EnvironmentVariables => []; + public string[] GlobalOptions => []; + public string[] InputOptions(InputFile inputFile) => []; + public string[] FilterOptions => []; + public string[] OutputOptions => ["-f", "webvtt"]; + public FrameState NextState(FrameState currentState) => currentState; +} diff --git a/ErsatzTV.FFmpeg/Pipeline/IPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/IPipelineBuilder.cs index 894232aca..69aa1871f 100644 --- a/ErsatzTV.FFmpeg/Pipeline/IPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/IPipelineBuilder.cs @@ -3,6 +3,7 @@ namespace ErsatzTV.FFmpeg.Pipeline; public interface IPipelineBuilder { FFmpegPipeline Resize(string outputFile, FrameSize scaledSize); + FFmpegPipeline Seek(string inputFile, TimeSpan seek); FFmpegPipeline Concat(ConcatInputFile concatInputFile, FFmpegState ffmpegState); FFmpegPipeline WrapSegmenter(ConcatInputFile concatInputFile, FFmpegState ffmpegState); FFmpegPipeline Build(FFmpegState ffmpegState, FrameState desiredState); diff --git a/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs index d466b04ee..63fadcb35 100644 --- a/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs @@ -542,8 +542,6 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder { if (context.HasSubtitleText) { - videoInputFile.AddOption(new CopyTimestampInputOption()); - if (videoInputFile.FilterSteps.Count == 0 && videoInputFile.InputOptions.OfType().Any()) { // change the hw accel output to software so the explicit download isn't needed diff --git a/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs b/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs index 8aa8dd24f..5ae833505 100644 --- a/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs +++ b/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs @@ -65,13 +65,37 @@ public abstract class PipelineBuilderBase : IPipelineBuilder IPipelineFilterStep scaleStep = new ScaleImageFilter(scaledSize); _videoInputFile.Iter(f => f.FilterSteps.Add(scaleStep)); - pipelineSteps.Add(new VideoFilter(new[] { scaleStep })); + pipelineSteps.Add(new VideoFilter([scaleStep])); pipelineSteps.Add(scaleStep); pipelineSteps.Add(new FileNameOutputOption(outputFile)); return new FFmpegPipeline(pipelineSteps, false); } + public FFmpegPipeline Seek(string inputFile, TimeSpan seek) + { + IPipelineStep outputFormat = Path.GetExtension(inputFile).ToLowerInvariant() switch + { + "ass" or "ssa" => new OutputFormatAss(), + "vtt" => new OutputFormatWebVtt(), + _ => new OutputFormatSrt() + }; + + var pipelineSteps = new List + { + new NoStandardInputOption(), + new HideBannerOption(), + new NoStatsOption(), + new LoglevelErrorOption(), + new StreamSeekFilterOption(seek), + new EncoderCopySubtitle(), + outputFormat, + new PipeProtocol(), + }; + + return new FFmpegPipeline(pipelineSteps, false); + } + public FFmpegPipeline Concat(ConcatInputFile concatInputFile, FFmpegState ffmpegState) { var pipelineSteps = new List @@ -823,10 +847,10 @@ public abstract class PipelineBuilderBase : IPipelineBuilder videoInputFile.AddOption(option); // need to seek text subtitle files - if (context.HasSubtitleText) - { - pipelineSteps.Add(new StreamSeekFilterOption(desiredStart)); - } + // if (context.HasSubtitleText) + // { + // pipelineSteps.Add(new StreamSeekFilterOption(desiredStart)); + // } } } diff --git a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs index 2456ed18b..df64f3bb3 100644 --- a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs @@ -508,8 +508,6 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder { if (context.HasSubtitleText) { - videoInputFile.AddOption(new CopyTimestampInputOption()); - var downloadFilter = new HardwareDownloadFilter(currentState); currentState = downloadFilter.NextState(currentState); videoInputFile.FilterSteps.Add(downloadFilter); diff --git a/ErsatzTV.FFmpeg/Pipeline/SoftwarePipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/SoftwarePipelineBuilder.cs index 9b26f1afe..18f1c39a6 100644 --- a/ErsatzTV.FFmpeg/Pipeline/SoftwarePipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/SoftwarePipelineBuilder.cs @@ -270,8 +270,6 @@ public class SoftwarePipelineBuilder : PipelineBuilderBase { if (context.HasSubtitleText) { - videoInputFile.AddOption(new CopyTimestampInputOption()); - var subtitlesFilter = new SubtitlesFilter(fontsFolder, subtitle); videoInputFile.FilterSteps.Add(subtitlesFilter); } diff --git a/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs index e64178ac2..99870f74e 100644 --- a/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs @@ -449,8 +449,6 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder { if (context.HasSubtitleText) { - videoInputFile.AddOption(new CopyTimestampInputOption()); - // if (videoInputFile.FilterSteps.Count == 0 && videoInputFile.InputOptions.OfType().Any()) // { // // change the hw accel output to software so the explicit download isn't needed diff --git a/ErsatzTV/Controllers/InternalController.cs b/ErsatzTV/Controllers/InternalController.cs index ad94d7763..991bd668c 100644 --- a/ErsatzTV/Controllers/InternalController.cs +++ b/ErsatzTV/Controllers/InternalController.cs @@ -190,27 +190,65 @@ public class InternalController : ControllerBase } [HttpGet("/media/subtitle/{id:int}")] - public async Task GetSubtitle(int id) + public async Task GetSubtitle(int id, [FromQuery] long? seekToMs) { - Either path = await _mediator.Send(new GetSubtitlePathById(id)); - return path.Match( - Left: _ => new NotFoundResult(), - Right: r => + Either maybePath = await _mediator.Send(new GetSubtitlePathById(id)); + + foreach (string path in maybePath.RightToSeq()) + { + string mimeType = Path.GetExtension(path).ToLowerInvariant() switch { - if (r.StartsWith("http", StringComparison.OrdinalIgnoreCase)) + "ass" or "ssa" => "text/x-ssa", + "vtt" => "text/vtt", + _ => "application/x-subrip" + }; + + if (seekToMs is > 0) + { + Either maybeProcess = await _mediator.Send( + new GetSeekTextSubtitleProcess(path, TimeSpan.FromMilliseconds(seekToMs.Value))); + foreach (SeekTextSubtitleProcess processModel in maybeProcess.RightToSeq()) { - return new RedirectResult(r); + Command command = processModel.Process; + + _logger.LogDebug("ffmpeg text subtitle arguments {FFmpegArguments}", command.Arguments); + + var process = new FFmpegProcess + { + StartInfo = new ProcessStartInfo + { + FileName = command.TargetFilePath, + Arguments = command.Arguments, + RedirectStandardOutput = true, + RedirectStandardError = false, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + HttpContext.Response.RegisterForDispose(process); + + foreach ((string key, string value) in command.EnvironmentVariables) + { + process.StartInfo.Environment[key] = value; + } + + process.Start(); + return new FileStreamResult(process.StandardOutput.BaseStream, mimeType); } - string mimeType = Path.GetExtension(r).ToLowerInvariant() switch - { - "ass" or "ssa" => "text/x-ssa", - "vtt" => "text/vtt", - _ => "application/x-subrip" - }; + return new NotFoundResult(); + } - return new PhysicalFileResult(r, mimeType); - }); + if (path.StartsWith("http", StringComparison.OrdinalIgnoreCase)) + { + return new RedirectResult(path); + } + + return new PhysicalFileResult(path, mimeType); + } + + return new NotFoundResult(); } private async Task GetSegmenterV2Stream(string channelNumber)