transcoding improvements (#1452)

* use noautoscale with vaapi encoder

* only use one input file for vaapi with radeonsi driver

* fix vaapi 8-bit to 10-bit

* fix nvidia subtitle scaling

* optimize nvidia subtitle scaling

* fix test pgs subtitle

CHANGELOG.md

@@ -14,7 +14,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Include `inputstream.ffmpegdirect` properties in channels.m3u when requested by Kodi
 
 ### Fixed
-- Fix playout bug that caused some schedule items with fixed start times to be pushed to the next day 
+- Fix playout bug that caused some schedule items with fixed start times to be pushed to the next day
+- Fix VAAPI transcoding 8-bit source content to 10-bit
+- Fix NVIDIA subtitle scaling when `scale_npp` filter is unavailable
 
 ### Changed
 - Upgrade ffmpeg to 6.1, which is now *required* for all installs

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -762,7 +762,8 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             audioInputFile,
             watermarkInputFile,
             concatInputFile,
-            pipeline.PipelineSteps);
+            pipeline.PipelineSteps,
+            pipeline.IsIntelVaapiOrQsv);
 
         if (environmentVariables.Any())
         {

ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs

@@ -480,7 +480,8 @@ public class PipelineBuilderBaseTests
             audioInputFile,
             watermarkInputFile,
             concatInputFile,
-            pipeline.PipelineSteps);
+            pipeline.PipelineSteps,
+            pipeline.IsIntelVaapiOrQsv);
 
         var command = string.Join(" ", arguments);
 

ErsatzTV.FFmpeg/CommandGenerator.cs

@@ -2,7 +2,6 @@
 using ErsatzTV.FFmpeg.Environment;
 using ErsatzTV.FFmpeg.Filter;
 using ErsatzTV.FFmpeg.GlobalOption;
-using ErsatzTV.FFmpeg.GlobalOption.HardwareAcceleration;
 using ErsatzTV.FFmpeg.InputOption;
 
 namespace ErsatzTV.FFmpeg;
@@ -17,7 +16,8 @@ public static class CommandGenerator
         Option<AudioInputFile> maybeAudioInputFile,
         Option<WatermarkInputFile> maybeWatermarkInputFile,
         Option<ConcatInputFile> maybeConcatInputFile,
-        IList<IPipelineStep> pipelineSteps)
+        IList<IPipelineStep> pipelineSteps,
+        bool isIntelVaapiOrQsv)
     {
         var arguments = new List<string>();
 
@@ -41,10 +41,7 @@ public static class CommandGenerator
 
         foreach (AudioInputFile audioInputFile in maybeAudioInputFile)
         {
-            bool isVaapiOrQsv =
-                pipelineSteps.Any(s => s is VaapiHardwareAccelerationOption or QsvHardwareAccelerationOption);
-
-            if (!includedPaths.Contains(audioInputFile.Path) || isVaapiOrQsv)
+            if (!includedPaths.Contains(audioInputFile.Path) || isIntelVaapiOrQsv)
             {
                 includedPaths.Add(audioInputFile.Path);
 

ErsatzTV.FFmpeg/FFmpegPipeline.cs

@@ -1,3 +1,3 @@
 namespace ErsatzTV.FFmpeg;
 
-public record FFmpegPipeline(IList<IPipelineStep> PipelineSteps);
+public record FFmpegPipeline(IList<IPipelineStep> PipelineSteps, bool IsIntelVaapiOrQsv);

ErsatzTV.FFmpeg/Filter/ComplexFilter.cs

@@ -73,9 +73,8 @@ public class ComplexFilter : IPipelineStep
         foreach ((string path, _) in _maybeAudioInputFile)
         {
             if (!distinctPaths.Contains(path) ||
-                // use audio as a separate input with vaapi/qsv
-                _pipelineContext.HardwareAccelerationMode is HardwareAccelerationMode.Vaapi
-                    or HardwareAccelerationMode.Qsv)
+                // use audio as a separate input with intel vaapi/qsv
+                _pipelineContext.IsIntelVaapiOrQsv)
             {
                 distinctPaths.Add(path);
             }

ErsatzTV.FFmpeg/Filter/Cuda/ScaleCudaFilter.cs

@@ -24,6 +24,8 @@ public class ScaleCudaFilter : BaseFilter
         _isAnamorphicEdgeCase = isAnamorphicEdgeCase;
     }
 
+    public bool IsFormatOnly => _currentState.ScaledSize == _scaledSize;
+
     public override string Filter
     {
         get

ErsatzTV.FFmpeg/OutputOption/NoAutoScaleOutputOption.cs

@@ -0,0 +1,6 @@
+namespace ErsatzTV.FFmpeg.OutputOption;
+
+public class NoAutoScaleOutputOption : OutputOption
+{
+    public override IList<string> OutputOptions => new List<string> { "-noautoscale" };
+}

ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs

@@ -16,6 +16,7 @@ namespace ErsatzTV.FFmpeg.Pipeline;
 
 public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
 {
+    private readonly IFFmpegCapabilities _ffmpegCapabilities;
     private readonly IHardwareCapabilities _hardwareCapabilities;
     private readonly ILogger _logger;
 
@@ -40,6 +41,7 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
         fontsFolder,
         logger)
     {
+        _ffmpegCapabilities = ffmpegCapabilities;
         _hardwareCapabilities = hardwareCapabilities;
         _logger = logger;
     }
@@ -417,7 +419,7 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
         return currentState;
     }
 
-    private static FrameState SetSubtitle(
+    private FrameState SetSubtitle(
         VideoInputFile videoInputFile,
         Option<SubtitleInputFile> subtitleInputFile,
         PipelineContext context,
@@ -466,17 +468,33 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
 
                 if (currentState.PixelFormat.Map(pf => pf.BitDepth).IfNone(8) == 8)
                 {
-                    var subtitleHardwareUpload = new HardwareUploadCudaFilter(
-                        currentState with { FrameDataLocation = FrameDataLocation.Software });
-                    subtitle.FilterSteps.Add(subtitleHardwareUpload);
-
-                    // only scale if scaling or padding was used for main video stream
-                    if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or ScaleCudaFilter or PadFilter))
+                    if (_ffmpegCapabilities.HasFilter("scale_npp"))
                     {
-                        var scaleFilter = new SubtitleScaleNppFilter(desiredState.PaddedSize);
-                        subtitle.FilterSteps.Add(scaleFilter);
-                    }
+                        var subtitleHardwareUpload = new HardwareUploadCudaFilter(
+                            currentState with { FrameDataLocation = FrameDataLocation.Software });
+                        subtitle.FilterSteps.Add(subtitleHardwareUpload);
 
+                        // only scale if scaling or padding was used for main video stream
+                        if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or ScaleCudaFilter { IsFormatOnly: false } or PadFilter))
+                        {
+                            var scaleFilter = new SubtitleScaleNppFilter(desiredState.PaddedSize);
+                            subtitle.FilterSteps.Add(scaleFilter);
+                        }
+                    }
+                    else
+                    {
+                        // only scale if scaling or padding was used for main video stream
+                        if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or ScaleCudaFilter { IsFormatOnly: false } or PadFilter))
+                        {
+                            var scaleFilter = new ScaleImageFilter(desiredState.PaddedSize);
+                            subtitle.FilterSteps.Add(scaleFilter);
+                        }
+                        
+                        var subtitleHardwareUpload = new HardwareUploadCudaFilter(
+                            currentState with { FrameDataLocation = FrameDataLocation.Software });
+                        subtitle.FilterSteps.Add(subtitleHardwareUpload);
+                    }
+                    
                     var subtitlesFilter = new OverlaySubtitleCudaFilter();
                     subtitleOverlayFilterSteps.Add(subtitlesFilter);
                 }

ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs

@@ -66,7 +66,7 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
         pipelineSteps.Add(scaleStep);
         pipelineSteps.Add(new FileNameOutputOption(outputFile));
 
-        return new FFmpegPipeline(pipelineSteps);
+        return new FFmpegPipeline(pipelineSteps, false);
     }
 
     public FFmpegPipeline Concat(ConcatInputFile concatInputFile, FFmpegState ffmpegState)
@@ -113,7 +113,7 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
             pipelineSteps.Add(new FFReportVariable(_reportsFolder, concatInputFile));
         }
 
-        return new FFmpegPipeline(pipelineSteps);
+        return new FFmpegPipeline(pipelineSteps, false);
     }
 
     public FFmpegPipeline WrapSegmenter(ConcatInputFile concatInputFile, FFmpegState ffmpegState)
@@ -138,7 +138,7 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
         pipelineSteps.Add(new OutputFormatMpegTs(false));
         pipelineSteps.Add(new PipeProtocol());
 
-        return new FFmpegPipeline(pipelineSteps);
+        return new FFmpegPipeline(pipelineSteps, false);
     }
 
     public FFmpegPipeline Build(FFmpegState ffmpegState, FrameState desiredState)
@@ -174,7 +174,8 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
             _subtitleInputFile.Map(s => s is { IsImageBased: true, Method: SubtitleMethod.Burn }).IfNone(false),
             _subtitleInputFile.Map(s => s is { IsImageBased: false, Method: SubtitleMethod.Burn }).IfNone(false),
             desiredState.Deinterlaced,
-            desiredState.PixelFormat.Map(pf => pf.BitDepth).IfNone(8) == 10);
+            desiredState.PixelFormat.Map(pf => pf.BitDepth).IfNone(8) == 10,
+            false);
 
         SetThreadCount(ffmpegState, desiredState, pipelineSteps);
         SetSceneDetect(videoStream, ffmpegState, desiredState, pipelineSteps);
@@ -190,6 +191,8 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
             context,
             pipelineSteps);
 
+        context = context with { IsIntelVaapiOrQsv = IsIntelVaapiOrQsv(ffmpegState) };
+
         if (_audioInputFile.IsNone)
         {
             pipelineSteps.Add(new EncoderCopyAudio());
@@ -219,7 +222,7 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
 
         pipelineSteps.Add(complexFilter);
 
-        return new FFmpegPipeline(pipelineSteps);
+        return new FFmpegPipeline(pipelineSteps, context.IsIntelVaapiOrQsv);
     }
 
     private void LogUnknownDecoder(
@@ -397,6 +400,8 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
         }
     }
 
+    protected abstract bool IsIntelVaapiOrQsv(FFmpegState ffmpegState);
+
     protected abstract FFmpegState SetAccelState(
         VideoStream videoStream,
         FFmpegState ffmpegState,

ErsatzTV.FFmpeg/Pipeline/PipelineContext.cs

@@ -6,4 +6,5 @@ public record PipelineContext(
     bool HasSubtitleOverlay,
     bool HasSubtitleText,
     bool ShouldDeinterlace,
-    bool Is10BitOutput);
+    bool Is10BitOutput,
+    bool IsIntelVaapiOrQsv);

ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs

@@ -44,6 +44,10 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
         _logger = logger;
     }
 
+    protected override bool IsIntelVaapiOrQsv(FFmpegState ffmpegState) =>
+        ffmpegState.DecoderHardwareAccelerationMode is HardwareAccelerationMode.Qsv ||
+        ffmpegState.EncoderHardwareAccelerationMode is HardwareAccelerationMode.Qsv;
+
     protected override FFmpegState SetAccelState(
         VideoStream videoStream,
         FFmpegState ffmpegState,

ErsatzTV.FFmpeg/Pipeline/SoftwarePipelineBuilder.cs

@@ -35,6 +35,8 @@ public class SoftwarePipelineBuilder : PipelineBuilderBase
         logger) =>
         _logger = logger;
 
+    protected override bool IsIntelVaapiOrQsv(FFmpegState ffmpegState) => false;
+
     protected override FFmpegState SetAccelState(
         VideoStream videoStream,
         FFmpegState ffmpegState,

ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs

@@ -8,6 +8,7 @@ using ErsatzTV.FFmpeg.Filter.Vaapi;
 using ErsatzTV.FFmpeg.Format;
 using ErsatzTV.FFmpeg.GlobalOption.HardwareAcceleration;
 using ErsatzTV.FFmpeg.InputOption;
+using ErsatzTV.FFmpeg.OutputOption;
 using ErsatzTV.FFmpeg.State;
 using Microsoft.Extensions.Logging;
 
@@ -43,6 +44,12 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
         _logger = logger;
     }
 
+    // check for intel vaapi (NOT radeon)
+    protected override bool IsIntelVaapiOrQsv(FFmpegState ffmpegState) =>
+        (ffmpegState.DecoderHardwareAccelerationMode is HardwareAccelerationMode.Vaapi ||
+         ffmpegState.EncoderHardwareAccelerationMode is HardwareAccelerationMode.Vaapi) &&
+        !ffmpegState.VaapiDriver.IfNone(string.Empty).StartsWith("radeon", StringComparison.OrdinalIgnoreCase);
+
     protected override FFmpegState SetAccelState(
         VideoStream videoStream,
         FFmpegState ffmpegState,
@@ -75,6 +82,12 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
             decodeCapability = FFmpegCapability.Software;
         }
 
+        // disable auto scaling when using hw encoding
+        if (encodeCapability is FFmpegCapability.Hardware)
+        {
+            pipelineSteps.Add(new NoAutoScaleOutputOption());
+        }
+        
         // disable hw accel if decoder/encoder isn't supported
         return ffmpegState with
         {
@@ -274,8 +287,12 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
                 {
                     _logger.LogDebug("FrameDataLocation == FrameDataLocation.Hardware");
 
-                    var hardwareDownload =
-                        new HardwareDownloadFilter(currentState with { PixelFormat = Some(format) });
+                    // don't try to download from 8-bit to 10-bit
+                    HardwareDownloadFilter hardwareDownload = currentState.BitDepth == 8 &&
+                                                              desiredPixelFormat.Map(pf => pf.BitDepth).IfNone(8) == 10
+                        ? new HardwareDownloadFilter(currentState)
+                        : new HardwareDownloadFilter(currentState with { PixelFormat = Some(format) });
+
                     currentState = hardwareDownload.NextState(currentState);
                     result.Add(hardwareDownload);
                 }