From 7a25996ab4ef931a13010622934d9cc623c66111 Mon Sep 17 00:00:00 2001 From: Jason Dove <1695733+jasongdove@users.noreply.github.com> Date: Mon, 19 Jun 2023 15:55:23 -0500 Subject: [PATCH] scale subtitles with all accels (#1311) * properly scale subtitles with qsv and vaapi * fixes --- CHANGELOG.md | 1 + .../Filter/AvailableDeinterlaceFilters.cs | 28 ------------ .../Filter/AvailableScaleFilters.cs | 38 ---------------- .../Filter/AvailableSubtitleOverlayFilters.cs | 16 ------- .../Filter/AvailableSubtitleScaleFilters.cs | 24 ---------- .../AvailableWatermarkOverlayFilters.cs | 30 ------------- .../Pipeline/QsvPipelineBuilder.cs | 7 +++ .../Pipeline/VaapiPipelineBuilder.cs | 21 +++++++-- .../Core/FFmpeg/TranscodingTests.cs | 44 +++++++++---------- 9 files changed, 47 insertions(+), 162 deletions(-) delete mode 100644 ErsatzTV.FFmpeg/Filter/AvailableDeinterlaceFilters.cs delete mode 100644 ErsatzTV.FFmpeg/Filter/AvailableScaleFilters.cs delete mode 100644 ErsatzTV.FFmpeg/Filter/AvailableSubtitleOverlayFilters.cs delete mode 100644 ErsatzTV.FFmpeg/Filter/AvailableSubtitleScaleFilters.cs delete mode 100644 ErsatzTV.FFmpeg/Filter/AvailableWatermarkOverlayFilters.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 469c48414..1633dfd73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Skip checking for subtitles to extract when subtitles are not enabled on a channel/schedule item +- Properly scale subtitles with QSV and some VAAPI configurations; now all configurations will scale subtitles ### Changed - `HLS Direct` streaming mode diff --git a/ErsatzTV.FFmpeg/Filter/AvailableDeinterlaceFilters.cs b/ErsatzTV.FFmpeg/Filter/AvailableDeinterlaceFilters.cs deleted file mode 100644 index 89099de00..000000000 --- a/ErsatzTV.FFmpeg/Filter/AvailableDeinterlaceFilters.cs +++ /dev/null @@ -1,28 +0,0 @@ -using ErsatzTV.FFmpeg.Filter.Cuda; -using ErsatzTV.FFmpeg.Filter.Vaapi; - -namespace ErsatzTV.FFmpeg.Filter; - -public static class AvailableDeinterlaceFilters -{ - public static IPipelineFilterStep ForAcceleration( - HardwareAccelerationMode accelMode, - FrameState currentState, - FrameState desiredState, - Option watermarkInputFile, - Option subtitleInputFile) => - accelMode switch - { - HardwareAccelerationMode.Nvenc => new YadifCudaFilter(currentState), - - // deinterlace_qsv seems to create timestamp issues - // HardwareAccelerationMode.Qsv => new DeinterlaceQsvFilter(currentState), - - // fall back to software deinterlace with watermark and no scaling - HardwareAccelerationMode.Vaapi when watermarkInputFile.IsNone && subtitleInputFile.IsNone || - currentState.ScaledSize != desiredState.ScaledSize => - new DeinterlaceVaapiFilter(currentState), - - _ => new YadifFilter(currentState) - }; -} diff --git a/ErsatzTV.FFmpeg/Filter/AvailableScaleFilters.cs b/ErsatzTV.FFmpeg/Filter/AvailableScaleFilters.cs deleted file mode 100644 index cc7fa5df6..000000000 --- a/ErsatzTV.FFmpeg/Filter/AvailableScaleFilters.cs +++ /dev/null @@ -1,38 +0,0 @@ -using ErsatzTV.FFmpeg.Filter.Cuda; -using ErsatzTV.FFmpeg.Filter.Qsv; -using ErsatzTV.FFmpeg.Filter.Vaapi; -using ErsatzTV.FFmpeg.Runtime; - -namespace ErsatzTV.FFmpeg.Filter; - -public static class AvailableScaleFilters -{ - public static IPipelineFilterStep ForAcceleration( - IRuntimeInfo _, - HardwareAccelerationMode accelMode, - FrameState currentState, - FrameSize scaledSize, - FrameSize paddedSize, - int extraHardwareFrames, - bool isAnamorphicEdgeCase, - string sampleAspectRatio) => - accelMode switch - { - HardwareAccelerationMode.Nvenc => - new ScaleCudaFilter(currentState, scaledSize, paddedSize, isAnamorphicEdgeCase), - HardwareAccelerationMode.Qsv when currentState.FrameDataLocation == FrameDataLocation.Hardware || - scaledSize == paddedSize => - new ScaleQsvFilter( - currentState, - scaledSize, - extraHardwareFrames, - isAnamorphicEdgeCase, - sampleAspectRatio), - HardwareAccelerationMode.Vaapi => new ScaleVaapiFilter( - currentState, - scaledSize, - paddedSize, - isAnamorphicEdgeCase), - _ => new ScaleFilter(currentState, scaledSize, paddedSize, isAnamorphicEdgeCase) - }; -} diff --git a/ErsatzTV.FFmpeg/Filter/AvailableSubtitleOverlayFilters.cs b/ErsatzTV.FFmpeg/Filter/AvailableSubtitleOverlayFilters.cs deleted file mode 100644 index e92255331..000000000 --- a/ErsatzTV.FFmpeg/Filter/AvailableSubtitleOverlayFilters.cs +++ /dev/null @@ -1,16 +0,0 @@ -using ErsatzTV.FFmpeg.Filter.Cuda; -using ErsatzTV.FFmpeg.Filter.Qsv; -using ErsatzTV.FFmpeg.Format; - -namespace ErsatzTV.FFmpeg.Filter; - -public static class AvailableSubtitleOverlayFilters -{ - public static IPipelineFilterStep ForAcceleration(HardwareAccelerationMode accelMode) => - accelMode switch - { - HardwareAccelerationMode.Nvenc => new OverlaySubtitleCudaFilter(), - HardwareAccelerationMode.Qsv => new OverlaySubtitleQsvFilter(), - _ => new OverlaySubtitleFilter(new PixelFormatYuv420P()) - }; -} diff --git a/ErsatzTV.FFmpeg/Filter/AvailableSubtitleScaleFilters.cs b/ErsatzTV.FFmpeg/Filter/AvailableSubtitleScaleFilters.cs deleted file mode 100644 index 2bd638c4f..000000000 --- a/ErsatzTV.FFmpeg/Filter/AvailableSubtitleScaleFilters.cs +++ /dev/null @@ -1,24 +0,0 @@ -using ErsatzTV.FFmpeg.Filter.Cuda; -using ErsatzTV.FFmpeg.Filter.Qsv; - -namespace ErsatzTV.FFmpeg.Filter; - -public static class AvailableSubtitleScaleFilters -{ - public static IPipelineFilterStep ForAcceleration( - HardwareAccelerationMode accelMode, - FrameState currentState, - FrameSize scaledSize, - FrameSize paddedSize, - int extraHardwareFrames) => - accelMode switch - { - HardwareAccelerationMode.Nvenc => new SubtitleScaleNppFilter(currentState, scaledSize, paddedSize), - HardwareAccelerationMode.Qsv => new SubtitleScaleQsvFilter( - currentState, - scaledSize, - paddedSize, - extraHardwareFrames), - _ => new ScaleImageFilter(paddedSize) - }; -} diff --git a/ErsatzTV.FFmpeg/Filter/AvailableWatermarkOverlayFilters.cs b/ErsatzTV.FFmpeg/Filter/AvailableWatermarkOverlayFilters.cs deleted file mode 100644 index cba1dffc4..000000000 --- a/ErsatzTV.FFmpeg/Filter/AvailableWatermarkOverlayFilters.cs +++ /dev/null @@ -1,30 +0,0 @@ -using ErsatzTV.FFmpeg.Filter.Cuda; -using ErsatzTV.FFmpeg.Filter.Qsv; -using ErsatzTV.FFmpeg.Format; -using ErsatzTV.FFmpeg.State; -using Microsoft.Extensions.Logging; - -namespace ErsatzTV.FFmpeg.Filter; - -public static class AvailableWatermarkOverlayFilters -{ - public static IPipelineFilterStep ForAcceleration( - HardwareAccelerationMode accelMode, - WatermarkState watermarkState, - FrameSize resolution, - FrameSize squarePixelFrameSize, - ILogger logger) => - accelMode switch - { - HardwareAccelerationMode.Nvenc => - new OverlayWatermarkCudaFilter(watermarkState, resolution, squarePixelFrameSize, logger), - HardwareAccelerationMode.Qsv => - new OverlayWatermarkQsvFilter(watermarkState, resolution, squarePixelFrameSize, logger), - _ => new OverlayWatermarkFilter( - watermarkState, - resolution, - squarePixelFrameSize, - new PixelFormatYuv420P(), - logger) - }; -} diff --git a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs index 57e255596..4e91d5437 100644 --- a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs @@ -491,6 +491,13 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder pf = availablePixelFormat; } } + + // only scale if scaling or padding was used for main video stream + if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or ScaleFilter or PadFilter)) + { + var scaleFilter = new ScaleImageFilter(desiredState.PaddedSize); + subtitle.FilterSteps.Add(scaleFilter); + } var subtitlesFilter = new OverlaySubtitleFilter(pf); subtitleOverlayFilterSteps.Add(subtitlesFilter); diff --git a/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs index 024977759..f334f05af 100644 --- a/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs @@ -418,6 +418,10 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder if (forceSoftwareOverlay) { + var downloadFilter = new HardwareDownloadFilter(currentState); + currentState = downloadFilter.NextState(currentState); + videoInputFile.FilterSteps.Add(downloadFilter); + foreach (IPixelFormat pixelFormat in desiredState.PixelFormat) { IPixelFormat pf = pixelFormat; @@ -429,6 +433,13 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder } } + // only scale if scaling or padding was used for main video stream + if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or ScaleVaapiFilter or PadFilter)) + { + var scaleFilter = new ScaleImageFilter(desiredState.PaddedSize); + subtitle.FilterSteps.Add(scaleFilter); + } + var subtitlesFilter = new OverlaySubtitleFilter(pf); subtitleOverlayFilterSteps.Add(subtitlesFilter); } @@ -438,10 +449,12 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder var subtitleHardwareUpload = new HardwareUploadVaapiFilter(false); subtitle.FilterSteps.Add(subtitleHardwareUpload); - // always scale - shouldn't really be needed outside of transcoding tests, which use picture subtitles - // that are too big - var scaleFilter = new SubtitleScaleVaapiFilter(desiredState.PaddedSize); - subtitle.FilterSteps.Add(scaleFilter); + // only scale if scaling or padding was used for main video stream + if (videoInputFile.FilterSteps.Any(s => s is ScaleFilter or ScaleVaapiFilter or PadFilter)) + { + var scaleFilter = new SubtitleScaleVaapiFilter(desiredState.PaddedSize); + subtitle.FilterSteps.Add(scaleFilter); + } var subtitlesFilter = new OverlaySubtitleVaapiFilter(); subtitleOverlayFilterSteps.Add(subtitlesFilter); diff --git a/ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs b/ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs index 392698530..6eeeca9b9 100644 --- a/ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs +++ b/ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs @@ -21,7 +21,6 @@ using ErsatzTV.FFmpeg.Filter.Vaapi; using ErsatzTV.FFmpeg.Format; using ErsatzTV.FFmpeg.Pipeline; using ErsatzTV.FFmpeg.State; -using ErsatzTV.Infrastructure.Data.Repositories; using ErsatzTV.Infrastructure.Images; using ErsatzTV.Infrastructure.Runtime; using ErsatzTV.Scanner.Core.Interfaces.Metadata; @@ -164,7 +163,7 @@ public class TranscodingTests // // // av1 yuv420p10le 51 // // // new("msmpeg4v2", "yuv420p"), - new("msmpeg4v3", "yuv420p") + // new("msmpeg4v3", "yuv420p") // // // wmv3 yuv420p 1 }; @@ -190,10 +189,10 @@ public class TranscodingTests public static HardwareAccelerationKind[] TestAccelerations = { - // HardwareAccelerationKind.None, + HardwareAccelerationKind.None, // HardwareAccelerationKind.Nvenc, HardwareAccelerationKind.Vaapi, - HardwareAccelerationKind.Qsv + // HardwareAccelerationKind.Qsv // HardwareAccelerationKind.VideoToolbox, // HardwareAccelerationKind.Amf }; @@ -251,7 +250,7 @@ public class TranscodingTests MemoryCache, LoggerFactory.CreateLogger()), LoggerFactory.CreateLogger()), - new Mock().Object, + new Mock().Object, LoggerFactory.CreateLogger()); var songVideoGenerator = new SongVideoGenerator(tempFilePool, mockImageCache.Object, service); @@ -352,7 +351,7 @@ public class TranscodingTests now, Option.None, GetWatermark(watermark), - VaapiDriver.Default, + VaapiDriver.RadeonSI, "/dev/dri/renderD128", Option.None, false, @@ -372,6 +371,7 @@ public class TranscodingTests profileBitDepth, profileVideoFormat, profileAcceleration, + VaapiDriver.RadeonSI, localStatisticsProvider, () => videoVersion); } @@ -618,7 +618,7 @@ public class TranscodingTests now, Option.None, channelWatermark, - VaapiDriver.Default, + VaapiDriver.RadeonSI, "/dev/dri/renderD128", Option.None, false, @@ -638,6 +638,7 @@ public class TranscodingTests profileBitDepth, profileVideoFormat, profileAcceleration, + VaapiDriver.RadeonSI, localStatisticsProvider, () => v); } @@ -739,20 +740,16 @@ public class TranscodingTests var args = $"-y -f lavfi -i anoisesrc=color=brown -f lavfi -i testsrc=duration=1:size={resolution}:rate=30 {videoFilter} -c:a aac -c:v {inputFormat.Encoder}{colorRange}{colorSpace}{colorTransfer}{colorPrimaries} -shortest -pix_fmt {inputFormat.PixelFormat} -strict -2 {flags} {file}"; - var p1 = new Process - { - StartInfo = new ProcessStartInfo - { - FileName = ExecutableName("ffmpeg"), - Arguments = args - } - }; + BufferedCommandResult p1 = await Cli.Wrap(ExecutableName("ffmpeg")) + .WithArguments(args) + .WithValidation(CommandResultValidation.None) + .ExecuteBufferedAsync(); + + string output = string.IsNullOrWhiteSpace(p1.StandardOutput) + ? p1.StandardError + : p1.StandardOutput; - p1.Start(); - await p1.WaitForExitAsync(); - // ReSharper disable once MethodHasAsyncOverload - p1.WaitForExit(); - p1.ExitCode.Should().Be(0); + p1.ExitCode.Should().Be(0, output); switch (subtitle) { @@ -871,7 +868,7 @@ public class TranscodingTests MemoryCache, LoggerFactory.CreateLogger()), LoggerFactory.CreateLogger()), - new Mock().Object, + new Mock().Object, LoggerFactory.CreateLogger()); return service; @@ -883,6 +880,7 @@ public class TranscodingTests FFmpegProfileBitDepth profileBitDepth, FFmpegProfileVideoFormat profileVideoFormat, HardwareAccelerationKind profileAcceleration, + VaapiDriver vaapiDriver, ILocalStatisticsProvider localStatisticsProvider, Func getFinalMediaVersion) { @@ -995,8 +993,10 @@ public class TranscodingTests // AMF doesn't seem to set this metadata properly // MPEG2Video doesn't always seem to set this properly + // RADEONSI driver doesn't set this properly if (profileAcceleration != HardwareAccelerationKind.Amf && - profileVideoFormat != FFmpegProfileVideoFormat.Mpeg2Video) + profileVideoFormat != FFmpegProfileVideoFormat.Mpeg2Video && + (profileAcceleration != HardwareAccelerationKind.Vaapi || vaapiDriver != VaapiDriver.RadeonSI)) { colorParams.IsBt709.Should().BeTrue($"{colorParams}"); }