From 5e530b9301d7f6ef882ddb9227696dd23bada6bd Mon Sep 17 00:00:00 2001 From: Jason Dove <1695733+jasongdove@users.noreply.github.com> Date: Fri, 12 Jan 2024 13:21:49 -0600 Subject: [PATCH] fix scale behavior crop with qsv (#1546) --- CHANGELOG.md | 2 +- ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs | 10 +- .../Pipeline/QsvPipelineBuilder.cs | 3 + .../Core/FFmpeg/TranscodingTests.cs | 150 ++++++++++-------- .../Core/Metadata/LocalFolderScanner.cs | 2 +- 5 files changed, 98 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e89488e3f..ec747a432 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Unlock playout when playout build fails - Ignore errors deleting old HLS segments; this should improve stream reliability - Update show year when changed within Plex -- Fix crop scale mode with NVIDIA acceleration +- Fix crop scale behavior with NVIDIA, QSV acceleration ### Changed - Upgrade from .NET 7 to .NET 8 diff --git a/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs index 9779efdd8..93cadb302 100644 --- a/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs @@ -9,16 +9,22 @@ public class ScaleQsvFilter : BaseFilter private readonly bool _isAnamorphicEdgeCase; private readonly string _sampleAspectRatio; private readonly FrameSize _scaledSize; + private readonly FrameSize _paddedSize; + private readonly Option _croppedSize; public ScaleQsvFilter( FrameState currentState, FrameSize scaledSize, + FrameSize paddedSize, + Option croppedSize, int extraHardwareFrames, bool isAnamorphicEdgeCase, string sampleAspectRatio) { _currentState = currentState; _scaledSize = scaledSize; + _paddedSize = paddedSize; + _croppedSize = croppedSize; _extraHardwareFrames = extraHardwareFrames; _isAnamorphicEdgeCase = isAnamorphicEdgeCase; _sampleAspectRatio = sampleAspectRatio; @@ -43,7 +49,9 @@ public class ScaleQsvFilter : BaseFilter else { string squareScale = string.Empty; - var targetSize = $"w={_scaledSize.Width}:h={_scaledSize.Height}"; + var targetSize = _croppedSize.IsSome + ? $"w={_paddedSize.Width}:h={_paddedSize.Height}" + : $"w={_scaledSize.Width}:h={_scaledSize.Height}"; string format = string.Empty; foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) { diff --git a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs index 4ee6b00ba..8a8f8013b 100644 --- a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs @@ -165,6 +165,7 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder // _logger.LogDebug("After scale: {PixelFormat}", currentState.PixelFormat); currentState = SetPad(videoInputFile, videoStream, desiredState, currentState); // _logger.LogDebug("After pad: {PixelFormat}", currentState.PixelFormat); + currentState = SetCrop(videoInputFile, desiredState, currentState); // need to download for any sort of overlay if (currentState.FrameDataLocation == FrameDataLocation.Hardware && @@ -572,6 +573,8 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder : Option.None }, desiredState.ScaledSize, + desiredState.PaddedSize, + desiredState.CroppedSize, ffmpegState.QsvExtraHardwareFrames, VideoStream.IsAnamorphicEdgeCase, videoStream.SampleAspectRatio); diff --git a/ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs b/ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs index 9c77bf214..33c5dc422 100644 --- a/ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs +++ b/ErsatzTV.Scanner.Tests/Core/FFmpeg/TranscodingTests.cs @@ -110,51 +110,58 @@ public class TranscodingTests private class TestData { public static Watermark[] Watermarks = - { - Watermark.None, - Watermark.PermanentOpaqueScaled, + [ + Watermark.None + //Watermark.PermanentOpaqueScaled, // Watermark.PermanentOpaqueActualSize, - Watermark.PermanentTransparentScaled + //Watermark.PermanentTransparentScaled // Watermark.PermanentTransparentActualSize - }; + ]; public static Subtitle[] Subtitles = - { - Subtitle.None, - Subtitle.Picture, - Subtitle.Text - }; + [ + Subtitle.None + //Subtitle.Picture, + //Subtitle.Text + ]; public static Padding[] Paddings = - { + [ Padding.NoPadding, Padding.WithPadding - }; + ]; + + public static ScalingBehavior[] ScalingBehaviors = + [ + ScalingBehavior.ScaleAndPad, + ScalingBehavior.Crop, + //ScalingBehavior.Stretch + ]; public static VideoScanKind[] VideoScanKinds = - { - VideoScanKind.Progressive, - VideoScanKind.Interlaced - }; + [ + VideoScanKind.Progressive + //VideoScanKind.Interlaced + ]; public static InputFormat[] InputFormats = - { + [ // // // example format that requires colorspace filter - new("libx264", "yuv420p", "tv", "smpte170m", "bt709", "smpte170m"), - // // - // // // example format that requires setparams filter - new("libx264", "yuv420p", string.Empty, string.Empty, string.Empty, string.Empty), - // // - // // // new("libx264", "yuvj420p"), - new("libx264", "yuv420p10le"), - // // // new("libx264", "yuv444p10le"), - // // - // // // new("mpeg1video", "yuv420p"), + // new("libx264", "yuv420p", "tv", "smpte170m", "bt709", "smpte170m"), // // // - new("mpeg2video", "yuv420p"), + // // // // example format that requires setparams filter + // new("libx264", "yuv420p", string.Empty, string.Empty, string.Empty, string.Empty), + // // // + // // // // new("libx264", "yuvj420p"), + // new("libx264", "yuv420p10le"), + // // // // new("libx264", "yuv444p10le"), + // // // + // // // // new("mpeg1video", "yuv420p"), + // // // // + // new("mpeg2video", "yuv420p"), // // - new("libx265", "yuv420p"), - new("libx265", "yuv420p10le") + new InputFormat("libx265", "yuv420p") + // new("libx265", "yuv420p10le") // // new("mpeg4", "yuv420p"), // @@ -168,38 +175,39 @@ public class TranscodingTests // new("msmpeg4v3", "yuv420p") // // // wmv3 yuv420p 1 - }; + ]; public static Resolution[] Resolutions = - { - new() { Width = 1920, Height = 1080 }, - new() { Width = 1280, Height = 720 } - }; + [ + new Resolution { Width = 1920, Height = 1080 }, + new Resolution { Width = 1280, Height = 720 }, + new Resolution { Width = 640, Height = 480 } + ]; public static FFmpegProfileBitDepth[] BitDepths = - { - FFmpegProfileBitDepth.EightBit, - FFmpegProfileBitDepth.TenBit - }; + [ + FFmpegProfileBitDepth.EightBit + //FFmpegProfileBitDepth.TenBit + ]; public static FFmpegProfileVideoFormat[] VideoFormats = - { + [ FFmpegProfileVideoFormat.H264, FFmpegProfileVideoFormat.Hevc // FFmpegProfileVideoFormat.Mpeg2Video - }; + ]; public static HardwareAccelerationKind[] TestAccelerations = - { + [ HardwareAccelerationKind.None, - // HardwareAccelerationKind.Nvenc, - HardwareAccelerationKind.Vaapi - // HardwareAccelerationKind.Qsv + HardwareAccelerationKind.Nvenc, + //HardwareAccelerationKind.Vaapi, + HardwareAccelerationKind.Qsv, // HardwareAccelerationKind.VideoToolbox, // HardwareAccelerationKind.Amf - }; + ]; - public static string[] FilesToTest => new[] { string.Empty }; + public static string[] FilesToTest => [string.Empty]; } [Test] @@ -388,6 +396,8 @@ public class TranscodingTests FFmpegProfileBitDepth profileBitDepth, [ValueSource(typeof(TestData), nameof(TestData.Paddings))] Padding padding, + [ValueSource(typeof(TestData), nameof(TestData.ScalingBehaviors))] + ScalingBehavior scalingBehavior, [ValueSource(typeof(TestData), nameof(TestData.VideoScanKinds))] VideoScanKind videoScanKind, [ValueSource(typeof(TestData), nameof(TestData.Watermarks))] @@ -415,22 +425,19 @@ public class TranscodingTests } } - string name = GetStringSha256Hash($"{inputFormat}_{videoScanKind}_{padding}_{subtitle}"); + string name = GetStringSha256Hash($"{inputFormat}_{videoScanKind}_{padding}_{scalingBehavior}_{subtitle}"); file = Path.Combine(TestContext.CurrentContext.TestDirectory, $"{name}.mkv"); if (!File.Exists(file)) { - await GenerateTestFile(inputFormat, padding, videoScanKind, subtitle, file); + await GenerateTestFile(inputFormat, padding, scalingBehavior, videoScanKind, subtitle, file); } } var v = new MediaVersion { - MediaFiles = new List - { - new() { Path = file } - }, - Streams = new List() + MediaFiles = [new MediaFile { Path = file }], + Streams = [] }; IMetadataRepository? metadataRepository = Substitute.For(); @@ -552,7 +559,7 @@ public class TranscodingTests .Any(); // TODO: sometimes scaling is used for pixel format, so this is harder to assert the absence - if (profileResolution.Width != 1920) + if (profileResolution.Width != 1920 && profileResolution.Width != 640) { hasScaling.Should().BeTrue(); } @@ -563,11 +570,17 @@ public class TranscodingTests // TODO: optimize out padding // hasPadding.Should().Be(padding == Padding.WithPadding); - if (padding == Padding.WithPadding) + if (padding is Padding.WithPadding && scalingBehavior is not ScalingBehavior.Crop) { hasPadding.Should().BeTrue(); } + bool hasCrop = filterChain.VideoFilterSteps.Any(s => s is CropFilter); + if (scalingBehavior is ScalingBehavior.Crop) + { + hasCrop.Should().BeTrue(); + } + bool hasSubtitleFilters = filterChain.VideoFilterSteps.Any(s => s is SubtitlesFilter) || filterChain.SubtitleOverlayFilterSteps.Any( @@ -599,7 +612,8 @@ public class TranscodingTests VideoFormat = profileVideoFormat, AudioFormat = FFmpegProfileAudioFormat.Aac, DeinterlaceVideo = true, - BitDepth = profileBitDepth + BitDepth = profileBitDepth, + ScalingBehavior = scalingBehavior }, StreamingMode = StreamingMode.TransportStream, SubtitleMode = subtitleMode @@ -711,11 +725,18 @@ public class TranscodingTests private static async Task GenerateTestFile( InputFormat inputFormat, Padding padding, + ScalingBehavior scalingBehavior, VideoScanKind videoScanKind, Subtitle subtitle, string file) { - string resolution = padding == Padding.WithPadding ? "1920x1060" : "1920x1080"; + string resolution = (scalingBehavior, padding) switch + { + (ScalingBehavior.Crop, Padding.NoPadding) => "1920x1080", + // TODO: (ScalingBehavior.Crop, Padding.WithPadding) => "632x480", + (ScalingBehavior.Stretch or ScalingBehavior.ScaleAndPad, Padding.WithPadding) => "1920x1060", + _ => "1920x1080" + }; string videoFilter = videoScanKind == VideoScanKind.Interlaced ? "-vf interlace=scan=tff:lowpass=complex" @@ -954,16 +975,13 @@ public class TranscodingTests ExecutableName("ffprobe"), new Movie { - MediaVersions = new List - { - new() + MediaVersions = + [ + new MediaVersion { - MediaFiles = new List - { - new() { Path = tempFile } - } + MediaFiles = [new MediaFile { Path = tempFile }] } - } + ] }); MediaVersion v = getFinalMediaVersion(); diff --git a/ErsatzTV.Scanner/Core/Metadata/LocalFolderScanner.cs b/ErsatzTV.Scanner/Core/Metadata/LocalFolderScanner.cs index be22d80e8..05e5951e4 100644 --- a/ErsatzTV.Scanner/Core/Metadata/LocalFolderScanner.cs +++ b/ErsatzTV.Scanner/Core/Metadata/LocalFolderScanner.cs @@ -21,7 +21,7 @@ public abstract class LocalFolderScanner public static readonly ImmutableHashSet VideoFileExtensions = new[] { ".mpg", ".mp2", ".mpeg", ".mpe", ".mpv", ".ogg", ".mp4", - ".m4p", ".m4v", ".avi", ".wmv", ".mov", ".mkv", ".ts", ".webm" + ".m4p", ".m4v", ".avi", ".wmv", ".mov", ".mkv", ".m2ts", ".ts", ".webm" }.ToImmutableHashSet(StringComparer.OrdinalIgnoreCase); public static readonly ImmutableHashSet AudioFileExtensions = new[]