From 0fc1e15cac59aa2a4f11a5b6442265d57b087149 Mon Sep 17 00:00:00 2001 From: Jason Dove Date: Fri, 23 Dec 2022 15:11:05 -0600 Subject: [PATCH] colorspace fixes; song playback fixes (#1072) * fix colorspace bug, vaapi song playback * more colorspace fixes, nvidia fixes * nvidia colorspace fixes * fix some qsv output color metadata * update changelog * update changelog --- CHANGELOG.md | 3 +- .../FFmpeg/TranscodingTests.cs | 33 ++++++++++++------- .../FFmpeg/FFmpegLibraryProcessService.cs | 2 +- .../Capabilities/AmfHardwareCapabilities.cs | 4 +-- .../DefaultHardwareCapabilities.cs | 4 +-- .../HardwareCapabilitiesFactory.cs | 2 +- .../Capabilities/IHardwareCapabilities.cs | 4 +-- .../Capabilities/NoHardwareCapabilities.cs | 4 +-- .../NvidiaHardwareCapabilities.cs | 4 +-- .../Capabilities/VaapiHardwareCapabilities.cs | 6 ++-- ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs | 28 +++++++++++----- ErsatzTV.FFmpeg/FrameState.cs | 2 +- .../Pipeline/NvidiaPipelineBuilder.cs | 17 ++++++++-- .../Pipeline/QsvPipelineBuilder.cs | 12 +++---- .../Pipeline/VaapiPipelineBuilder.cs | 3 +- 15 files changed, 79 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e7d3cdc2..bbcf20583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Fixed -- Fix some transcoding failures caused by the colorspace filter +- Fix many transcoding failures caused by the colorspace filter +- Fix song playback with VAAPI ### Changed - Upgrade to dotnet 7 diff --git a/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs b/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs index aa44bea8b..39c502818 100644 --- a/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs +++ b/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs @@ -99,30 +99,30 @@ public class TranscodingTests { public static Watermark[] Watermarks = { - // Watermark.None, + Watermark.None, Watermark.PermanentOpaqueScaled, // Watermark.PermanentOpaqueActualSize, - // Watermark.PermanentTransparentScaled, + Watermark.PermanentTransparentScaled, // Watermark.PermanentTransparentActualSize }; public static Subtitle[] Subtitles = { - // Subtitle.None, - // Subtitle.Picture, + Subtitle.None, + Subtitle.Picture, Subtitle.Text }; public static Padding[] Paddings = { - // Padding.NoPadding, + Padding.NoPadding, Padding.WithPadding }; public static VideoScanKind[] VideoScanKinds = { VideoScanKind.Progressive, - // VideoScanKind.Interlaced + VideoScanKind.Interlaced }; public static InputFormat[] InputFormats = @@ -131,15 +131,15 @@ public class TranscodingTests new("libx264", "yuv420p", "tv", "smpte170m", "bt709", "smpte170m"), // new("libx264", "yuvj420p"), - // new("libx264", "yuv420p10le"), + new("libx264", "yuv420p10le"), // new("libx264", "yuv444p10le"), // new("mpeg1video", "yuv420p"), // // new("mpeg2video", "yuv420p"), - // new("libx265", "yuv420p"), - // new("libx265", "yuv420p10le"), + new("libx265", "yuv420p"), + new("libx265", "yuv420p10le"), // new("mpeg4", "yuv420p"), // @@ -156,14 +156,14 @@ public class TranscodingTests public static Resolution[] Resolutions = { - // new() { Width = 1920, Height = 1080 }, + new() { Width = 1920, Height = 1080 }, new() { Width = 1280, Height = 720 } }; public static FFmpegProfileBitDepth[] BitDepths = { FFmpegProfileBitDepth.EightBit, - // FFmpegProfileBitDepth.TenBit + FFmpegProfileBitDepth.TenBit }; public static HardwareAccelerationKind[] NoAcceleration = @@ -201,6 +201,14 @@ public class TranscodingTests { HardwareAccelerationKind.Qsv }; + + public static HardwareAccelerationKind[] LinuxTestAccelerations = + { + HardwareAccelerationKind.None, + HardwareAccelerationKind.Nvenc, + HardwareAccelerationKind.Vaapi, + HardwareAccelerationKind.Qsv + }; } [Test] @@ -222,8 +230,9 @@ public class TranscodingTests Subtitle subtitle, [ValueSource(typeof(TestData), nameof(TestData.VideoFormats))] FFmpegProfileVideoFormat profileVideoFormat, + [ValueSource(typeof(TestData), nameof(TestData.LinuxTestAccelerations))] HardwareAccelerationKind profileAcceleration) // [ValueSource(typeof(TestData), nameof(TestData.NoAcceleration))] HardwareAccelerationKind profileAcceleration) - [ValueSource(typeof(TestData), nameof(TestData.NvidiaAcceleration))] HardwareAccelerationKind profileAcceleration) + // [ValueSource(typeof(TestData), nameof(TestData.NvidiaAcceleration))] HardwareAccelerationKind profileAcceleration) // [ValueSource(typeof(TestData), nameof(TestData.VaapiAcceleration))] HardwareAccelerationKind profileAcceleration) // [ValueSource(typeof(TestData), nameof(TestData.QsvAcceleration))] HardwareAccelerationKind profileAcceleration) // [ValueSource(typeof(TestData), nameof(TestData.VideoToolboxAcceleration))] HardwareAccelerationKind profileAcceleration) diff --git a/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs b/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs index 4aba848ea..e5bc99fff 100644 --- a/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs +++ b/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs @@ -218,7 +218,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService playbackSettings.RealtimeOutput, false, // TODO: fallback filler needs to loop videoFormat, - videoStream.Profile, + Optional(videoStream.Profile), Optional(desiredPixelFormat), ffmpegVideoStream.SquarePixelFrameSize( new FrameSize(channel.FFmpegProfile.Resolution.Width, channel.FFmpegProfile.Resolution.Height)), diff --git a/ErsatzTV.FFmpeg/Capabilities/AmfHardwareCapabilities.cs b/ErsatzTV.FFmpeg/Capabilities/AmfHardwareCapabilities.cs index 3246e3d0d..a2fdb5e83 100644 --- a/ErsatzTV.FFmpeg/Capabilities/AmfHardwareCapabilities.cs +++ b/ErsatzTV.FFmpeg/Capabilities/AmfHardwareCapabilities.cs @@ -4,9 +4,9 @@ namespace ErsatzTV.FFmpeg.Capabilities; public class AmfHardwareCapabilities : IHardwareCapabilities { - public bool CanDecode(string videoFormat, string videoProfile, Option maybePixelFormat) => false; + public bool CanDecode(string videoFormat, Option videoProfile, Option maybePixelFormat) => false; - public bool CanEncode(string videoFormat, string videoProfile, Option maybePixelFormat) + public bool CanEncode(string videoFormat, Option videoProfile, Option maybePixelFormat) { int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8); diff --git a/ErsatzTV.FFmpeg/Capabilities/DefaultHardwareCapabilities.cs b/ErsatzTV.FFmpeg/Capabilities/DefaultHardwareCapabilities.cs index 7edcb72dc..568425dd5 100644 --- a/ErsatzTV.FFmpeg/Capabilities/DefaultHardwareCapabilities.cs +++ b/ErsatzTV.FFmpeg/Capabilities/DefaultHardwareCapabilities.cs @@ -4,9 +4,9 @@ namespace ErsatzTV.FFmpeg.Capabilities; public class DefaultHardwareCapabilities : IHardwareCapabilities { - public bool CanDecode(string videoFormat, string videoProfile, Option maybePixelFormat) => true; + public bool CanDecode(string videoFormat, Option videoProfile, Option maybePixelFormat) => true; - public bool CanEncode(string videoFormat, string videoProfile, Option maybePixelFormat) + public bool CanEncode(string videoFormat, Option videoProfile, Option maybePixelFormat) { int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8); diff --git a/ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs b/ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs index 08eb6b9dd..ebbdc0770 100644 --- a/ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs +++ b/ErsatzTV.FFmpeg/Capabilities/HardwareCapabilitiesFactory.cs @@ -103,7 +103,7 @@ public class HardwareCapabilitiesFactory : IHardwareCapabilitiesFactory if (profileEntrypoints.Any()) { - _logger.LogWarning( + _logger.LogInformation( "Detected {Count} VAAPI profile entrypoints for using {Driver} {Device}", profileEntrypoints.Count, driver, diff --git a/ErsatzTV.FFmpeg/Capabilities/IHardwareCapabilities.cs b/ErsatzTV.FFmpeg/Capabilities/IHardwareCapabilities.cs index 4da0be0ba..2f516addd 100644 --- a/ErsatzTV.FFmpeg/Capabilities/IHardwareCapabilities.cs +++ b/ErsatzTV.FFmpeg/Capabilities/IHardwareCapabilities.cs @@ -4,6 +4,6 @@ namespace ErsatzTV.FFmpeg.Capabilities; public interface IHardwareCapabilities { - public bool CanDecode(string videoFormat, string videoProfile, Option maybePixelFormat); - public bool CanEncode(string videoFormat, string videoProfile, Option maybePixelFormat); + public bool CanDecode(string videoFormat, Option videoProfile, Option maybePixelFormat); + public bool CanEncode(string videoFormat, Option videoProfile, Option maybePixelFormat); } diff --git a/ErsatzTV.FFmpeg/Capabilities/NoHardwareCapabilities.cs b/ErsatzTV.FFmpeg/Capabilities/NoHardwareCapabilities.cs index 73ed7aaed..9e0b01da4 100644 --- a/ErsatzTV.FFmpeg/Capabilities/NoHardwareCapabilities.cs +++ b/ErsatzTV.FFmpeg/Capabilities/NoHardwareCapabilities.cs @@ -4,6 +4,6 @@ namespace ErsatzTV.FFmpeg.Capabilities; public class NoHardwareCapabilities : IHardwareCapabilities { - public bool CanDecode(string videoFormat, string videoProfile, Option maybePixelFormat) => false; - public bool CanEncode(string videoFormat, string videoProfile, Option maybePixelFormat) => false; + public bool CanDecode(string videoFormat, Option videoProfile, Option maybePixelFormat) => false; + public bool CanEncode(string videoFormat, Option videoProfile, Option maybePixelFormat) => false; } diff --git a/ErsatzTV.FFmpeg/Capabilities/NvidiaHardwareCapabilities.cs b/ErsatzTV.FFmpeg/Capabilities/NvidiaHardwareCapabilities.cs index 0b5586093..1b5cd9eba 100644 --- a/ErsatzTV.FFmpeg/Capabilities/NvidiaHardwareCapabilities.cs +++ b/ErsatzTV.FFmpeg/Capabilities/NvidiaHardwareCapabilities.cs @@ -14,7 +14,7 @@ public class NvidiaHardwareCapabilities : IHardwareCapabilities _model = model; } - public bool CanDecode(string videoFormat, string videoProfile, Option maybePixelFormat) + public bool CanDecode(string videoFormat, Option videoProfile, Option maybePixelFormat) { int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8); @@ -36,7 +36,7 @@ public class NvidiaHardwareCapabilities : IHardwareCapabilities }; } - public bool CanEncode(string videoFormat, string videoProfile, Option maybePixelFormat) + public bool CanEncode(string videoFormat, Option videoProfile, Option maybePixelFormat) { int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8); diff --git a/ErsatzTV.FFmpeg/Capabilities/VaapiHardwareCapabilities.cs b/ErsatzTV.FFmpeg/Capabilities/VaapiHardwareCapabilities.cs index 90e5c9d09..657ed6d6e 100644 --- a/ErsatzTV.FFmpeg/Capabilities/VaapiHardwareCapabilities.cs +++ b/ErsatzTV.FFmpeg/Capabilities/VaapiHardwareCapabilities.cs @@ -15,11 +15,11 @@ public class VaapiHardwareCapabilities : IHardwareCapabilities _logger = logger; } - public bool CanDecode(string videoFormat, string videoProfile, Option maybePixelFormat) + public bool CanDecode(string videoFormat, Option videoProfile, Option maybePixelFormat) { int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8); - bool result = (videoFormat, videoProfile.ToLowerInvariant()) switch + bool result = (videoFormat, videoProfile.IfNone(string.Empty).ToLowerInvariant()) switch { // no hardware decoding of 10-bit h264 (VideoFormat.H264, _) when bitDepth == 10 => false, @@ -86,7 +86,7 @@ public class VaapiHardwareCapabilities : IHardwareCapabilities return result; } - public bool CanEncode(string videoFormat, string videoProfile, Option maybePixelFormat) + public bool CanEncode(string videoFormat, Option videoProfile, Option maybePixelFormat) { int bitDepth = maybePixelFormat.Map(pf => pf.BitDepth).IfNone(8); diff --git a/ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs b/ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs index 0e6c855d7..02cf1ce69 100644 --- a/ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs +++ b/ErsatzTV.FFmpeg/Filter/ColorspaceFilter.cs @@ -8,29 +8,30 @@ public class ColorspaceFilter : BaseFilter private readonly VideoStream _videoStream; private readonly IPixelFormat _desiredPixelFormat; private readonly bool _forceInputOverrides; - private readonly FrameDataLocation _nextDataLocation; public ColorspaceFilter( FrameState currentState, VideoStream videoStream, IPixelFormat desiredPixelFormat, - bool forceInputOverrides = false, - FrameDataLocation nextDataLocation = FrameDataLocation.Software) + bool forceInputOverrides = false) { _currentState = currentState; _videoStream = videoStream; _desiredPixelFormat = desiredPixelFormat; _forceInputOverrides = forceInputOverrides; - _nextDataLocation = nextDataLocation; } public override FrameState NextState(FrameState currentState) { - FrameState nextState = currentState with { FrameDataLocation = _nextDataLocation }; + FrameState nextState = currentState; if (!_videoStream.ColorParams.IsUnknown && _desiredPixelFormat.BitDepth is 10 or 8) { - nextState = nextState with { PixelFormat = Some(_desiredPixelFormat) }; + nextState = nextState with + { + FrameDataLocation = FrameDataLocation.Software, + PixelFormat = Some(_desiredPixelFormat) + }; } return nextState; @@ -43,7 +44,14 @@ public class ColorspaceFilter : BaseFilter string hwdownload = string.Empty; if (_currentState.FrameDataLocation == FrameDataLocation.Hardware) { - hwdownload = "hwdownload"; + hwdownload = "hwdownload,"; + foreach (IPixelFormat pixelFormat in _currentState.PixelFormat) + { + if (!string.IsNullOrWhiteSpace(pixelFormat.FFmpegName)) + { + hwdownload = $"hwdownload,format={pixelFormat.FFmpegName},"; + } + } } string inputOverrides = string.Empty; @@ -65,8 +73,10 @@ public class ColorspaceFilter : BaseFilter string colorspace = _desiredPixelFormat.BitDepth switch { _ when cp.IsUnknown => "setparams=range=tv:colorspace=bt709:color_trc=bt709:color_primaries=bt709", - 10 or 8 when !cp.IsUnknown => - $"{hwdownload},colorspace={inputOverrides}all=bt709:format={_desiredPixelFormat.FFmpegName}", + 10 when !cp.IsUnknown => + $"{hwdownload}colorspace={inputOverrides}all=bt709:format=yuv420p10", + 8 when !cp.IsUnknown => + $"{hwdownload}colorspace={inputOverrides}all=bt709:format=yuv420p", _ => string.Empty }; diff --git a/ErsatzTV.FFmpeg/FrameState.cs b/ErsatzTV.FFmpeg/FrameState.cs index e8d4ed42e..e51ed603b 100644 --- a/ErsatzTV.FFmpeg/FrameState.cs +++ b/ErsatzTV.FFmpeg/FrameState.cs @@ -6,7 +6,7 @@ public record FrameState( bool Realtime, bool InfiniteLoop, string VideoFormat, - string VideoProfile, + Option VideoProfile, Option PixelFormat, FrameSize ScaledSize, FrameSize PaddedSize, diff --git a/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs index 8c5718d84..ab8829b11 100644 --- a/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/NvidiaPipelineBuilder.cs @@ -269,7 +269,7 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder if (!videoStream.ColorParams.IsBt709) { _logger.LogDebug("Adding colorspace filter"); - var colorspace = new ColorspaceFilter(currentState, videoStream, format, false, currentState.FrameDataLocation); + var colorspace = new ColorspaceFilter(currentState, videoStream, format, false); currentState = colorspace.NextState(currentState); result.Add(colorspace); @@ -300,7 +300,20 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder if (currentState.FrameDataLocation == FrameDataLocation.Hardware) { - result.Add(new CudaFormatFilter(format)); + bool noPipelineFilters = !pipelineSteps.OfType().Any(); + bool softwareColorspace = result is [ColorspaceFilter colorspace] && + !colorspace.Filter.StartsWith("setparams="); + + bool softwareDecoder = ffmpegState.DecoderHardwareAccelerationMode == HardwareAccelerationMode.None; + + if (softwareDecoder || (noPipelineFilters && softwareColorspace)) + { + result.Add(new CudaFormatFilter(format)); + } + else + { + pipelineSteps.Add(new PixelFormatOutputOption(format)); + } } else { diff --git a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs index 44fe515e4..a47bbc42b 100644 --- a/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/QsvPipelineBuilder.cs @@ -237,20 +237,18 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder } IPixelFormat formatForDownload = pixelFormat; - - if (!videoStream.ColorParams.IsBt709) + + bool usesVppQsv = videoInputFile.FilterSteps.Any(f => f is QsvFormatFilter or ScaleQsvFilter); + if (!videoStream.ColorParams.IsBt709 || usesVppQsv) { _logger.LogDebug("Adding colorspace filter"); // vpp_qsv seems to strip color info, so if we use that at all, force overriding input color info - bool forceInputOverrides = videoInputFile.FilterSteps.Any(f => f is QsvFormatFilter or ScaleQsvFilter); - var colorspace = new ColorspaceFilter( currentState, videoStream, format, - forceInputOverrides, - currentState.FrameDataLocation); + forceInputOverrides: usesVppQsv); // force nv12 if we're still in hardware if (currentState.FrameDataLocation == FrameDataLocation.Hardware) @@ -289,7 +287,7 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder // remind qsv that it uses qsv if (currentState.FrameDataLocation == FrameDataLocation.Hardware && - result.Count == 1 && result[0] is ColorspaceFilter colorspace) + result is [ColorspaceFilter colorspace]) { if (colorspace.Filter.StartsWith("setparams=")) { diff --git a/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs b/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs index 8f61decae..49475fd57 100644 --- a/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/Pipeline/VaapiPipelineBuilder.cs @@ -250,8 +250,7 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder currentState, videoStream, format, - false, - currentState.FrameDataLocation); + false); currentState = colorspace.NextState(currentState); result.Add(colorspace); }