pipeline fixes when colorspace filter is used (#1068)

* fix colorspace filter with missing input transfer or input primaries

* properly download before applying colorspace filter

* fix extra hwupload/hwdownload with nvidia pipeline

* colorspace tests

* update dependencies
This commit is contained in:
Jason Dove
2022-12-20 20:12:27 -06:00
committed by GitHub
parent 93673fce03
commit d2040eaac9
14 changed files with 84 additions and 38 deletions
@@ -16,7 +16,7 @@
<PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="7.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
<PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.4.27">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+48 -24
View File
@@ -35,7 +35,7 @@ namespace ErsatzTV.Core.Tests.FFmpeg;
public class TranscodingTests
{
private static readonly ILoggerFactory LoggerFactory;
private static readonly MemoryCache _memoryCache;
private static readonly MemoryCache MemoryCache;
static TranscodingTests()
{
@@ -46,7 +46,7 @@ public class TranscodingTests
LoggerFactory = new LoggerFactory().AddSerilog(Log.Logger);
_memoryCache = new MemoryCache(new MemoryCacheOptions());
MemoryCache = new MemoryCache(new MemoryCacheOptions());
}
[Test]
@@ -61,7 +61,13 @@ public class TranscodingTests
Assert.Pass();
}
public record InputFormat(string Encoder, string PixelFormat);
public record InputFormat(
string Encoder,
string PixelFormat,
string ColorRange = "tv",
string ColorSpace = "bt709",
string ColorTransfer = "bt709",
string ColorPrimaries = "bt709");
public enum Padding
{
@@ -93,45 +99,47 @@ public class TranscodingTests
{
public static Watermark[] Watermarks =
{
Watermark.None,
// Watermark.None,
Watermark.PermanentOpaqueScaled,
Watermark.PermanentOpaqueActualSize,
Watermark.PermanentTransparentScaled,
Watermark.PermanentTransparentActualSize
// Watermark.PermanentOpaqueActualSize,
// Watermark.PermanentTransparentScaled,
// Watermark.PermanentTransparentActualSize
};
public static Subtitle[] Subtitles =
{
Subtitle.None,
Subtitle.Picture,
// Subtitle.None,
// Subtitle.Picture,
Subtitle.Text
};
public static Padding[] Paddings =
{
Padding.NoPadding,
// Padding.NoPadding,
Padding.WithPadding
};
public static VideoScanKind[] VideoScanKinds =
{
VideoScanKind.Progressive,
VideoScanKind.Interlaced
// VideoScanKind.Interlaced
};
public static InputFormat[] InputFormats =
{
new("libx264", "yuv420p"),
// example format that requires colorspace filter
new("libx264", "yuv420p", "tv", "smpte170m", "bt709", "smpte170m"),
// new("libx264", "yuvj420p"),
new("libx264", "yuv420p10le"),
// new("libx264", "yuv420p10le"),
// new("libx264", "yuv444p10le"),
// new("mpeg1video", "yuv420p"),
//
// new("mpeg2video", "yuv420p"),
new("libx265", "yuv420p"),
new("libx265", "yuv420p10le"),
// new("libx265", "yuv420p"),
// new("libx265", "yuv420p10le"),
// new("mpeg4", "yuv420p"),
//
@@ -148,14 +156,14 @@ public class TranscodingTests
public static Resolution[] Resolutions =
{
new() { Width = 1920, Height = 1080 },
// new() { Width = 1920, Height = 1080 },
new() { Width = 1280, Height = 720 }
};
public static FFmpegProfileBitDepth[] BitDepths =
{
FFmpegProfileBitDepth.EightBit,
FFmpegProfileBitDepth.TenBit
// FFmpegProfileBitDepth.TenBit
};
public static HardwareAccelerationKind[] NoAcceleration =
@@ -166,7 +174,7 @@ public class TranscodingTests
public static FFmpegProfileVideoFormat[] VideoFormats =
{
FFmpegProfileVideoFormat.H264,
FFmpegProfileVideoFormat.Hevc
// FFmpegProfileVideoFormat.Hevc
};
public static HardwareAccelerationKind[] NvidiaAcceleration =
@@ -215,8 +223,8 @@ public class TranscodingTests
[ValueSource(typeof(TestData), nameof(TestData.VideoFormats))]
FFmpegProfileVideoFormat profileVideoFormat,
// [ValueSource(typeof(TestData), nameof(TestData.NoAcceleration))] HardwareAccelerationKind profileAcceleration)
// [ValueSource(typeof(TestData), nameof(TestData.NvidiaAcceleration))] HardwareAccelerationKind profileAcceleration)
[ValueSource(typeof(TestData), nameof(TestData.VaapiAcceleration))] HardwareAccelerationKind profileAcceleration)
[ValueSource(typeof(TestData), nameof(TestData.NvidiaAcceleration))] HardwareAccelerationKind profileAcceleration)
// [ValueSource(typeof(TestData), nameof(TestData.VaapiAcceleration))] HardwareAccelerationKind profileAcceleration)
// [ValueSource(typeof(TestData), nameof(TestData.QsvAcceleration))] HardwareAccelerationKind profileAcceleration)
// [ValueSource(typeof(TestData), nameof(TestData.VideoToolboxAcceleration))] HardwareAccelerationKind profileAcceleration)
// [ValueSource(typeof(TestData), nameof(TestData.AmfAcceleration))] HardwareAccelerationKind profileAcceleration)
@@ -230,7 +238,7 @@ public class TranscodingTests
}
}
string name = GetStringSha256Hash($"{inputFormat.Encoder}_{inputFormat.PixelFormat}_{videoScanKind}_{padding}_{subtitle}");
string name = GetStringSha256Hash($"{inputFormat}_{videoScanKind}_{padding}_{subtitle}");
string file = Path.Combine(TestContext.CurrentContext.TestDirectory, $"{name}.mkv");
if (!File.Exists(file))
@@ -242,8 +250,24 @@ public class TranscodingTests
: string.Empty;
string flags = videoScanKind == VideoScanKind.Interlaced ? "-field_order tt -flags +ildct+ilme" : string.Empty;
string colorRange = !string.IsNullOrWhiteSpace(inputFormat.ColorRange)
? $" -color_range {inputFormat.ColorRange}"
: string.Empty;
string colorSpace = !string.IsNullOrWhiteSpace(inputFormat.ColorSpace)
? $" -colorspace {inputFormat.ColorSpace}"
: string.Empty;
string colorTransfer = !string.IsNullOrWhiteSpace(inputFormat.ColorTransfer)
? $" -color_trc {inputFormat.ColorTransfer}"
: string.Empty;
string colorPrimaries = !string.IsNullOrWhiteSpace(inputFormat.ColorPrimaries)
? $" -color_primaries {inputFormat.ColorPrimaries}"
: string.Empty;
string args =
$"-y -f lavfi -i anoisesrc=color=brown -f lavfi -i testsrc=duration=1:size={resolution}:rate=30 {videoFilter} -c:a aac -c:v {inputFormat.Encoder} -shortest -pix_fmt {inputFormat.PixelFormat} -strict -2 {flags} {file}";
$"-y -f lavfi -i anoisesrc=color=brown -f lavfi -i testsrc=duration=1:size={resolution}:rate=30 {videoFilter} -c:a aac -c:v {inputFormat.Encoder}{colorRange}{colorSpace}{colorTransfer}{colorPrimaries} -shortest -pix_fmt {inputFormat.PixelFormat} -strict -2 {flags} {file}";
var p1 = new Process
{
StartInfo = new ProcessStartInfo
@@ -321,7 +345,7 @@ public class TranscodingTests
imageCache.Object,
new Mock<ITempFilePool>().Object,
new Mock<IClient>().Object,
_memoryCache,
MemoryCache,
LoggerFactory.CreateLogger<FFmpegProcessService>());
var service = new FFmpegLibraryProcessService(
@@ -333,7 +357,7 @@ public class TranscodingTests
new RuntimeInfo(),
//new FakeNvidiaCapabilitiesFactory(),
new HardwareCapabilitiesFactory(
_memoryCache,
MemoryCache,
LoggerFactory.CreateLogger<HardwareCapabilitiesFactory>()),
LoggerFactory.CreateLogger<PipelineBuilderFactory>()),
LoggerFactory.CreateLogger<FFmpegLibraryProcessService>());
+1 -1
View File
@@ -9,7 +9,7 @@
<ItemGroup>
<PackageReference Include="Bugsnag" Version="3.1.0" />
<PackageReference Include="Destructurama.Attributed" Version="3.0.0" />
<PackageReference Include="Flurl" Version="3.0.6" />
<PackageReference Include="Flurl" Version="3.0.7" />
<PackageReference Include="LanguageExt.Core" Version="4.4.0" />
<PackageReference Include="LanguageExt.Transformers" Version="4.4.0" />
<PackageReference Include="MediatR" Version="11.1.0" />
@@ -9,7 +9,7 @@
<ItemGroup>
<PackageReference Include="FluentAssertions" Version="6.8.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
<PackageReference Include="Moq" Version="4.18.3" />
<PackageReference Include="NUnit" Version="3.13.3" />
<PackageReference Include="NUnit3TestAdapter" Version="4.3.1" />
+17 -2
View File
@@ -4,17 +4,20 @@ namespace ErsatzTV.FFmpeg.Filter;
public class ColorspaceFilter : BaseFilter
{
private readonly FrameState _currentState;
private readonly VideoStream _videoStream;
private readonly IPixelFormat _desiredPixelFormat;
private readonly bool _forceInputOverrides;
private readonly FrameDataLocation _nextDataLocation;
public ColorspaceFilter(
FrameState currentState,
VideoStream videoStream,
IPixelFormat desiredPixelFormat,
bool forceInputOverrides = false,
FrameDataLocation nextDataLocation = FrameDataLocation.Software)
{
_currentState = currentState;
_videoStream = videoStream;
_desiredPixelFormat = desiredPixelFormat;
_forceInputOverrides = forceInputOverrides;
@@ -37,21 +40,33 @@ public class ColorspaceFilter : BaseFilter
{
get
{
string hwdownload = string.Empty;
if (_currentState.FrameDataLocation == FrameDataLocation.Hardware)
{
hwdownload = "hwdownload";
}
string inputOverrides = string.Empty;
ColorParams cp = _videoStream.ColorParams;
if (cp.IsMixed || _forceInputOverrides)
{
string range = string.IsNullOrWhiteSpace(cp.ColorRange) ? "tv" : cp.ColorRange;
string transfer = string.IsNullOrWhiteSpace(cp.ColorTransfer)
? "bt709"
: cp.ColorTransfer;
string primaries = string.IsNullOrWhiteSpace(cp.ColorPrimaries)
? "bt709"
: cp.ColorPrimaries;
inputOverrides =
$"irange={range}:ispace={cp.ColorSpace}:itrc={cp.ColorTransfer}:iprimaries={cp.ColorPrimaries}:";
$"irange={range}:ispace={cp.ColorSpace}:itrc={transfer}:iprimaries={primaries}:";
}
string colorspace = _desiredPixelFormat.BitDepth switch
{
_ when cp.IsUnknown => "setparams=range=tv:colorspace=bt709:color_trc=bt709:color_primaries=bt709",
10 or 8 when !cp.IsUnknown =>
$"colorspace={inputOverrides}all=bt709:format={_desiredPixelFormat.FFmpegName}",
$"{hwdownload},colorspace={inputOverrides}all=bt709:format={_desiredPixelFormat.FFmpegName}",
_ => string.Empty
};
+1 -1
View File
@@ -317,7 +317,7 @@ public class ComplexFilter : IPipelineStep
if (!videoStream.ColorParams.IsBt709)
{
_logger.LogDebug("Adding colorspace filter");
var colorspace = new ColorspaceFilter(videoStream, pixelFormat);
var colorspace = new ColorspaceFilter(_currentState, videoStream, pixelFormat);
_pipelineSteps.Add(colorspace);
filter = colorspace.Filter;
}
@@ -93,7 +93,7 @@ public class AmfPipelineBuilder : SoftwarePipelineBuilder
if (!videoStream.ColorParams.IsBt709)
{
_logger.LogDebug("Adding colorspace filter");
var colorspace = new ColorspaceFilter(videoStream, pixelFormat);
var colorspace = new ColorspaceFilter(currentState, videoStream, pixelFormat);
currentState = colorspace.NextState(currentState);
result.Add(colorspace);
}
@@ -181,7 +181,8 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
// need to upload for any sort of overlay
if (currentState.FrameDataLocation == FrameDataLocation.Software &&
currentState.BitDepth == 8 && (context.HasSubtitleOverlay || context.HasWatermark))
currentState.BitDepth == 8 && context.HasSubtitleText == false
&& (context.HasSubtitleOverlay || context.HasWatermark))
{
var hardwareUpload = new HardwareUploadCudaFilter(currentState);
currentState = hardwareUpload.NextState(currentState);
@@ -268,7 +269,7 @@ public class NvidiaPipelineBuilder : SoftwarePipelineBuilder
if (!videoStream.ColorParams.IsBt709)
{
_logger.LogDebug("Adding colorspace filter");
var colorspace = new ColorspaceFilter(videoStream, format, false, currentState.FrameDataLocation);
var colorspace = new ColorspaceFilter(currentState, videoStream, format, false, currentState.FrameDataLocation);
currentState = colorspace.NextState(currentState);
result.Add(colorspace);
@@ -246,6 +246,7 @@ public class QsvPipelineBuilder : SoftwarePipelineBuilder
bool forceInputOverrides = videoInputFile.FilterSteps.Any(f => f is QsvFormatFilter or ScaleQsvFilter);
var colorspace = new ColorspaceFilter(
currentState,
videoStream,
format,
forceInputOverrides,
@@ -140,7 +140,7 @@ public class SoftwarePipelineBuilder : PipelineBuilderBase
if (!videoStream.ColorParams.IsBt709)
{
_logger.LogDebug("Adding colorspace filter");
var colorspace = new ColorspaceFilter(videoStream, pixelFormat);
var colorspace = new ColorspaceFilter(currentState, videoStream, pixelFormat);
currentState = colorspace.NextState(currentState);
result.Add(colorspace);
}
@@ -246,7 +246,12 @@ public class VaapiPipelineBuilder : SoftwarePipelineBuilder
if (!videoStream.ColorParams.IsBt709)
{
_logger.LogDebug("Adding colorspace filter");
var colorspace = new ColorspaceFilter(videoStream, format, false, currentState.FrameDataLocation);
var colorspace = new ColorspaceFilter(
currentState,
videoStream,
format,
false,
currentState.FrameDataLocation);
currentState = colorspace.NextState(currentState);
result.Add(colorspace);
}
@@ -114,7 +114,7 @@ public class VideoToolboxPipelineBuilder : SoftwarePipelineBuilder
if (!videoStream.ColorParams.IsBt709)
{
_logger.LogDebug("Adding colorspace filter");
var colorspace = new ColorspaceFilter(videoStream, pixelFormat);
var colorspace = new ColorspaceFilter(currentState, videoStream, pixelFormat);
currentState = colorspace.NextState(currentState);
result.Add(colorspace);
}
@@ -10,7 +10,7 @@
<ItemGroup>
<PackageReference Include="FluentAssertions" Version="6.8.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
<PackageReference Include="Moq" Version="4.18.3" />
<PackageReference Include="NUnit" Version="3.13.3" />
<PackageReference Include="NUnit3TestAdapter" Version="4.3.1" />
+1 -1
View File
@@ -70,7 +70,7 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="MudBlazor" Version="6.1.5" />
<PackageReference Include="MudBlazor" Version="6.1.6" />
<PackageReference Include="NaturalSort.Extension" Version="4.0.0" />
<PackageReference Include="PPioli.FluentValidation.Blazor" Version="11.1.0" />
<PackageReference Include="Refit.HttpClientFactory" Version="6.3.2" />