From 904cdb8780f9e6414d44180f2a07f169bdb7ec66 Mon Sep 17 00:00:00 2001 From: Jason Dove Date: Sat, 2 Oct 2021 13:42:44 -0500 Subject: [PATCH] vaapi improvements (#392) * add transcoding tests * dont use full paths for transcoding tests * add error details * use 1-second videos for transcoding tests * vaapi fixes * include format in scale_vaapi * more vaapi fixes * unsupported errors * fix unsupported checks * maybe not failure? * fix formatting * ignore nvdec warnings * update changelog * fix tests --- CHANGELOG.md | 1 + .../FFmpeg/FFmpegComplexFilterBuilderTests.cs | 16 +- .../FFmpeg/TranscodingTests.cs | 242 ++++++++++++++++++ ErsatzTV.Core/Domain/Resolution.cs | 2 + .../FFmpeg/FFmpegComplexFilterBuilder.cs | 16 +- 5 files changed, 268 insertions(+), 9 deletions(-) create mode 100644 ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c7b22b6d..793f36532 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Fixed - Include Specials/Season 0 `episode-num` entry in XMLTV +- Fix some transcoding edge cases with VAAPI and pixel formats `yuv420p10le`, `yuv444p10le` and `yuv444p` ## [0.0.61-alpha] - 2021-09-30 ### Fixed diff --git a/ErsatzTV.Core.Tests/FFmpeg/FFmpegComplexFilterBuilderTests.cs b/ErsatzTV.Core.Tests/FFmpeg/FFmpegComplexFilterBuilderTests.cs index 7c250b6e3..f94392097 100644 --- a/ErsatzTV.Core.Tests/FFmpeg/FFmpegComplexFilterBuilderTests.cs +++ b/ErsatzTV.Core.Tests/FFmpeg/FFmpegComplexFilterBuilderTests.cs @@ -437,7 +437,7 @@ namespace ErsatzTV.Core.Tests.FFmpeg true, true, false, - "[0:0]deinterlace_vaapi,scale_vaapi=w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,hwupload[v]", + "[0:0]deinterlace_vaapi,scale_vaapi=format=nv12:w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,hwupload[v]", "[v]")] [TestCase( "h264", @@ -451,14 +451,14 @@ namespace ErsatzTV.Core.Tests.FFmpeg true, true, true, - "[0:0]deinterlace_vaapi,scale_vaapi=w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,hwupload[v]", + "[0:0]deinterlace_vaapi,scale_vaapi=format=nv12:w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,hwupload[v]", "[v]")] [TestCase( "h264", false, true, false, - "[0:0]scale_vaapi=w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,hwupload[v]", + "[0:0]scale_vaapi=format=nv12:w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,hwupload[v]", "[v]")] [TestCase( "h264", @@ -472,7 +472,7 @@ namespace ErsatzTV.Core.Tests.FFmpeg false, true, true, - "[0:0]scale_vaapi=w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,hwupload[v]", + "[0:0]scale_vaapi=format=nv12:w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,hwupload[v]", "[v]")] [TestCase("mpeg4", true, false, false, "[0:0]hwupload,deinterlace_vaapi[v]", "[v]")] [TestCase( @@ -480,7 +480,7 @@ namespace ErsatzTV.Core.Tests.FFmpeg true, true, false, - "[0:0]hwupload,deinterlace_vaapi,scale_vaapi=w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,hwupload[v]", + "[0:0]hwupload,deinterlace_vaapi,scale_vaapi=format=nv12:w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,hwupload[v]", "[v]")] [TestCase( "mpeg4", @@ -494,14 +494,14 @@ namespace ErsatzTV.Core.Tests.FFmpeg true, true, true, - "[0:0]hwupload,deinterlace_vaapi,scale_vaapi=w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,hwupload[v]", + "[0:0]hwupload,deinterlace_vaapi,scale_vaapi=format=nv12:w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,hwupload[v]", "[v]")] [TestCase( "mpeg4", false, true, false, - "[0:0]hwupload,scale_vaapi=w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,hwupload[v]", + "[0:0]hwupload,scale_vaapi=format=nv12:w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,hwupload[v]", "[v]")] [TestCase( "mpeg4", @@ -515,7 +515,7 @@ namespace ErsatzTV.Core.Tests.FFmpeg false, true, true, - "[0:0]hwupload,scale_vaapi=w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,hwupload[v]", + "[0:0]hwupload,scale_vaapi=format=nv12:w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,hwupload[v]", "[v]")] public void Should_Return_VAAPI_Video_Filter( string codec, diff --git a/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs b/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs new file mode 100644 index 000000000..3604c7afa --- /dev/null +++ b/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs @@ -0,0 +1,242 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using ErsatzTV.Core.Domain; +using ErsatzTV.Core.FFmpeg; +using ErsatzTV.Core.Interfaces.FFmpeg; +using ErsatzTV.Core.Interfaces.Images; +using ErsatzTV.Core.Interfaces.Repositories; +using ErsatzTV.Core.Metadata; +using FluentAssertions; +using LanguageExt; +using Microsoft.Extensions.Logging; +using Moq; +using NUnit.Framework; +using static LanguageExt.Prelude; + +namespace ErsatzTV.Core.Tests.FFmpeg +{ + [TestFixture] + [Explicit] + public class TranscodingTests + { + [Test] + [Explicit] + public void DeleteTestVideos() + { + foreach (string file in Directory.GetFiles(TestContext.CurrentContext.TestDirectory, "*.mkv")) + { + File.Delete(file); + } + + Assert.Pass(); + } + + private class TestData + { + public static string[] InputCodecs = + { + "h264", + "mpeg2video", + "hevc", + "mpeg4" + }; + + public static string[] InputPixelFormats = + { + "yuv420p", + "yuv420p10le", + "yuvj420p", + "yuv444p", + "yuv444p10le" + }; + + public static Resolution[] Resolutions = + { + new() { Width = 1920, Height = 1080 }, + new() { Width = 1280, Height = 720 } + }; + + public static string[] SoftwareCodecs = + { + "libx264", + "libx265" + }; + + public static HardwareAccelerationKind[] NoAcceleration = + { + HardwareAccelerationKind.None + }; + + public static string[] NvidiaCodecs = + { + "h264_nvenc", + "hevc_nvenc" + }; + + public static HardwareAccelerationKind[] NvidiaAcceleration = + { + HardwareAccelerationKind.Nvenc + }; + + public static string[] VaapiCodecs = + { + "h264_vaapi", + "hevc_vaapi" + }; + + public static HardwareAccelerationKind[] VaapiAcceleration = + { + HardwareAccelerationKind.Vaapi + }; + } + + [Test, Combinatorial] + public async Task Transcode( + [ValueSource(typeof(TestData), nameof(TestData.InputCodecs))] + string inputCodec, + [ValueSource(typeof(TestData), nameof(TestData.InputPixelFormats))] + string inputPixelFormat, + [ValueSource(typeof(TestData), nameof(TestData.Resolutions))] + Resolution profileResolution, + // [ValueSource(typeof(TestData), nameof(TestData.SoftwareCodecs))] string profileCodec, + // [ValueSource(typeof(TestData), nameof(TestData.NoAcceleration))] HardwareAccelerationKind profileAcceleration) + [ValueSource(typeof(TestData), nameof(TestData.NvidiaCodecs))] string profileCodec, + [ValueSource(typeof(TestData), nameof(TestData.NvidiaAcceleration))] HardwareAccelerationKind profileAcceleration) + // [ValueSource(typeof(TestData), nameof(TestData.VaapiCodecs))] string profileCodec, + // [ValueSource(typeof(TestData), nameof(TestData.VaapiAcceleration))] HardwareAccelerationKind profileAcceleration) + { + string name = GetStringSha256Hash( + $"{inputCodec}_{inputPixelFormat}_{profileResolution}_{profileCodec}_{profileAcceleration}"); + + string file = Path.Combine(TestContext.CurrentContext.TestDirectory, $"{name}.mkv"); + if (!File.Exists(file)) + { + var args = + $"-y -f lavfi -i anullsrc=channel_layout=stereo:sample_rate=44100 -f lavfi -i testsrc=duration=1:size=1920x1080:rate=30 -c:a aac -c:v {inputCodec} -shortest -pix_fmt {inputPixelFormat} -strict -2 {file}"; + var p1 = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = "ffmpeg", + Arguments = args + } + }; + + p1.Start(); + await p1.WaitForExitAsync(); + p1.ExitCode.Should().Be(0); + } + + var service = new FFmpegProcessService( + new FFmpegPlaybackSettingsCalculator(), + new FakeStreamSelector(), + new Mock().Object); + + MediaVersion v = new MediaVersion(); + + var metadataRepository = new Mock(); + metadataRepository + .Setup(r => r.UpdateLocalStatistics(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, version, _) => v = version); + + var localStatisticsProvider = new LocalStatisticsProvider( + metadataRepository.Object, + new LocalFileSystem(), + new Mock>().Object); + + await localStatisticsProvider.RefreshStatistics( + "ffprobe", + new Movie + { + MediaVersions = new List + { + new() + { + MediaFiles = new List + { + new() { Path = file } + } + } + } + }); + + DateTimeOffset now = DateTimeOffset.Now; + + Process process = await service.ForPlayoutItem( + "ffmpeg", + false, + new Channel(Guid.NewGuid()) + { + FFmpegProfile = FFmpegProfile.New("test", profileResolution) with + { + HardwareAcceleration = profileAcceleration, + VideoCodec = profileCodec + }, + StreamingMode = StreamingMode.TransportStream + }, + v, + file, + now, + now, + None, + None); + + process.StartInfo.RedirectStandardError = true; + + process.Start().Should().BeTrue(); + + await process.StandardOutput.ReadToEndAsync(); + string error = await process.StandardError.ReadToEndAsync(); + await process.WaitForExitAsync(); + + string[] unsupportedMessages = + { + "No support for codec", + "No usable", + "Provided device doesn't support" + }; + + if (profileAcceleration != HardwareAccelerationKind.None && unsupportedMessages.Any(error.Contains)) + { + IEnumerable quotedArgs = process.StartInfo.ArgumentList.Map(a => $"\'{a}\'"); + process.ExitCode.Should().Be(1, $"Error message with successful exit code? {string.Join(" ", quotedArgs)}"); + Assert.Warn("Unsupported on this hardware"); + } + else if (error.Contains("Impossible to convert between")) + { + IEnumerable quotedArgs = process.StartInfo.ArgumentList.Map(a => $"\'{a}\'"); + Assert.Fail($"Transcode failure: ffmpeg {string.Join(" ", quotedArgs)}"); + } + else + { + process.ExitCode.Should().Be(0, error); + } + } + + private static string GetStringSha256Hash(string text) + { + if (string.IsNullOrEmpty(text)) + { + return string.Empty; + } + + using var sha = new System.Security.Cryptography.SHA256Managed(); + byte[] textData = System.Text.Encoding.UTF8.GetBytes(text); + byte[] hash = sha.ComputeHash(textData); + return BitConverter.ToString(hash).Replace("-", string.Empty); + } + + private class FakeStreamSelector : IFFmpegStreamSelector + { + public Task SelectVideoStream(Channel channel, MediaVersion version) => + version.Streams.First(s => s.MediaStreamKind == MediaStreamKind.Video).AsTask(); + + public Task> SelectAudioStream(Channel channel, MediaVersion version) => + Optional(version.Streams.First(s => s.MediaStreamKind == MediaStreamKind.Audio)).AsTask(); + } + } +} diff --git a/ErsatzTV.Core/Domain/Resolution.cs b/ErsatzTV.Core/Domain/Resolution.cs index 7baea1333..f34af22f1 100644 --- a/ErsatzTV.Core/Domain/Resolution.cs +++ b/ErsatzTV.Core/Domain/Resolution.cs @@ -8,5 +8,7 @@ namespace ErsatzTV.Core.Domain public string Name { get; set; } public int Height { get; set; } public int Width { get; set; } + + public override string ToString() => $"{Width}x{Height}"; } } diff --git a/ErsatzTV.Core/FFmpeg/FFmpegComplexFilterBuilder.cs b/ErsatzTV.Core/FFmpeg/FFmpegComplexFilterBuilder.cs index 68dd0665f..3c91928d0 100644 --- a/ErsatzTV.Core/FFmpeg/FFmpegComplexFilterBuilder.cs +++ b/ErsatzTV.Core/FFmpeg/FFmpegComplexFilterBuilder.cs @@ -129,6 +129,20 @@ namespace ErsatzTV.Core.FFmpeg } } + string[] h264hevc = { "h264", "hevc" }; + + if (acceleration == HardwareAccelerationKind.Vaapi && (_pixelFormat ?? string.Empty).EndsWith("p10le") && + h264hevc.Contains(_inputCodec) + && (_pixelFormat != "yuv420p10le" || _inputCodec != "hevc")) + { + videoFilterQueue.Add("format=p010le,format=nv12|vaapi,hwupload"); + } + + if (acceleration == HardwareAccelerationKind.Vaapi && _pixelFormat == "yuv444p" && h264hevc.Contains(_inputCodec)) + { + videoFilterQueue.Add("format=nv12|vaapi,hwupload"); + } + _scaleToSize.IfSome( size => { @@ -138,7 +152,7 @@ namespace ErsatzTV.Core.FFmpeg HardwareAccelerationKind.Nvenc when _pixelFormat == "yuv420p10le" => $"hwdownload,format=p010le,format=nv12,hwupload,scale_npp={size.Width}:{size.Height}", HardwareAccelerationKind.Nvenc => $"scale_npp={size.Width}:{size.Height}", - HardwareAccelerationKind.Vaapi => $"scale_vaapi=w={size.Width}:h={size.Height}", + HardwareAccelerationKind.Vaapi => $"scale_vaapi=format=nv12:w={size.Width}:h={size.Height}", _ => $"scale={size.Width}:{size.Height}:flags=fast_bilinear" };