From 176f136c23edf7b21fa8da057d822eeaddf3b3cc Mon Sep 17 00:00:00 2001
From: Jason Dove <jason@jasondove.me>
Date: Fri, 7 Jan 2022 18:53:28 -0600
Subject: [PATCH] fix some nvenc edge cases where only padding is needed for
 normalization (#565)

---
 CHANGELOG.md                                       |  1 +
 ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs     | 10 ++++++++--
 ErsatzTV.Core/FFmpeg/FFmpegComplexFilterBuilder.cs |  5 +++++
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b566eb871..aea42b91e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [Unreleased]
 ### Fixed
 - Properly index `minutes` field when adding new items during scan (vs when rebuilding index)
+- Fix some nvenc edge cases where only padding is needed for normalization
 
 ### Added
 - Add music video `artist` to search index
diff --git a/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs b/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs
index ea986db7a..44d0fb3a3 100644
--- a/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs
+++ b/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs
@@ -104,6 +104,8 @@ namespace ErsatzTV.Core.Tests.FFmpeg
             string inputPixelFormat,
             [ValueSource(typeof(TestData), nameof(TestData.Resolutions))]
             Resolution profileResolution,
+            [Values(true, false)]
+            bool pad,
             // [ValueSource(typeof(TestData), nameof(TestData.SoftwareCodecs))] string profileCodec,
             // [ValueSource(typeof(TestData), nameof(TestData.NoAcceleration))] HardwareAccelerationKind profileAcceleration)
             [ValueSource(typeof(TestData), nameof(TestData.NvidiaCodecs))] string profileCodec,
@@ -112,13 +114,15 @@ namespace ErsatzTV.Core.Tests.FFmpeg
             // [ValueSource(typeof(TestData), nameof(TestData.VaapiAcceleration))] HardwareAccelerationKind profileAcceleration)
         {
             string name = GetStringSha256Hash(
-                $"{inputCodec}_{inputPixelFormat}_{profileResolution}_{profileCodec}_{profileAcceleration}");
+                $"{inputCodec}_{inputPixelFormat}_{pad}_{profileResolution}_{profileCodec}_{profileAcceleration}");
 
             string file = Path.Combine(TestContext.CurrentContext.TestDirectory, $"{name}.mkv");
             if (!File.Exists(file))
             {
+                string resolution = pad ? "1920x1060" : "1920x1080";
+                
                 var args =
-                    $"-y -f lavfi -i anullsrc=channel_layout=stereo:sample_rate=44100 -f lavfi -i testsrc=duration=1:size=1920x1080:rate=30 -c:a aac -c:v {inputCodec} -shortest -pix_fmt {inputPixelFormat} -strict -2 {file}";
+                    $"-y -f lavfi -i anullsrc=channel_layout=stereo:sample_rate=44100 -f lavfi -i testsrc=duration=1:size={resolution}:rate=30 -c:a aac -c:v {inputCodec} -shortest -pix_fmt {inputPixelFormat} -strict -2 {file}";
                 var p1 = new Process
                 {
                     StartInfo = new ProcessStartInfo
@@ -200,6 +204,8 @@ namespace ErsatzTV.Core.Tests.FFmpeg
                 TimeSpan.FromSeconds(5));
 
             process.StartInfo.RedirectStandardError = true;
+            
+            // Console.WriteLine($"ffmpeg arguments {string.Join(" ", process.StartInfo.ArgumentList)}");
 
             process.Start().Should().BeTrue();
 
diff --git a/ErsatzTV.Core/FFmpeg/FFmpegComplexFilterBuilder.cs b/ErsatzTV.Core/FFmpeg/FFmpegComplexFilterBuilder.cs
index 8d52504d8..06fa4815a 100644
--- a/ErsatzTV.Core/FFmpeg/FFmpegComplexFilterBuilder.cs
+++ b/ErsatzTV.Core/FFmpeg/FFmpegComplexFilterBuilder.cs
@@ -142,6 +142,11 @@ namespace ErsatzTV.Core.FFmpeg
             bool isHardwareDecode = acceleration switch
             {
                 HardwareAccelerationKind.Vaapi => !isSong && _inputCodec != "mpeg4",
+                
+                // we need an initial hwupload_cuda when only padding with these pixel formats
+                HardwareAccelerationKind.Nvenc when _scaleToSize.IsNone && _padToSize.IsSome =>
+                    !isSong && !_pixelFormat.Contains("p10le") && !_pixelFormat.Contains("444"),
+
                 HardwareAccelerationKind.Nvenc => !isSong,
                 HardwareAccelerationKind.Qsv => !isSong,
                 _ => false