update changelog for release 62 [no ci]

* add transcoding tests

* dont use full paths for transcoding tests

* add error details

* use 1-second videos for transcoding tests

* vaapi fixes

* include format in scale_vaapi

* more vaapi fixes

* unsupported errors

* fix unsupported checks

* maybe not failure?

* fix formatting

* ignore nvdec warnings

* update changelog

* fix tests

CHANGELOG.md

@@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
 
+## [0.0.62-alpha] - 2021-10-05
+### Added
+- Support IMDB ids from Plex libraries, which may improve Trakt matching for some items
+
+### Fixed
+- Include Specials/Season 0 `episode-num` entry in XMLTV
+- Fix some transcoding edge cases with VAAPI and pixel formats `yuv420p10le`, `yuv444p10le` and `yuv444p`
+- Update Plex movie and episode paths when they are changed within Plex
+- Always use `libx264` software encoder for error messages
+
 ## [0.0.61-alpha] - 2021-09-30
 ### Fixed
 - Revert nvenc/cuda filter change from v60
@@ -634,7 +644,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Initial release to facilitate testing outside of Docker.
 
 
-[Unreleased]: https://github.com/jasongdove/ErsatzTV/compare/v0.0.61-alpha...HEAD
+[Unreleased]: https://github.com/jasongdove/ErsatzTV/compare/v0.0.62-alpha...HEAD
+[0.0.62-alpha]: https://github.com/jasongdove/ErsatzTV/compare/v0.0.61-alpha...v0.0.62-alpha
 [0.0.61-alpha]: https://github.com/jasongdove/ErsatzTV/compare/v0.0.60-alpha...v0.0.61-alpha
 [0.0.60-alpha]: https://github.com/jasongdove/ErsatzTV/compare/v0.0.59-alpha...v0.0.60-alpha
 [0.0.59-alpha]: https://github.com/jasongdove/ErsatzTV/compare/v0.0.58-alpha...v0.0.59-alpha

ErsatzTV.Core.Tests/FFmpeg/FFmpegComplexFilterBuilderTests.cs

@@ -437,7 +437,7 @@ namespace ErsatzTV.Core.Tests.FFmpeg
                 true,
                 true,
                 false,
-                "[0:0]deinterlace_vaapi,scale_vaapi=w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,hwupload[v]",
+                "[0:0]deinterlace_vaapi,scale_vaapi=format=nv12:w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,hwupload[v]",
                 "[v]")]
             [TestCase(
                 "h264",
@@ -451,14 +451,14 @@ namespace ErsatzTV.Core.Tests.FFmpeg
                 true,
                 true,
                 true,
-                "[0:0]deinterlace_vaapi,scale_vaapi=w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,hwupload[v]",
+                "[0:0]deinterlace_vaapi,scale_vaapi=format=nv12:w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,hwupload[v]",
                 "[v]")]
             [TestCase(
                 "h264",
                 false,
                 true,
                 false,
-                "[0:0]scale_vaapi=w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,hwupload[v]",
+                "[0:0]scale_vaapi=format=nv12:w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,hwupload[v]",
                 "[v]")]
             [TestCase(
                 "h264",
@@ -472,7 +472,7 @@ namespace ErsatzTV.Core.Tests.FFmpeg
                 false,
                 true,
                 true,
-                "[0:0]scale_vaapi=w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,hwupload[v]",
+                "[0:0]scale_vaapi=format=nv12:w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,hwupload[v]",
                 "[v]")]
             [TestCase("mpeg4", true, false, false, "[0:0]hwupload,deinterlace_vaapi[v]", "[v]")]
             [TestCase(
@@ -480,7 +480,7 @@ namespace ErsatzTV.Core.Tests.FFmpeg
                 true,
                 true,
                 false,
-                "[0:0]hwupload,deinterlace_vaapi,scale_vaapi=w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,hwupload[v]",
+                "[0:0]hwupload,deinterlace_vaapi,scale_vaapi=format=nv12:w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,hwupload[v]",
                 "[v]")]
             [TestCase(
                 "mpeg4",
@@ -494,14 +494,14 @@ namespace ErsatzTV.Core.Tests.FFmpeg
                 true,
                 true,
                 true,
-                "[0:0]hwupload,deinterlace_vaapi,scale_vaapi=w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,hwupload[v]",
+                "[0:0]hwupload,deinterlace_vaapi,scale_vaapi=format=nv12:w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,hwupload[v]",
                 "[v]")]
             [TestCase(
                 "mpeg4",
                 false,
                 true,
                 false,
-                "[0:0]hwupload,scale_vaapi=w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,hwupload[v]",
+                "[0:0]hwupload,scale_vaapi=format=nv12:w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,hwupload[v]",
                 "[v]")]
             [TestCase(
                 "mpeg4",
@@ -515,7 +515,7 @@ namespace ErsatzTV.Core.Tests.FFmpeg
                 false,
                 true,
                 true,
-                "[0:0]hwupload,scale_vaapi=w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,hwupload[v]",
+                "[0:0]hwupload,scale_vaapi=format=nv12:w=1920:h=1000,hwdownload,format=nv12|vaapi,setsar=1,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,hwupload[v]",
                 "[v]")]
             public void Should_Return_VAAPI_Video_Filter(
                 string codec,

ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs

@@ -0,0 +1,242 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using ErsatzTV.Core.Domain;
+using ErsatzTV.Core.FFmpeg;
+using ErsatzTV.Core.Interfaces.FFmpeg;
+using ErsatzTV.Core.Interfaces.Images;
+using ErsatzTV.Core.Interfaces.Repositories;
+using ErsatzTV.Core.Metadata;
+using FluentAssertions;
+using LanguageExt;
+using Microsoft.Extensions.Logging;
+using Moq;
+using NUnit.Framework;
+using static LanguageExt.Prelude;
+
+namespace ErsatzTV.Core.Tests.FFmpeg
+{
+    [TestFixture]
+    [Explicit]
+    public class TranscodingTests
+    {
+        [Test]
+        [Explicit]
+        public void DeleteTestVideos()
+        {
+            foreach (string file in Directory.GetFiles(TestContext.CurrentContext.TestDirectory, "*.mkv"))
+            {
+                File.Delete(file);
+            }
+
+            Assert.Pass();
+        }
+
+        private class TestData
+        {
+            public static string[] InputCodecs =
+            {
+                "h264",
+                "mpeg2video",
+                "hevc",
+                "mpeg4"
+            };
+
+            public static string[] InputPixelFormats =
+            {
+                "yuv420p",
+                "yuv420p10le",
+                "yuvj420p",
+                "yuv444p",
+                "yuv444p10le"
+            };
+            
+            public static Resolution[] Resolutions =
+            {
+                new() { Width = 1920, Height = 1080 },
+                new() { Width = 1280, Height = 720 }
+            };
+
+            public static string[] SoftwareCodecs =
+            {
+                "libx264",
+                "libx265"
+            };
+
+            public static HardwareAccelerationKind[] NoAcceleration =
+            {
+                HardwareAccelerationKind.None
+            };
+
+            public static string[] NvidiaCodecs =
+            {
+                "h264_nvenc",
+                "hevc_nvenc"
+            };
+
+            public static HardwareAccelerationKind[] NvidiaAcceleration =
+            {
+                HardwareAccelerationKind.Nvenc
+            };
+
+            public static string[] VaapiCodecs =
+            {
+                "h264_vaapi",
+                "hevc_vaapi"
+            };
+
+            public static HardwareAccelerationKind[] VaapiAcceleration =
+            {
+                HardwareAccelerationKind.Vaapi
+            };
+        }
+
+        [Test, Combinatorial]
+        public async Task Transcode(
+            [ValueSource(typeof(TestData), nameof(TestData.InputCodecs))]
+            string inputCodec,
+            [ValueSource(typeof(TestData), nameof(TestData.InputPixelFormats))]
+            string inputPixelFormat,
+            [ValueSource(typeof(TestData), nameof(TestData.Resolutions))]
+            Resolution profileResolution,
+            // [ValueSource(typeof(TestData), nameof(TestData.SoftwareCodecs))] string profileCodec,
+            // [ValueSource(typeof(TestData), nameof(TestData.NoAcceleration))] HardwareAccelerationKind profileAcceleration)
+            [ValueSource(typeof(TestData), nameof(TestData.NvidiaCodecs))] string profileCodec,
+            [ValueSource(typeof(TestData), nameof(TestData.NvidiaAcceleration))] HardwareAccelerationKind profileAcceleration)
+            // [ValueSource(typeof(TestData), nameof(TestData.VaapiCodecs))] string profileCodec,
+            // [ValueSource(typeof(TestData), nameof(TestData.VaapiAcceleration))] HardwareAccelerationKind profileAcceleration)
+        {
+            string name = GetStringSha256Hash(
+                $"{inputCodec}_{inputPixelFormat}_{profileResolution}_{profileCodec}_{profileAcceleration}");
+
+            string file = Path.Combine(TestContext.CurrentContext.TestDirectory, $"{name}.mkv");
+            if (!File.Exists(file))
+            {
+                var args =
+                    $"-y -f lavfi -i anullsrc=channel_layout=stereo:sample_rate=44100 -f lavfi -i testsrc=duration=1:size=1920x1080:rate=30 -c:a aac -c:v {inputCodec} -shortest -pix_fmt {inputPixelFormat} -strict -2 {file}";
+                var p1 = new Process
+                {
+                    StartInfo = new ProcessStartInfo
+                    {
+                        FileName = "ffmpeg",
+                        Arguments = args
+                    }
+                };
+
+                p1.Start();
+                await p1.WaitForExitAsync();
+                p1.ExitCode.Should().Be(0);
+            }
+
+            var service = new FFmpegProcessService(
+                new FFmpegPlaybackSettingsCalculator(),
+                new FakeStreamSelector(),
+                new Mock<IImageCache>().Object);
+
+            MediaVersion v = new MediaVersion();
+
+            var metadataRepository = new Mock<IMetadataRepository>();
+            metadataRepository
+                .Setup(r => r.UpdateLocalStatistics(It.IsAny<int>(), It.IsAny<MediaVersion>(), It.IsAny<bool>()))
+                .Callback<int, MediaVersion, bool>((_, version, _) => v = version);
+
+            var localStatisticsProvider = new LocalStatisticsProvider(
+                metadataRepository.Object,
+                new LocalFileSystem(),
+                new Mock<ILogger<LocalStatisticsProvider>>().Object);
+
+            await localStatisticsProvider.RefreshStatistics(
+                "ffprobe",
+                new Movie
+                {
+                    MediaVersions = new List<MediaVersion>
+                    {
+                        new()
+                        {
+                            MediaFiles = new List<MediaFile>
+                            {
+                                new() { Path = file }
+                            }
+                        }
+                    }
+                });
+
+            DateTimeOffset now = DateTimeOffset.Now;
+
+            Process process = await service.ForPlayoutItem(
+                "ffmpeg",
+                false,
+                new Channel(Guid.NewGuid())
+                {
+                    FFmpegProfile = FFmpegProfile.New("test", profileResolution) with
+                    {
+                        HardwareAcceleration = profileAcceleration,
+                        VideoCodec = profileCodec
+                    },
+                    StreamingMode = StreamingMode.TransportStream
+                },
+                v,
+                file,
+                now,
+                now,
+                None,
+                None);
+
+            process.StartInfo.RedirectStandardError = true;
+
+            process.Start().Should().BeTrue();
+
+            await process.StandardOutput.ReadToEndAsync();
+            string error = await process.StandardError.ReadToEndAsync();
+            await process.WaitForExitAsync();
+
+            string[] unsupportedMessages =
+            {
+                "No support for codec",
+                "No usable",
+                "Provided device doesn't support"
+            };
+            
+            if (profileAcceleration != HardwareAccelerationKind.None && unsupportedMessages.Any(error.Contains))
+            {
+                IEnumerable<string> quotedArgs = process.StartInfo.ArgumentList.Map(a => $"\'{a}\'");
+                process.ExitCode.Should().Be(1, $"Error message with successful exit code? {string.Join(" ", quotedArgs)}");
+                Assert.Warn("Unsupported on this hardware");
+            }
+            else if (error.Contains("Impossible to convert between"))
+            {
+                IEnumerable<string> quotedArgs = process.StartInfo.ArgumentList.Map(a => $"\'{a}\'");
+                Assert.Fail($"Transcode failure: ffmpeg {string.Join(" ", quotedArgs)}");
+            }
+            else
+            {
+                process.ExitCode.Should().Be(0, error);
+            }
+        }
+        
+        private static string GetStringSha256Hash(string text)
+        {
+            if (string.IsNullOrEmpty(text))
+            {
+                return string.Empty;
+            }
+
+            using var sha = new System.Security.Cryptography.SHA256Managed();
+            byte[] textData = System.Text.Encoding.UTF8.GetBytes(text);
+            byte[] hash = sha.ComputeHash(textData);
+            return BitConverter.ToString(hash).Replace("-", string.Empty);
+        }
+
+        private class FakeStreamSelector : IFFmpegStreamSelector
+        {
+            public Task<MediaStream> SelectVideoStream(Channel channel, MediaVersion version) =>
+                version.Streams.First(s => s.MediaStreamKind == MediaStreamKind.Video).AsTask();
+
+            public Task<Option<MediaStream>> SelectAudioStream(Channel channel, MediaVersion version) =>
+                Optional(version.Streams.First(s => s.MediaStreamKind == MediaStreamKind.Audio)).AsTask();
+        }
+    }
+}

ErsatzTV.Core.Tests/Fakes/FakeTelevisionRepository.cs

@@ -97,5 +97,6 @@ namespace ErsatzTV.Core.Tests.Fakes
         public Task<bool> AddDirector(EpisodeMetadata metadata, Director director) => throw new NotSupportedException();
 
         public Task<bool> AddWriter(EpisodeMetadata metadata, Writer writer) => throw new NotSupportedException();
+        public Task<Unit> UpdatePath(int mediaFileId, string path) => throw new NotSupportedException();
     }
 }

ErsatzTV.Core/Domain/Resolution.cs

@@ -8,5 +8,7 @@ namespace ErsatzTV.Core.Domain
         public string Name { get; set; }
         public int Height { get; set; }
         public int Width { get; set; }
+
+        public override string ToString() => $"{Width}x{Height}";
     }
 }

ErsatzTV.Core/FFmpeg/FFmpegComplexFilterBuilder.cs

@@ -129,6 +129,20 @@ namespace ErsatzTV.Core.FFmpeg
                 }
             }
 
+            string[] h264hevc = { "h264", "hevc" };
+
+            if (acceleration == HardwareAccelerationKind.Vaapi && (_pixelFormat ?? string.Empty).EndsWith("p10le") &&
+                h264hevc.Contains(_inputCodec)
+                && (_pixelFormat != "yuv420p10le" || _inputCodec != "hevc"))
+            {
+                videoFilterQueue.Add("format=p010le,format=nv12|vaapi,hwupload");
+            }
+
+            if (acceleration == HardwareAccelerationKind.Vaapi && _pixelFormat == "yuv444p" && h264hevc.Contains(_inputCodec))
+            {
+                videoFilterQueue.Add("format=nv12|vaapi,hwupload");
+            }
+
             _scaleToSize.IfSome(
                 size =>
                 {
@@ -138,7 +152,7 @@ namespace ErsatzTV.Core.FFmpeg
                         HardwareAccelerationKind.Nvenc when _pixelFormat == "yuv420p10le" =>
                             $"hwdownload,format=p010le,format=nv12,hwupload,scale_npp={size.Width}:{size.Height}",
                         HardwareAccelerationKind.Nvenc => $"scale_npp={size.Width}:{size.Height}",
-                        HardwareAccelerationKind.Vaapi => $"scale_vaapi=w={size.Width}:h={size.Height}",
+                        HardwareAccelerationKind.Vaapi => $"scale_vaapi=format=nv12:w={size.Width}:h={size.Height}",
                         _ => $"scale={size.Width}:{size.Height}:flags=fast_bilinear"
                     };
 

ErsatzTV.Core/FFmpeg/FFmpegPlaybackSettingsCalculator.cs

@@ -147,8 +147,8 @@ namespace ErsatzTV.Core.FFmpeg
             {
                 ThreadCount = ffmpegProfile.ThreadCount,
                 FormatFlags = CommonFormatFlags,
-                VideoCodec = ffmpegProfile.VideoCodec,
-                AudioCodec = ffmpegProfile.AudioCodec
+                VideoCodec = "libx264",
+                AudioCodec = ffmpegProfile.AudioCodec,
             };
 
         private static bool NeedToScale(FFmpegProfile ffmpegProfile, MediaVersion version) =>

ErsatzTV.Core/Interfaces/Repositories/IMovieRepository.cs

@@ -33,5 +33,6 @@ namespace ErsatzTV.Core.Interfaces.Repositories
         Task<Option<EmbyMovie>> UpdateEmby(EmbyMovie movie);
         Task<bool> AddDirector(MovieMetadata metadata, Director director);
         Task<bool> AddWriter(MovieMetadata metadata, Writer writer);
+        Task<Unit> UpdatePath(int mediaFileId, string path);
     }
 }

ErsatzTV.Core/Interfaces/Repositories/ITelevisionRepository.cs

@@ -51,5 +51,6 @@ namespace ErsatzTV.Core.Interfaces.Repositories
         Task<Unit> RemoveMetadata(Episode episode, EpisodeMetadata metadata);
         Task<bool> AddDirector(EpisodeMetadata metadata, Director director);
         Task<bool> AddWriter(EpisodeMetadata metadata, Writer writer);
+        Task<Unit> UpdatePath(int mediaFileId, string path);
     }
 }

ErsatzTV.Core/Iptv/ChannelGuide.cs

@@ -200,10 +200,10 @@ namespace ErsatzTV.Core.Iptv
 
                         if (!isSameCustomShow)
                         {
-                            int s = Optional(episode.Season?.SeasonNumber).IfNone(0);
+                            int s = Optional(episode.Season?.SeasonNumber).IfNone(-1);
                             // TODO: multi-episode?
                             int e = episode.EpisodeMetadata.Head().EpisodeNumber;
-                            if (s > 0 && e > 0)
+                            if (s >= 0 && e > 0)
                             {
                                 xml.WriteStartElement("episode-num");
                                 xml.WriteAttributeString("system", "onscreen");

ErsatzTV.Core/Plex/PlexMovieLibraryScanner.cs

@@ -156,6 +156,24 @@ namespace ErsatzTV.Core.Plex
             MediaVersion existingVersion = existing.MediaVersions.Head();
             MediaVersion incomingVersion = incoming.MediaVersions.Head();
 
+            foreach (MediaFile incomingFile in incomingVersion.MediaFiles.HeadOrNone())
+            {
+                foreach (MediaFile existingFile in existingVersion.MediaFiles.HeadOrNone())
+                {
+                    if (incomingFile.Path != existingFile.Path)
+                    {
+                        _logger.LogDebug(
+                            "Plex movie has moved from {OldPath} to {NewPath}",
+                            existingFile.Path,
+                            incomingFile.Path);
+
+                        existingFile.Path = incomingFile.Path;
+
+                        await _movieRepository.UpdatePath(existingFile.Id, incomingFile.Path);
+                    }
+                }
+            }
+
             if (incomingVersion.DateUpdated > existingVersion.DateUpdated || !existingVersion.Streams.Any())
             {
                 string localPath = _plexPathReplacementService.GetReplacementPlexPath(

ErsatzTV.Core/Plex/PlexTelevisionLibraryScanner.cs

@@ -514,6 +514,24 @@ namespace ErsatzTV.Core.Plex
         {
             MediaVersion existingVersion = existing.MediaVersions.Head();
             MediaVersion incomingVersion = incoming.MediaVersions.Head();
+            
+            foreach (MediaFile incomingFile in incomingVersion.MediaFiles.HeadOrNone())
+            {
+                foreach (MediaFile existingFile in existingVersion.MediaFiles.HeadOrNone())
+                {
+                    if (incomingFile.Path != existingFile.Path)
+                    {
+                        _logger.LogDebug(
+                            "Plex episode has moved from {OldPath} to {NewPath}",
+                            existingFile.Path,
+                            incomingFile.Path);
+
+                        existingFile.Path = incomingFile.Path;
+
+                        await _televisionRepository.UpdatePath(existingFile.Id, incomingFile.Path);
+                    }
+                }
+            }
 
             if (incomingVersion.DateUpdated > existingVersion.DateUpdated || !existingVersion.Streams.Any())
             {

ErsatzTV.Infrastructure/Data/Repositories/MovieRepository.cs

@@ -774,6 +774,11 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
                 "INSERT INTO Writer (Name, MovieMetadataId) VALUES (@Name, @MetadataId)",
                 new { writer.Name, MetadataId = metadata.Id }).Map(result => result > 0);
 
+        public Task<Unit> UpdatePath(int mediaFileId, string path) =>
+            _dbConnection.ExecuteAsync(
+                "UPDATE MediaFile SET Path = @Path WHERE Id = @MediaFileId",
+                new { Path = path, MediaFileId = mediaFileId }).Map(_ => Unit.Default);
+
         private static async Task<Either<BaseError, MediaItemScanResult<Movie>>> AddMovie(
             TvContext dbContext,
             int libraryPathId,

ErsatzTV.Infrastructure/Data/Repositories/TelevisionRepository.cs

@@ -568,6 +568,11 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
                 "INSERT INTO Writer (Name, EpisodeMetadataId) VALUES (@Name, @MetadataId)",
                 new { writer.Name, MetadataId = metadata.Id }).Map(result => result > 0);
 
+        public Task<Unit> UpdatePath(int mediaFileId, string path) =>
+            _dbConnection.ExecuteAsync(
+                "UPDATE MediaFile SET Path = @Path WHERE Id = @MediaFileId",
+                new { Path = path, MediaFileId = mediaFileId }).Map(_ => Unit.Default);
+
         public async Task<List<Episode>> GetShowItems(int showId)
         {
             IEnumerable<int> ids = await _dbConnection.QueryAsync<int>(

ErsatzTV.Infrastructure/Plex/PlexServerApiClient.cs

@@ -801,6 +801,13 @@ namespace ErsatzTV.Infrastructure.Plex
             {
                 return guid;
             }
+            
+            if (guid.StartsWith("com.plexapp.agents.imdb"))
+            {
+                string strip1 = guid.Replace("com.plexapp.agents.imdb://", string.Empty);
+                string strip2 = strip1.Split("?").Head();
+                return $"imdb://{strip2}";
+            }
 
             if (guid.StartsWith("com.plexapp.agents.thetvdb"))
             {