ffmpeg lib fixes (#633)

* try to fix vaapi inconsistencies

* log unexpected data

* vaapi fixes

* disable 444 test

* add qsv deinterlace filter; qsv fixes

* add videotoolbox acceleration

ErsatzTV.Core.Tests/ErsatzTV.Core.Tests.csproj

@@ -9,6 +9,8 @@
       <PackageReference Include="FluentAssertions" Version="6.4.0" />
       <PackageReference Include="LanguageExt.Core" Version="4.0.3" />
       <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
+      <PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
+      <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="6.0.0" />
       <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
       <PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.0.64">
         <PrivateAssets>all</PrivateAssets>

ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs

@@ -66,7 +66,7 @@ namespace ErsatzTV.Core.Tests.FFmpeg
                 new("libx264", "yuv420p"),
                 new("libx264", "yuvj420p"),
                 new("libx264", "yuv420p10le"),
-                new("libx264", "yuv444p10le"),
+                // new("libx264", "yuv444p10le"),
 
                 new("mpeg1video", "yuv420p"),
 
@@ -137,6 +137,17 @@ namespace ErsatzTV.Core.Tests.FFmpeg
             {
                 HardwareAccelerationKind.VideoToolbox
             };
+
+            public static string[] QsvCodecs =
+            {
+                "h264_qsv",
+                "hevc_qsv"
+            };
+
+            public static HardwareAccelerationKind[] QsvAcceleration =
+            {
+                HardwareAccelerationKind.Qsv
+            };
         }
 
         [Test, Combinatorial]
@@ -155,6 +166,8 @@ namespace ErsatzTV.Core.Tests.FFmpeg
             [ValueSource(typeof(TestData), nameof(TestData.NvidiaAcceleration))] HardwareAccelerationKind profileAcceleration)
             // [ValueSource(typeof(TestData), nameof(TestData.VaapiCodecs))] string profileCodec,
             // [ValueSource(typeof(TestData), nameof(TestData.VaapiAcceleration))] HardwareAccelerationKind profileAcceleration)
+            // [ValueSource(typeof(TestData), nameof(TestData.QsvCodecs))] string profileCodec,
+            // [ValueSource(typeof(TestData), nameof(TestData.QsvAcceleration))] HardwareAccelerationKind profileAcceleration)
             // [ValueSource(typeof(TestData), nameof(TestData.VideoToolboxCodecs))] string profileCodec,
             // [ValueSource(typeof(TestData), nameof(TestData.VideoToolboxAcceleration))] HardwareAccelerationKind profileAcceleration)
         {

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -73,19 +73,16 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             hlsRealtime,
             targetFramerate);
 
+        var ffmpegVideoStream = new VideoStream(
+            videoStream.Index,
+            videoStream.Codec,
+            AvailablePixelFormats.ForPixelFormat(videoStream.PixelFormat, _logger),
+            new FrameSize(videoVersion.Width, videoVersion.Height),
+            videoVersion.RFrameRate);
+
         var inputFiles = new List<InputFile>
         {
-            new(
-                videoVersion.MediaFiles.Head().Path,
-                new List<ErsatzTV.FFmpeg.MediaStream>
-                {
-                    new VideoStream(
-                        videoStream.Index,
-                        videoStream.Codec,
-                        Some(AvailablePixelFormats.ForPixelFormat(videoStream.PixelFormat)),
-                        new FrameSize(videoVersion.Width, videoVersion.Height),
-                        videoVersion.RFrameRate)
-                })
+            new(videoVersion.MediaFiles.Head().Path, new List<ErsatzTV.FFmpeg.MediaStream> { ffmpegVideoStream })
         };
 
         foreach (MediaStream audioStream in maybeAudioStream)
@@ -97,8 +94,8 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
         // TODO: need formats for these codecs
         string videoFormat = channel.FFmpegProfile.VideoCodec switch
         {
-            "libx265" or "hevc_nvenc" or "hevc_qsv" or "hevc_vaapi" => VideoFormat.Hevc,
-            "libx264" or "h264_nvenc" or "h264_qsv" or "h264_vaapi" => VideoFormat.H264,
+            "libx265" or "hevc_nvenc" or "hevc_qsv" or "hevc_vaapi" or "hevc_videotoolbox" => VideoFormat.Hevc,
+            "libx264" or "h264_nvenc" or "h264_qsv" or "h264_vaapi" or "h264_videotoolbox" => VideoFormat.H264,
             "mpeg2video" => VideoFormat.Mpeg2Video,
             _ => throw new ArgumentOutOfRangeException($"unexpected video codec {channel.FFmpegProfile.VideoCodec}")
         };
@@ -116,6 +113,14 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             ? OutputFormatKind.Hls
             : OutputFormatKind.MpegTs;
 
+        Option<string> hlsPlaylistPath = outputFormat == OutputFormatKind.Hls
+            ? Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live.m3u8")
+            : Option<string>.None;
+
+        Option<string> hlsSegmentTemplate = outputFormat == OutputFormatKind.Hls
+            ? Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live%06d.ts")
+            : Option<string>.None;
+
         var desiredState = new FrameState(
             hwAccel,
             playbackSettings.RealtimeOutput,
@@ -123,7 +128,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             playbackSettings.StreamSeek,
             finish - now,
             videoFormat,
-            Some(AvailablePixelFormats.ForPixelFormat(videoStream.PixelFormat)),
+            ffmpegVideoStream.PixelFormat,
             await playbackSettings.ScaledSize.Map(ss => new FrameSize(ss.Width, ss.Height))
                 .IfNoneAsync(new FrameSize(videoVersion.Width, videoVersion.Height)),
             new FrameSize(channel.FFmpegProfile.Resolution.Width, channel.FFmpegProfile.Resolution.Height),
@@ -144,10 +149,12 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             channel.Name,
             maybeAudioStream.Map(s => Optional(s.Language)).Flatten(),
             outputFormat,
-            Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live.m3u8"),
-            Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live%06d.ts"),
+            hlsPlaylistPath,
+            hlsSegmentTemplate,
             ptsOffset);
 
+        _logger.LogDebug("FFmpeg desired state {FrameState}", desiredState);
+
         return GetProcess(ffmpegPath, inputFiles, desiredState);
     }
 
@@ -215,6 +222,8 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
 
         IList<IPipelineStep> pipelineSteps = pipelineBuilder.Build(desiredState);
 
+        _logger.LogDebug("FFmpeg pipeline {PipelineSteps}", pipelineSteps);
+
         IList<string> arguments = CommandGenerator.GenerateArguments(inputFiles, pipelineSteps);
 
         var startInfo = new ProcessStartInfo

ErsatzTV.FFmpeg/Decoder/AvailableDecoders.cs

@@ -2,12 +2,13 @@
 using ErsatzTV.FFmpeg.Decoder.Qsv;
 using ErsatzTV.FFmpeg.Format;
 using LanguageExt;
+using Microsoft.Extensions.Logging;
 
 namespace ErsatzTV.FFmpeg.Decoder;
 
 public static class AvailableDecoders
 {
-    public static Option<IDecoder> ForVideoFormat(FrameState currentState, FrameState desiredState)
+    public static Option<IDecoder> ForVideoFormat(FrameState currentState, FrameState desiredState, ILogger logger)
     {
         return (currentState.HardwareAccelerationMode, currentState.VideoFormat,
                 currentState.PixelFormat.Match(pf => pf.Name, () => string.Empty)) switch
@@ -27,6 +28,10 @@ public static class AvailableDecoders
                 // hevc_qsv decoder sometimes causes green lines with 10-bit content
                 (HardwareAccelerationMode.Qsv, VideoFormat.Hevc, PixelFormat.YUV420P10LE) => new DecoderHevc(),
 
+                // h264_qsv does not support decoding 10-bit content
+                (HardwareAccelerationMode.Qsv, VideoFormat.H264, PixelFormat.YUV420P10LE or PixelFormat.YUV444P10LE) =>
+                    new DecoderH264(),
+
                 (HardwareAccelerationMode.Qsv, VideoFormat.Hevc, _) => new DecoderHevcQsv(),
                 (HardwareAccelerationMode.Qsv, VideoFormat.H264, _) => new DecoderH264Qsv(),
                 (HardwareAccelerationMode.Qsv, VideoFormat.Mpeg2Video, _) => new DecoderMpeg2Qsv(),
@@ -45,15 +50,24 @@ public static class AvailableDecoders
                 (_, VideoFormat.MsMpeg4V3, _) => new DecoderMsMpeg4V3(),
                 (_, VideoFormat.Mpeg4, _) => new DecoderMpeg4(),
                 (_, VideoFormat.Vp9, _) => new DecoderVp9(),
-                
+
                 (_, VideoFormat.Undetermined, _) => new DecoderImplicit(),
 
-                // TODO: log warning and fall back to automatic decoder (i.e. None) instead of throwing?
-                // maybe have a special "unknown decoder" that sets frame loc to software without specifying any decoder
-                _ => throw new ArgumentOutOfRangeException(
-                    nameof(currentState.VideoFormat),
-                    currentState.VideoFormat,
-                    null)
+                var (accel, videoFormat, pixelFormat) => LogUnknownDecoder(accel, videoFormat, pixelFormat, logger)
             };
     }
+
+    private static Option<IDecoder> LogUnknownDecoder(
+        HardwareAccelerationMode hardwareAccelerationMode,
+        string videoFormat,
+        string pixelFormat,
+        ILogger logger)
+    {
+        logger.LogWarning(
+            "Unable to determine decoder for {AccelMode} - {VideoFormat} - {PixelFormat}; may have playback issues",
+            hardwareAccelerationMode,
+            videoFormat,
+            pixelFormat);
+        return Option<IDecoder>.None;
+    }
 }

ErsatzTV.FFmpeg/Decoder/DecoderVaapi.cs

@@ -4,7 +4,7 @@ namespace ErsatzTV.FFmpeg.Decoder;
 
 public class DecoderVaapi : DecoderBase
 {
-    public override FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Hardware;
+    public override FrameDataLocation OutputFrameDataLocation => FrameDataLocation.Software;
     public override string Name => "implicit_vaapi";
     public override IList<string> InputOptions => Array.Empty<string>();
 

ErsatzTV.FFmpeg/Encoder/AvailableEncoders.cs

@@ -1,46 +1,69 @@
 using ErsatzTV.FFmpeg.Encoder.Nvenc;
 using ErsatzTV.FFmpeg.Encoder.Qsv;
 using ErsatzTV.FFmpeg.Encoder.Vaapi;
+using ErsatzTV.FFmpeg.Encoder.VideoToolbox;
 using ErsatzTV.FFmpeg.Format;
+using Microsoft.Extensions.Logging;
+using LanguageExt;
 
 namespace ErsatzTV.FFmpeg.Encoder;
 
 public static class AvailableEncoders
 {
-    public static IEncoder ForVideoFormat(FrameState currentState, FrameState desiredState) =>
+    public static Option<IEncoder> ForVideoFormat(FrameState currentState, FrameState desiredState, ILogger logger) =>
         (desiredState.HardwareAccelerationMode, desiredState.VideoFormat) switch
         {
             (HardwareAccelerationMode.Nvenc, VideoFormat.Hevc) => new EncoderHevcNvenc(),
             (HardwareAccelerationMode.Nvenc, VideoFormat.H264) => new EncoderH264Nvenc(),
 
             (HardwareAccelerationMode.Qsv, VideoFormat.Hevc) => new EncoderHevcQsv(),
-            (HardwareAccelerationMode.Qsv, VideoFormat.H264) => new EncoderH264Qsv(),
+            (HardwareAccelerationMode.Qsv, VideoFormat.H264) => new EncoderH264Qsv(currentState),
 
             (HardwareAccelerationMode.Vaapi, VideoFormat.Hevc) => new EncoderHevcVaapi(currentState),
             (HardwareAccelerationMode.Vaapi, VideoFormat.H264) => new EncoderH264Vaapi(currentState),
 
+            (HardwareAccelerationMode.VideoToolbox, VideoFormat.Hevc) => new EncoderHevcVideoToolbox(),
+            (HardwareAccelerationMode.VideoToolbox, VideoFormat.H264) => new EncoderH264VideoToolbox(),
+
             (_, VideoFormat.Hevc) => new EncoderLibx265(),
             (_, VideoFormat.H264) => new EncoderLibx264(),
             (_, VideoFormat.Mpeg2Video) => new EncoderMpeg2Video(),
 
             (_, VideoFormat.Undetermined) => new EncoderImplicitVideo(),
 
-            _ => throw new ArgumentOutOfRangeException(nameof(desiredState.VideoFormat), desiredState.VideoFormat, null)
+            var (accel, videoFormat) => LogUnknownEncoder(accel, videoFormat, logger)
         };
+    
+    private static Option<IEncoder> LogUnknownEncoder(
+        HardwareAccelerationMode hardwareAccelerationMode,
+        string videoFormat,
+        ILogger logger)
+    {
+        logger.LogWarning(
+            "Unable to determine video encoder for {AccelMode} - {VideoFormat}; may have playback issues",
+            hardwareAccelerationMode,
+            videoFormat);
+        return Option<IEncoder>.None;
+    }
 
-    public static IEncoder ForAudioFormat(FrameState desiredState)
+    public static Option<IEncoder> ForAudioFormat(FrameState desiredState, ILogger logger)
     {
         return desiredState.AudioFormat.Match(
             audioFormat =>
                 audioFormat switch
                 {
-                    AudioFormat.Aac => (IEncoder)new EncoderAac(),
+                    AudioFormat.Aac => (Option<IEncoder>)new EncoderAac(),
                     AudioFormat.Ac3 => new EncoderAc3(),
-                    _ => throw new ArgumentOutOfRangeException(nameof(audioFormat), audioFormat, null)
+                    _ => LogUnknownEncoder(audioFormat, logger)
                 },
-            () => throw new ArgumentOutOfRangeException(
-                nameof(desiredState.AudioFormat),
-                desiredState.AudioFormat,
-                null));
+            () => LogUnknownEncoder(string.Empty, logger));
+    }
+    
+    private static Option<IEncoder> LogUnknownEncoder(
+        string audioFormat,
+        ILogger logger)
+    {
+        logger.LogWarning("Unable to determine audio encoder for {AudioFormat}; may have playback issues", audioFormat);
+        return Option<IEncoder>.None;
     }
 }

ErsatzTV.FFmpeg/Encoder/Qsv/EncoderH264Qsv.cs

@@ -4,6 +4,13 @@ namespace ErsatzTV.FFmpeg.Encoder.Qsv;
 
 public class EncoderH264Qsv : EncoderBase
 {
+    private readonly FrameState _currentState;
+
+    public EncoderH264Qsv(FrameState currentState)
+    {
+        _currentState = currentState;
+    }
+
     public override FrameState NextState(FrameState currentState) => currentState with
     {
         VideoFormat = VideoFormat.H264,
@@ -12,4 +19,25 @@ public class EncoderH264Qsv : EncoderBase
 
     public override string Name => "h264_qsv";
     public override StreamKind Kind => StreamKind.Video;
+
+    // need to convert to nv12 if we're still in software
+    public override string Filter
+    {
+        get
+        {
+            if (_currentState.FrameDataLocation == FrameDataLocation.Software)
+            {
+                // pixel format should already be converted to a supported format by QsvHardwareAccelerationOption
+                foreach (IPixelFormat pixelFormat in _currentState.PixelFormat)
+                {
+                    return $"format={pixelFormat.FFmpegName},hwupload=extra_hw_frames=64";
+                }
+                
+                // default to nv12
+                return "format=nv12,hwupload=extra_hw_frames=64";
+            }
+
+            return string.Empty;
+        }
+    }
 }

ErsatzTV.FFmpeg/Encoder/Vaapi/EncoderH264Vaapi.cs

@@ -21,6 +21,6 @@ public class EncoderH264Vaapi : EncoderBase
     public override StreamKind Kind => StreamKind.Video;
 
     public override string Filter => _currentState.FrameDataLocation == FrameDataLocation.Software
-        ? "hwupload"
+        ? "format=nv12|vaapi,hwupload"
         : string.Empty;
 }

ErsatzTV.FFmpeg/Encoder/Vaapi/EncoderHevcVaapi.cs

@@ -21,6 +21,6 @@ public class EncoderHevcVaapi : EncoderBase
     public override StreamKind Kind => StreamKind.Video;
 
     public override string Filter => _currentState.FrameDataLocation == FrameDataLocation.Software
-        ? "hwupload"
+        ? "format=nv12|vaapi,hwupload"
         : string.Empty;
 }

ErsatzTV.FFmpeg/Encoder/VideoToolbox/EncoderH264VideoToolbox.cs

@@ -0,0 +1,15 @@
+using ErsatzTV.FFmpeg.Format;
+
+namespace ErsatzTV.FFmpeg.Encoder.VideoToolbox;
+
+public class EncoderH264VideoToolbox : EncoderBase
+{
+    public override FrameState NextState(FrameState currentState) => currentState with
+    {
+        VideoFormat = VideoFormat.H264,
+        FrameDataLocation = FrameDataLocation.Hardware
+    };
+
+    public override string Name => "h264_videotoolbox";
+    public override StreamKind Kind => StreamKind.Video;
+}

ErsatzTV.FFmpeg/Encoder/VideoToolbox/EncoderHevcVideoToolbox.cs

@@ -0,0 +1,15 @@
+using ErsatzTV.FFmpeg.Format;
+
+namespace ErsatzTV.FFmpeg.Encoder.VideoToolbox;
+
+public class EncoderHevcVideoToolbox : EncoderBase
+{
+    public override FrameState NextState(FrameState currentState) => currentState with
+    {
+        VideoFormat = VideoFormat.Hevc,
+        FrameDataLocation = FrameDataLocation.Hardware
+    };
+
+    public override string Name => "hevc_videotoolbox";
+    public override StreamKind Kind => StreamKind.Video;
+}

ErsatzTV.FFmpeg/Filter/AvailableDeinterlaceFilters.cs

@@ -1,4 +1,5 @@
 using ErsatzTV.FFmpeg.Filter.Cuda;
+using ErsatzTV.FFmpeg.Filter.Qsv;
 using ErsatzTV.FFmpeg.Filter.Vaapi;
 
 namespace ErsatzTV.FFmpeg.Filter;
@@ -11,6 +12,7 @@ public static class AvailableDeinterlaceFilters
         accelMode switch
         {
             HardwareAccelerationMode.Nvenc => new YadifCudaFilter(currentState),
+            HardwareAccelerationMode.Qsv => new DeinterlaceQsvFilter(currentState),
             HardwareAccelerationMode.Vaapi => new DeinterlaceVaapiFilter(currentState),
             _ => new YadifFilter(currentState)
         };

ErsatzTV.FFmpeg/Filter/PadFilter.cs

@@ -17,10 +17,18 @@ public class PadFilter : BaseFilter
         {
             string pad = $"pad={_paddedSize.Width}:{_paddedSize.Height}:-1:-1:color=black";
             string pixelFormat = _currentState.PixelFormat.Match(pf => pf.FFmpegName, () => string.Empty);
-            
-            return _currentState.FrameDataLocation == FrameDataLocation.Hardware && !string.IsNullOrWhiteSpace(pixelFormat)
-                ? $"hwdownload,format={pixelFormat},{pad}" // TODO: does this apply to other accels?
-                : pad;
+
+            if (_currentState.FrameDataLocation == FrameDataLocation.Hardware)
+            {
+                if (!string.IsNullOrWhiteSpace(pixelFormat))
+                {
+                    return $"hwdownload,format={pixelFormat},{pad}";
+                }
+
+                return $"hwdownload,{pad}";
+            }
+
+            return pad;
         }
     }
 

ErsatzTV.FFmpeg/Filter/Qsv/DeinterlaceQsvFilter.cs

@@ -0,0 +1,41 @@
+using ErsatzTV.FFmpeg.Format;
+
+namespace ErsatzTV.FFmpeg.Filter.Qsv;
+
+public class DeinterlaceQsvFilter : BaseFilter
+{
+    private readonly FrameState _currentState;
+
+    public DeinterlaceQsvFilter(FrameState currentState)
+    {
+        _currentState = currentState;
+    }
+
+    // deinterlace_qsv seems to only support nv12, not p010le
+    public override string Filter => _currentState.FrameDataLocation == FrameDataLocation.Software
+        ? "format=nv12,hwupload=extra_hw_frames=64,deinterlace_qsv"
+        : "deinterlace_qsv";
+
+    public override FrameState NextState(FrameState currentState)
+    {
+        FrameState result = currentState with
+        {
+            Deinterlaced = true,
+            FrameDataLocation = FrameDataLocation.Hardware
+        };
+
+        // deinterlace_qsv seems to only support nv12, not p010le
+        foreach (IPixelFormat pixelFormat in currentState.PixelFormat)
+        {
+            if (pixelFormat.FFmpegName != FFmpegFormat.NV12)
+            {
+                result = result with
+                {
+                    PixelFormat = new PixelFormatNv12(pixelFormat.Name)
+                };
+            }
+        }
+
+        return result;
+    }
+}

ErsatzTV.FFmpeg/Filter/Qsv/ScaleQsvFilter.cs

@@ -39,15 +39,18 @@ public class ScaleQsvFilter : BaseFilter
                 scale = $"scale_qsv={targetSize}{format}";
             }
 
-            // TODO: this might not always upload to hardware, so NextState could be inaccurate
-            if (string.IsNullOrWhiteSpace(scale))
+            if (_currentState.FrameDataLocation == FrameDataLocation.Hardware)
             {
                 return scale;
             }
 
-            return _currentState.FrameDataLocation == FrameDataLocation.Hardware
-                ? scale
-                : $"hwupload=extra_hw_frames=64,{scale}";
+            string initialPixelFormat = _currentState.PixelFormat.Match(pf => pf.FFmpegName, FFmpegFormat.NV12);
+            if (!string.IsNullOrWhiteSpace(scale))
+            {
+                return $"format={initialPixelFormat},hwupload=extra_hw_frames=64,{scale}";
+            }
+
+            return $"format={initialPixelFormat},hwupload=extra_hw_frames=64";
         }
     }
 

ErsatzTV.FFmpeg/Filter/Vaapi/DeinterlaceVaapiFilter.cs

@@ -12,7 +12,7 @@ public class DeinterlaceVaapiFilter : BaseFilter
     public override string Filter =>
         _currentState.FrameDataLocation == FrameDataLocation.Hardware
             ? "deinterlace_vaapi"
-            : "hwupload,deinterlace_vaapi";
+            : "format=nv12|vaapi,hwupload,deinterlace_vaapi";
 
     public override FrameState NextState(FrameState currentState) => currentState with
     {

ErsatzTV.FFmpeg/Filter/Vaapi/ScaleVaapiFilter.cs

@@ -41,15 +41,17 @@ public class ScaleVaapiFilter : BaseFilter
                 scale = $"scale_vaapi={targetSize}:force_original_aspect_ratio=1:force_divisible_by=2{format}";
             }
 
-            // TODO: this might not always upload to hardware, so NextState could be inaccurate
-            if (string.IsNullOrWhiteSpace(scale))
+            if (_currentState.FrameDataLocation == FrameDataLocation.Hardware)
             {
                 return scale;
             }
 
-            return _currentState.FrameDataLocation == FrameDataLocation.Hardware
-                ? scale
-                : $"hwupload,{scale}";
+            if (!string.IsNullOrWhiteSpace(scale))
+            {
+                return $"format=nv12|vaapi,hwupload,{scale}";
+            }
+
+            return "format=nv12|vaapi,hwupload";
         }
     }
 

ErsatzTV.FFmpeg/Format/AvailablePixelFormats.cs

@@ -1,10 +1,11 @@
 using LanguageExt;
+using Microsoft.Extensions.Logging;
 
 namespace ErsatzTV.FFmpeg.Format;
 
 public static class AvailablePixelFormats
 {
-    public static IPixelFormat ForPixelFormat(string pixelFormat)
+    public static Option<IPixelFormat> ForPixelFormat(string pixelFormat, ILogger logger)
     {
         return pixelFormat switch
         {
@@ -12,7 +13,13 @@ public static class AvailablePixelFormats
             PixelFormat.YUV420P10LE => new PixelFormatYuv420P10Le(),
             PixelFormat.YUVJ420P => new PixelFormatYuvJ420P(),
             PixelFormat.YUV444P => new PixelFormatYuv444P(),
-            _ => throw new ArgumentOutOfRangeException(nameof(pixelFormat), pixelFormat, null)
+            _ => LogUnknownPixelFormat(pixelFormat, logger)
         };
     }
+
+    private static Option<IPixelFormat> LogUnknownPixelFormat(string pixelFormat, ILogger logger)
+    {
+        logger.LogWarning("Unexpected pixel format {PixelFormat} may have playback issues", pixelFormat);
+        return Option<IPixelFormat>.None;
+    }
 }

ErsatzTV.FFmpeg/Option/HardwareAcceleration/AvailableHardwareAccelerationOptions.cs

@@ -9,6 +9,7 @@ public static class AvailableHardwareAccelerationOptions
             HardwareAccelerationMode.Nvenc => new CudaHardwareAccelerationOption(),
             HardwareAccelerationMode.Qsv => new QsvHardwareAccelerationOption(),
             HardwareAccelerationMode.Vaapi => new VaapiHardwareAccelerationOption(),
+            HardwareAccelerationMode.VideoToolbox => new VideoToolboxHardwareAccelerationOption(),
             _ => throw new ArgumentOutOfRangeException(nameof(mode), mode, null)
         };
     }

ErsatzTV.FFmpeg/Option/HardwareAcceleration/QsvHardwareAccelerationOption.cs

@@ -1,15 +1,41 @@
-namespace ErsatzTV.FFmpeg.Option.HardwareAcceleration;
+using ErsatzTV.FFmpeg.Format;
+
+namespace ErsatzTV.FFmpeg.Option.HardwareAcceleration;
 
 public class QsvHardwareAccelerationOption : GlobalOption
 {
+    // TODO: read this from ffmpeg output
+    private readonly List<string> _supportedFFmpegFormats = new()
+    {
+        FFmpegFormat.NV12,
+        FFmpegFormat.P010LE
+    };
+
     public override IList<string> GlobalOptions => new List<string>
     {
         "-hwaccel", "qsv",
         "-init_hw_device", "qsv=qsv:MFX_IMPL_hw_any"
     };
 
-    public override FrameState NextState(FrameState currentState) => currentState with
+    // qsv encoders want nv12
+    public override FrameState NextState(FrameState currentState)
     {
-        HardwareAccelerationMode = HardwareAccelerationMode.Qsv
-    };
+        FrameState result = currentState with { HardwareAccelerationMode = HardwareAccelerationMode.Qsv }; 
+        
+        foreach (IPixelFormat pixelFormat in currentState.PixelFormat)
+        {
+            if (_supportedFFmpegFormats.Contains(pixelFormat.FFmpegName))
+            {
+                return result;
+            }
+
+            return currentState with { PixelFormat = new PixelFormatNv12(pixelFormat.Name) };
+        }
+
+        return currentState with
+        {
+            HardwareAccelerationMode = HardwareAccelerationMode.Qsv,
+            PixelFormat = new PixelFormatNv12(new PixelFormatUnknown().Name)
+        };
+    }
 }

ErsatzTV.FFmpeg/Option/HardwareAcceleration/VideoToolboxHardwareAccelerationOption.cs

@@ -0,0 +1,11 @@
+namespace ErsatzTV.FFmpeg.Option.HardwareAcceleration;
+
+public class VideoToolboxHardwareAccelerationOption : GlobalOption
+{
+    public override IList<string> GlobalOptions => new List<string> { "-hwaccel", "videotoolbox" };
+
+    public override FrameState NextState(FrameState currentState) => currentState with
+    {
+        HardwareAccelerationMode = HardwareAccelerationMode.VideoToolbox
+    };
+}

ErsatzTV.FFmpeg/PipelineBuilder.cs

@@ -138,7 +138,7 @@ public class PipelineBuilder
                     _pipelineSteps.Add(accel);
                 }
 
-                foreach (IDecoder decoder in AvailableDecoders.ForVideoFormat(currentState, desiredState))
+                foreach (IDecoder decoder in AvailableDecoders.ForVideoFormat(currentState, desiredState, _logger))
                 {
                     currentState = decoder.NextState(currentState);
                     _pipelineSteps.Add(decoder);
@@ -305,10 +305,13 @@ public class PipelineBuilder
                 // after everything else is done, apply the encoder
                 if (!_pipelineSteps.OfType<IEncoder>().Any())
                 {
-                    encoder = AvailableEncoders.ForVideoFormat(currentState, desiredState);
-                    _pipelineSteps.Add(encoder);
-                    _videoFilterSteps.Add(encoder);
-                    currentState = encoder.NextState(currentState);
+                    foreach (IEncoder e in AvailableEncoders.ForVideoFormat(currentState, desiredState, _logger))
+                    {
+                        encoder = e;
+                        _pipelineSteps.Add(encoder);
+                        _videoFilterSteps.Add(encoder);
+                        currentState = encoder.NextState(currentState);
+                    }
                 }
             }
             
@@ -325,9 +328,11 @@ public class PipelineBuilder
             {
                 if (currentState.AudioFormat != desiredState.AudioFormat)
                 {
-                    IEncoder step = AvailableEncoders.ForAudioFormat(desiredState);
-                    currentState = step.NextState(currentState);
-                    _pipelineSteps.Add(step);
+                    foreach (IEncoder step in AvailableEncoders.ForAudioFormat(desiredState, _logger))
+                    {
+                        currentState = step.NextState(currentState);
+                        _pipelineSteps.Add(step);
+                    }
                 }
 
                 foreach (int desiredAudioChannels in desiredState.AudioChannels)

ErsatzTV.sln.DotSettings

@@ -60,6 +60,7 @@
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=uniqueid/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Vaapi/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=VCHIP/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=videotoolbox/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=xmltv/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=yadif/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=YUVJ/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>