generate slug instead of probing and transcoding resource (#2824)

* generate slug instead of probing and using slug resource

* refactor

* more fixes

ErsatzTV.Application/Streaming/HlsSessionWorker.cs

@@ -393,7 +393,8 @@ public class HlsSessionWorker : IHlsSessionWorker
             HlsSessionState.SlugAndRealtime => HlsSessionState.ZeroAndRealtime,
 
             // after completing the item, insert a slug
-            _ when isComplete && _slugSeconds.IsSome => HlsSessionState.SlugAndWorkAhead,
+            HlsSessionState.ZeroAndWorkAhead or HlsSessionState.SeekAndWorkAhead when isComplete && _slugSeconds.IsSome => HlsSessionState.SlugAndWorkAhead,
+            HlsSessionState.ZeroAndRealtime or HlsSessionState.SeekAndRealtime when isComplete && _slugSeconds.IsSome => HlsSessionState.SlugAndRealtime,
 
             // after seeking and completing the item, start at zero
             HlsSessionState.SeekAndWorkAhead => HlsSessionState.ZeroAndWorkAhead,
@@ -454,6 +455,7 @@ public class HlsSessionWorker : IHlsSessionWorker
                 {
                     HlsSessionState.SeekAndWorkAhead => HlsSessionState.SeekAndRealtime,
                     HlsSessionState.ZeroAndWorkAhead => HlsSessionState.ZeroAndRealtime,
+                    HlsSessionState.SlugAndWorkAhead => HlsSessionState.SlugAndRealtime,
                     _ => _state
                 };
 

ErsatzTV.Application/Streaming/Queries/GetSlugProcessByChannelNumberHandler.cs

@@ -1,22 +1,15 @@
-using System.IO.Abstractions;
 using ErsatzTV.Core;
 using ErsatzTV.Core.Domain;
-using ErsatzTV.Core.Domain.Filler;
-using ErsatzTV.Core.FFmpeg;
 using ErsatzTV.Core.Interfaces.FFmpeg;
-using ErsatzTV.Core.Interfaces.Metadata;
+using ErsatzTV.Core.Interfaces.Streaming;
-using ErsatzTV.FFmpeg;
 using ErsatzTV.Infrastructure.Data;
-using ErsatzTV.Infrastructure.Extensions;
 using Microsoft.EntityFrameworkCore;
 
 namespace ErsatzTV.Application.Streaming;
 
 public class GetSlugProcessByChannelNumberHandler(
     IDbContextFactory<TvContext> dbContextFactory,
-    IFileSystem fileSystem,
+    IFFmpegProcessService ffmpegProcessService)
-    IFFmpegProcessService ffmpegProcessService,
-    ILocalStatisticsProvider localStatisticsProvider)
     : FFmpegProcessHandler<GetSlugProcessByChannelNumber>(dbContextFactory)
 {
     protected override async Task<Either<BaseError, PlayoutItemProcessModel>> GetProcess(
@@ -27,82 +20,30 @@ public class GetSlugProcessByChannelNumberHandler(
         string ffprobePath,
         CancellationToken cancellationToken)
     {
-        string videoPath = fileSystem.Path.Combine(FileSystemLayout.ResourcesCacheFolder, "slug.mp4");
+        var duration = TimeSpan.FromSeconds(await request.SlugSeconds.IfNoneAsync(0));
-
+        if (duration <= TimeSpan.Zero)
-        bool saveReports = await dbContext.ConfigElements
-            .GetValue<bool>(ConfigElementKey.FFmpegSaveReports, cancellationToken)
-            .Map(result => result.IfNone(false));
-
-        Either<BaseError, MediaVersion> maybeVersion =
-            await localStatisticsProvider.GetStatistics(ffprobePath, videoPath);
-        foreach (var error in maybeVersion.LeftToSeq())
         {
-            return error;
+            return BaseError.New("Slug seconds must be non-zero");
-        }
-
-        var version = maybeVersion.RightToSeq().Head();
-
-        var mediaItem = new OtherVideo
-        {
-            MediaVersions = [version]
-        };
-
-        TimeSpan duration = version.Duration;
-        foreach (double slugSeconds in request.SlugSeconds)
-        {
-            TimeSpan seconds = TimeSpan.FromSeconds(slugSeconds);
-            if (seconds > TimeSpan.Zero && seconds < duration)
-            {
-                duration = seconds;
-            }
         }
 
         DateTimeOffset finish = request.Now.Add(duration);
 
-        PlayoutItemResult playoutItemResult = await ffmpegProcessService.ForPlayoutItem(
+        var playoutItemResult = await ffmpegProcessService.Slug(
             ffmpegPath,
-            ffprobePath,
-            saveReports,
             channel,
-            new MediaItemVideoVersion(mediaItem, version),
-            new MediaItemAudioVersion(mediaItem, version),
-            videoPath,
-            videoPath,
-            _ => Task.FromResult<List<Subtitle>>([]),
-            string.Empty,
-            string.Empty,
-            string.Empty,
-            ChannelSubtitleMode.None,
-            request.Now,
-            finish,
             request.Now,
             duration,
-            [],
-            [],
-            channel.FFmpegProfile.VaapiDisplay,
-            channel.FFmpegProfile.VaapiDriver,
-            channel.FFmpegProfile.VaapiDevice,
-            Optional(channel.FFmpegProfile.QsvExtraHardwareFrames),
             request.HlsRealtime,
-            StreamInputKind.Vod,
+            request.PtsOffset);
-            FillerKind.None,
-            inPoint: TimeSpan.Zero,
-            request.ChannelStartTime,
-            request.PtsOffset,
-            request.TargetFramerate,
-            Option<string>.None,
-            _ => { },
-            canProxy: true,
-            cancellationToken);
 
         var result = new PlayoutItemProcessModel(
-            playoutItemResult.Process,
+            playoutItemResult,
-            playoutItemResult.GraphicsEngineContext,
+            Option<GraphicsEngineContext>.None,
             duration,
             finish,
             isComplete: true,
             request.Now.ToUnixTimeSeconds(),
-            playoutItemResult.MediaItemId,
+            Option<int>.None,
             Optional(channel.PlayoutOffset),
             !request.HlsRealtime);
 

ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs

@@ -665,7 +665,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
         string vaapiDevice,
         Option<int> qsvExtraHardwareFrames)
     {
-        FFmpegPlaybackSettings playbackSettings = FFmpegPlaybackSettingsCalculator.CalculateErrorSettings(
+        FFmpegPlaybackSettings playbackSettings = FFmpegPlaybackSettingsCalculator.CalculateGeneratedImageSettings(
             channel.StreamingMode,
             channel.FFmpegProfile,
             hlsRealtime);
@@ -726,57 +726,6 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             playbackSettings.NormalizeColors,
             playbackSettings.Deinterlace);
 
-        OutputFormatKind outputFormat = OutputFormatKind.MpegTs;
-        switch (channel.StreamingMode)
-        {
-            case StreamingMode.HttpLiveStreamingSegmenter:
-                outputFormat = OutputFormatKind.Hls;
-                break;
-        }
-
-        Option<string> hlsPlaylistPath = outputFormat is OutputFormatKind.Hls or OutputFormatKind.HlsMp4
-            ? Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live.m3u8")
-            : Option<string>.None;
-
-        long nowSeconds = now.ToUnixTimeSeconds();
-
-        Option<string> hlsSegmentTemplate = outputFormat switch
-        {
-            OutputFormatKind.Hls => Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live%06d.ts"),
-            OutputFormatKind.HlsMp4 => Path.Combine(
-                FileSystemLayout.TranscodeFolder,
-                channel.Number,
-                $"live_{nowSeconds}_%06d.m4s"),
-            _ => Option<string>.None
-        };
-
-        Option<string> hlsInitTemplate = outputFormat switch
-        {
-            OutputFormatKind.HlsMp4 => $"{nowSeconds}_init.mp4",
-            _ =>  Option<string>.None
-        };
-
-        Option<string> hlsSegmentOptions = Option<string>.None;
-        if (outputFormat is OutputFormatKind.Hls)
-        {
-            string options = string.Empty;
-
-            if (ptsOffset == TimeSpan.Zero)
-            {
-                options += "+initial_discontinuity";
-            }
-
-            if (audioFormat == AudioFormat.AacLatm)
-            {
-                options += "+latm";
-            }
-
-            if (!string.IsNullOrWhiteSpace(options))
-            {
-                hlsSegmentOptions = $"mpegts_flags={options}";
-            }
-        }
-
         string videoPath = _localFileSystem.GetCustomOrDefaultFile(
             FileSystemLayout.ResourcesCacheFolder,
             "background.png");
@@ -802,6 +751,8 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
         HardwareAccelerationMode hwAccel = GetHardwareAccelerationMode(playbackSettings);
         _logger.LogDebug("HW accel mode: {HwAccel}", hwAccel);
 
+        var hlsOptions = GetHlsOptions(channel, now, ptsOffset, audioFormat);
+
         var ffmpegState = new FFmpegState(
             channel.Number == FileSystemLayout.TranscodeTroubleshootingChannel,
             HardwareAccelerationMode.None, // no hw accel decode since errors loop
@@ -816,11 +767,11 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
             None,
             None,
             None,
-            outputFormat,
+            hlsOptions.OutputFormat,
-            hlsPlaylistPath,
+            hlsOptions.HlsPlaylistPath,
-            hlsSegmentTemplate,
+            hlsOptions.HlsSegmentTemplate,
-            hlsInitTemplate,
+            hlsOptions.HlsInitTemplate,
-            hlsSegmentOptions,
+            hlsOptions.HlsSegmentOptions,
             ptsOffset,
             Option<int>.None,
             qsvExtraHardwareFrames,
@@ -862,6 +813,136 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
         return GetCommand(ffmpegPath, videoInputFile, audioInputFile, None, None, None, pipeline);
     }
 
+    public async Task<Command> Slug(
+        string ffmpegPath,
+        Channel channel,
+        DateTimeOffset now,
+        TimeSpan duration,
+        bool hlsRealtime,
+        TimeSpan ptsOffset)
+    {
+        FFmpegPlaybackSettings playbackSettings = FFmpegPlaybackSettingsCalculator.CalculateGeneratedImageSettings(
+            channel.StreamingMode,
+            channel.FFmpegProfile,
+            hlsRealtime);
+
+        Resolution desiredResolution = channel.FFmpegProfile.Resolution;
+
+        string audioFormat = playbackSettings.AudioFormat switch
+        {
+            FFmpegProfileAudioFormat.Ac3 => AudioFormat.Ac3,
+            FFmpegProfileAudioFormat.AacLatm => AudioFormat.AacLatm,
+            _ => AudioFormat.Aac
+        };
+
+        var audioState = new AudioState(
+            audioFormat,
+            playbackSettings.AudioChannels,
+            playbackSettings.AudioBitrate,
+            playbackSettings.AudioBufferSize,
+            playbackSettings.AudioSampleRate,
+            false,
+            AudioFilter.None,
+            playbackSettings.TargetLoudness);
+
+        string videoFormat = GetVideoFormat(playbackSettings);
+
+        var desiredState = new FrameState(
+            playbackSettings.RealtimeOutput,
+            InfiniteLoop: false,
+            videoFormat,
+            GetVideoProfile(videoFormat, channel.FFmpegProfile.VideoProfile),
+            VideoPreset.Unset,
+            channel.FFmpegProfile.AllowBFrames,
+            new PixelFormatYuv420P(),
+            new FrameSize(desiredResolution.Width, desiredResolution.Height),
+            new FrameSize(desiredResolution.Width, desiredResolution.Height),
+            Option<FrameSize>.None,
+            channel.FFmpegProfile.PadMode is FilterMode.HardwareIfPossible
+                ? FFmpegFilterMode.HardwareIfPossible
+                : FFmpegFilterMode.Software,
+            IsAnamorphic: false,
+            Option<FrameRate>.None,
+            playbackSettings.VideoBitrate,
+            playbackSettings.VideoBufferSize,
+            playbackSettings.VideoTrackTimeScale,
+            playbackSettings.NormalizeColors,
+            playbackSettings.Deinterlace);
+
+        var frameRate = playbackSettings.FrameRate.IfNone(new FrameRate("24"));
+
+        var ffmpegVideoStream = new VideoStream(
+            0,
+            VideoFormat.GeneratedImage,
+            string.Empty,
+            new PixelFormatUnknown(), // leave this unknown so we convert to desired yuv420p
+            ColorParams.Default,
+            desiredState.PaddedSize,
+            MaybeSampleAspectRatio: "1:1",
+            DisplayAspectRatio: string.Empty,
+            frameRate,
+            true,
+            ScanKind.Progressive);
+
+        var videoInputFile = new LavfiInputFile(
+            $"color=c=black:s={desiredState.PaddedSize.Width}x{desiredState.PaddedSize.Height}:r={frameRate.FrameRateString}:d={duration.TotalSeconds}",
+            ffmpegVideoStream);
+
+        HardwareAccelerationMode hwAccel = GetHardwareAccelerationMode(playbackSettings);
+
+        var hlsOptions = GetHlsOptions(channel, now, ptsOffset, audioFormat);
+
+        var ffmpegState = new FFmpegState(
+            channel.Number == FileSystemLayout.TranscodeTroubleshootingChannel,
+            HardwareAccelerationMode.None, // no hw accel decode since errors loop
+            hwAccel,
+            VaapiDriverName(hwAccel, channel.FFmpegProfile.VaapiDriver),
+            VaapiDeviceName(hwAccel, channel.FFmpegProfile.VaapiDevice),
+            playbackSettings.StreamSeek,
+            duration,
+            channel.StreamingMode != StreamingMode.HttpLiveStreamingDirect,
+            "ErsatzTV",
+            channel.Name,
+            None,
+            None,
+            None,
+            hlsOptions.OutputFormat,
+            hlsOptions.HlsPlaylistPath,
+            hlsOptions.HlsSegmentTemplate,
+            hlsOptions.HlsInitTemplate,
+            hlsOptions.HlsSegmentOptions,
+            ptsOffset,
+            Option<int>.None,
+            Optional(channel.FFmpegProfile.QsvExtraHardwareFrames),
+            false,
+            false,
+            GetTonemapAlgorithm(playbackSettings),
+            channel.Number == FileSystemLayout.TranscodeTroubleshootingChannel);
+
+        var audioInputFile = new NullAudioInputFile(audioState);
+
+        IPipelineBuilder pipelineBuilder = await _pipelineBuilderFactory.GetBuilder(
+            hwAccel,
+            videoInputFile,
+            audioInputFile,
+            Option<WatermarkInputFile>.None,
+            Option<SubtitleInputFile>.None,
+            Option<ConcatInputFile>.None,
+            Option<GraphicsEngineInput>.None,
+            VaapiDisplayName(hwAccel, channel.FFmpegProfile.VaapiDisplay),
+            VaapiDriverName(hwAccel, channel.FFmpegProfile.VaapiDriver),
+            VaapiDeviceName(hwAccel, channel.FFmpegProfile.VaapiDevice),
+            channel.Number == FileSystemLayout.TranscodeTroubleshootingChannel
+                ? FileSystemLayout.TranscodeTroubleshootingFolder
+                : FileSystemLayout.FFmpegReportsFolder,
+            FileSystemLayout.FontsCacheFolder,
+            ffmpegPath);
+
+        FFmpegPipeline pipeline = pipelineBuilder.Build(ffmpegState, desiredState);
+
+        return GetCommand(ffmpegPath, videoInputFile, audioInputFile, None, None, None, pipeline);
+    }
+
     public async Task<Command> ConcatChannel(
         string ffmpegPath,
         bool saveReports,
@@ -1260,4 +1341,67 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService
 
         return false;
     }
+
+    private static HlsOptions GetHlsOptions(Channel channel, DateTimeOffset now, TimeSpan ptsOffset, string audioFormat)
+    {
+        OutputFormatKind outputFormat = OutputFormatKind.MpegTs;
+        switch (channel.StreamingMode)
+        {
+            case StreamingMode.HttpLiveStreamingSegmenter:
+                outputFormat = OutputFormatKind.Hls;
+                break;
+        }
+
+        Option<string> hlsPlaylistPath = outputFormat is OutputFormatKind.Hls or OutputFormatKind.HlsMp4
+            ? Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live.m3u8")
+            : Option<string>.None;
+
+        long nowSeconds = now.ToUnixTimeSeconds();
+
+        Option<string> hlsSegmentTemplate = outputFormat switch
+        {
+            OutputFormatKind.Hls => Path.Combine(FileSystemLayout.TranscodeFolder, channel.Number, "live%06d.ts"),
+            OutputFormatKind.HlsMp4 => Path.Combine(
+                FileSystemLayout.TranscodeFolder,
+                channel.Number,
+                $"live_{nowSeconds}_%06d.m4s"),
+            _ => Option<string>.None
+        };
+
+        Option<string> hlsInitTemplate = outputFormat switch
+        {
+            OutputFormatKind.HlsMp4 => $"{nowSeconds}_init.mp4",
+            _ => Option<string>.None
+        };
+
+        Option<string> hlsSegmentOptions = Option<string>.None;
+        if (outputFormat is OutputFormatKind.Hls)
+        {
+            string options = string.Empty;
+
+            if (ptsOffset == TimeSpan.Zero)
+            {
+                options += "+initial_discontinuity";
+            }
+
+            if (audioFormat == AudioFormat.AacLatm)
+            {
+                options += "+latm";
+            }
+
+            if (!string.IsNullOrWhiteSpace(options))
+            {
+                hlsSegmentOptions = $"mpegts_flags={options}";
+            }
+        }
+
+        return new HlsOptions(outputFormat, hlsPlaylistPath, hlsSegmentTemplate, hlsInitTemplate, hlsSegmentOptions);
+    }
+
+    private sealed record HlsOptions(
+        OutputFormatKind OutputFormat,
+        Option<string> HlsPlaylistPath,
+        Option<string> HlsSegmentTemplate,
+        Option<string> HlsInitTemplate,
+        Option<string> HlsSegmentOptions);
 }

ErsatzTV.Core/FFmpeg/FFmpegPlaybackSettingsCalculator.cs

@@ -181,7 +181,7 @@ public static class FFmpegPlaybackSettingsCalculator
         return result;
     }
 
-    public static FFmpegPlaybackSettings CalculateErrorSettings(
+    public static FFmpegPlaybackSettings CalculateGeneratedImageSettings(
         StreamingMode streamingMode,
         FFmpegProfile ffmpegProfile,
         bool hlsRealtime) =>

ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs

@@ -71,7 +71,7 @@ public class FFmpegProcessService
             }
 
             FFmpegPlaybackSettings playbackSettings =
-                FFmpegPlaybackSettingsCalculator.CalculateErrorSettings(
+                FFmpegPlaybackSettingsCalculator.CalculateGeneratedImageSettings(
                     StreamingMode.TransportStream,
                     channel.FFmpegProfile,
                     false);

ErsatzTV.Core/FFmpeg/FFmpegProgress.cs

@@ -39,7 +39,7 @@ public partial class FFmpegProgress
                         speed);
                 }
             }
-            else if (speed < 1.0)
+            else if (speed < 0.99)
             {
                 logger.LogWarning(
                     "Media item {MediaItemId} on channel {Channel} transcoded at {Speed}x (throttled) which may not be fast enough to support playback",

ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegProcessService.cs

@@ -58,6 +58,14 @@ public interface IFFmpegProcessService
         string vaapiDevice,
         Option<int> qsvExtraHardwareFrames);
 
+    Task<Command> Slug(
+        string ffmpegPath,
+        Channel channel,
+        DateTimeOffset now,
+        TimeSpan duration,
+        bool hlsRealtime,
+        TimeSpan ptsOffset);
+
     Task<Command> ConcatChannel(string ffmpegPath, bool saveReports, Channel channel, string scheme, string host);
 
     Task<Command> WrapSegmenter(

ErsatzTV.FFmpeg/InputFile.cs

@@ -60,7 +60,7 @@ public record NullAudioInputFile : AudioInputFile
         DesiredState) =>
         InputOptions.Add(new LavfiInputOption());
 
-    public void Deconstruct(out AudioState DesiredState) => DesiredState = this.DesiredState;
+    public void Deconstruct(out AudioState desiredState) => desiredState = DesiredState;
 }
 
 public record VideoInputFile(
@@ -79,6 +79,12 @@ public record VideoInputFile(
     }
 }
 
+public record LavfiInputFile : VideoInputFile
+{
+    public LavfiInputFile(string parameters, VideoStream videoStream) : base(parameters, [videoStream])
+        => InputOptions.Add(new LavfiInputOption());
+}
+
 public record WatermarkInputFile(string Path, IList<VideoStream> VideoStreams, WatermarkState DesiredState)
     : VideoInputFile(Path, VideoStreams);
 

ErsatzTV/ErsatzTV.csproj

@@ -119,7 +119,6 @@
           <Generator>ResXFileCodeGenerator</Generator>
           <LastGenOutput>Common.Designer.cs</LastGenOutput>
         </EmbeddedResource>
-        <EmbeddedResource Include="Resources\slug.mp4" />
     </ItemGroup>
 
     <ItemGroup>

ErsatzTV/Services/RunOnce/ResourceExtractorService.cs

@@ -27,7 +27,6 @@ public class ResourceExtractorService : BackgroundService
         await ExtractResource(assembly, "sequential-schedule.schema.json", stoppingToken);
         await ExtractResource(assembly, "sequential-schedule-import.schema.json", stoppingToken);
         await ExtractResource(assembly, "test.avs", stoppingToken);
-        await ExtractResource(assembly, "slug.mp4", stoppingToken);
 
         await ExtractFontResource(assembly, "Sen.ttf", stoppingToken);
         await ExtractFontResource(assembly, "Roboto-Regular.ttf", stoppingToken);