diff --git a/CHANGELOG.md b/CHANGELOG.md index cee1f548d..fc20ab2e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Changed -- Reduce initial `HLS Segmenter` delay from 5 seconds to 3 seconds +- Remove forced initial delay from `HLS Segmenter` streaming mode ## [0.1.0-alpha] - 2021-10-08 ### Added diff --git a/ErsatzTV.Application/ErsatzTV.Application.csproj b/ErsatzTV.Application/ErsatzTV.Application.csproj index db7d17faa..8c5e5adcc 100644 --- a/ErsatzTV.Application/ErsatzTV.Application.csproj +++ b/ErsatzTV.Application/ErsatzTV.Application.csproj @@ -7,10 +7,6 @@ - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/ErsatzTV.Application/Streaming/Commands/CleanUpFFmpegSessions.cs b/ErsatzTV.Application/Streaming/Commands/CleanUpFFmpegSessions.cs deleted file mode 100644 index b1a8cc1b0..000000000 --- a/ErsatzTV.Application/Streaming/Commands/CleanUpFFmpegSessions.cs +++ /dev/null @@ -1,9 +0,0 @@ -using ErsatzTV.Core; -using LanguageExt; -using MediatR; -using Unit = LanguageExt.Unit; - -namespace ErsatzTV.Application.Streaming.Commands -{ - public record CleanUpFFmpegSessions : IRequest>, IFFmpegWorkerRequest; -} diff --git a/ErsatzTV.Application/Streaming/Commands/CleanUpFFmpegSessionsHandler.cs b/ErsatzTV.Application/Streaming/Commands/CleanUpFFmpegSessionsHandler.cs deleted file mode 100644 index b54477cd6..000000000 --- a/ErsatzTV.Application/Streaming/Commands/CleanUpFFmpegSessionsHandler.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System.Threading; -using System.Threading.Channels; -using System.Threading.Tasks; -using ErsatzTV.Core; -using LanguageExt; -using MediatR; -using Unit = LanguageExt.Unit; - -namespace ErsatzTV.Application.Streaming.Commands -{ - public class CleanUpFFmpegSessionsHandler : IRequestHandler> - { - private readonly ChannelWriter _channel; - - public CleanUpFFmpegSessionsHandler(ChannelWriter channel) - { - _channel = channel; - } - - public async Task> - Handle(CleanUpFFmpegSessions request, CancellationToken cancellationToken) - { - await _channel.WriteAsync(request, cancellationToken); - return Unit.Default; - } - } -} diff --git a/ErsatzTV.Application/Streaming/Commands/StartFFmpegSessionHandler.cs b/ErsatzTV.Application/Streaming/Commands/StartFFmpegSessionHandler.cs index 121994046..61637325b 100644 --- a/ErsatzTV.Application/Streaming/Commands/StartFFmpegSessionHandler.cs +++ b/ErsatzTV.Application/Streaming/Commands/StartFFmpegSessionHandler.cs @@ -1,14 +1,13 @@ using System; using System.IO; using System.Threading; -using System.Threading.Channels; using System.Threading.Tasks; using ErsatzTV.Core; using ErsatzTV.Core.Errors; -using ErsatzTV.Core.FFmpeg; using ErsatzTV.Core.Interfaces.FFmpeg; using ErsatzTV.Core.Interfaces.Metadata; using LanguageExt; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using static LanguageExt.Prelude; @@ -16,21 +15,21 @@ namespace ErsatzTV.Application.Streaming.Commands { public class StartFFmpegSessionHandler : MediatR.IRequestHandler> { - private readonly ChannelWriter _channel; private readonly ILogger _logger; + private readonly IServiceScopeFactory _serviceScopeFactory; private readonly IFFmpegSegmenterService _ffmpegSegmenterService; private readonly ILocalFileSystem _localFileSystem; public StartFFmpegSessionHandler( - IFFmpegSegmenterService ffmpegSegmenterService, ILocalFileSystem localFileSystem, - ChannelWriter channel, - ILogger logger) + ILogger logger, + IServiceScopeFactory serviceScopeFactory, + IFFmpegSegmenterService ffmpegSegmenterService) { - _ffmpegSegmenterService = ffmpegSegmenterService; _localFileSystem = localFileSystem; - _channel = channel; _logger = logger; + _serviceScopeFactory = serviceScopeFactory; + _ffmpegSegmenterService = ffmpegSegmenterService; } public Task> Handle(StartFFmpegSession request, CancellationToken cancellationToken) => @@ -43,24 +42,51 @@ namespace ErsatzTV.Application.Streaming.Commands private async Task StartProcess(StartFFmpegSession request) { - await _channel.WriteAsync(request); + using IServiceScope scope = _serviceScopeFactory.CreateScope(); + HlsSessionWorker worker = scope.ServiceProvider.GetRequiredService(); + _ffmpegSegmenterService.SessionWorkers.AddOrUpdate(request.ChannelNumber, _ => worker, (_, _) => worker); - // TODO: find some other way to let ffmpeg get ahead - await Task.Delay(FFmpegSegmenterService.SegmenterDelay); + // fire and forget worker + _ = worker.Run(request.ChannelNumber) + .ContinueWith( + _ => _ffmpegSegmenterService.SessionWorkers.TryRemove( + request.ChannelNumber, + out IHlsSessionWorker _), + TaskScheduler.Default); + + string playlistFileName = Path.Combine( + FileSystemLayout.TranscodeFolder, + request.ChannelNumber, + "live.m3u8"); + + while (!File.Exists(playlistFileName)) + { + await Task.Delay(TimeSpan.FromMilliseconds(100)); + } return Unit.Default; } private Task> Validate(StartFFmpegSession request) => - ProcessMustNotExist(request) + SessionMustBeInactive(request) .BindT(_ => FolderMustBeEmpty(request)); - private Task> ProcessMustNotExist(StartFFmpegSession request) => - Optional(_ffmpegSegmenterService.ProcessExistsForChannel(request.ChannelNumber)) - .Filter(exists => exists == false) + private Task> SessionMustBeInactive(StartFFmpegSession request) + { + var result = Optional(_ffmpegSegmenterService.SessionWorkers.TryAdd(request.ChannelNumber, null)) + .Filter(success => success) .Map(_ => Unit.Default) - .ToValidation(new ChannelHasProcess()) - .AsTask(); + .ToValidation(new ChannelSessionAlreadyActive()); + + if (result.IsFail && _ffmpegSegmenterService.SessionWorkers.TryGetValue( + request.ChannelNumber, + out IHlsSessionWorker worker)) + { + worker?.Touch(); + } + + return result.AsTask(); + } private Task> FolderMustBeEmpty(StartFFmpegSession request) { diff --git a/ErsatzTV.Application/Streaming/HlsSessionWorker.cs b/ErsatzTV.Application/Streaming/HlsSessionWorker.cs new file mode 100644 index 000000000..6dacb719f --- /dev/null +++ b/ErsatzTV.Application/Streaming/HlsSessionWorker.cs @@ -0,0 +1,207 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using System.Timers; +using ErsatzTV.Application.Streaming.Queries; +using ErsatzTV.Core; +using ErsatzTV.Core.FFmpeg; +using ErsatzTV.Core.Interfaces.FFmpeg; +using LanguageExt; +using MediatR; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Timer = System.Timers.Timer; + +namespace ErsatzTV.Application.Streaming +{ + public class HlsSessionWorker : IHlsSessionWorker + { + private readonly IServiceScopeFactory _serviceScopeFactory; + private readonly ILogger _logger; + private DateTimeOffset _lastAccess; + private DateTimeOffset _transcodedUntil; + private readonly Timer _timer = new(TimeSpan.FromMinutes(2).TotalMilliseconds) { AutoReset = false }; + private readonly object _sync = new(); + private DateTimeOffset _playlistStart; + + public HlsSessionWorker(IServiceScopeFactory serviceScopeFactory, ILogger logger) + { + _serviceScopeFactory = serviceScopeFactory; + _logger = logger; + } + + public DateTimeOffset PlaylistStart => _playlistStart; + + public void Touch() + { + lock (_sync) + { + _lastAccess = DateTimeOffset.Now; + + _timer.Stop(); + _timer.Start(); + } + } + + public async Task Run(string channelNumber) + { + var cts = new CancellationTokenSource(); + void Cancel(object o, ElapsedEventArgs e) => cts.Cancel(); + + try + { + _timer.Elapsed += Cancel; + + CancellationToken cancellationToken = cts.Token; + + _logger.LogInformation("Starting HLS session for channel {Channel}", channelNumber); + + Touch(); + _transcodedUntil = DateTimeOffset.Now; + _playlistStart = _transcodedUntil; + + // start initial transcode WITHOUT realtime throttle + if (!await Transcode(channelNumber, true, false, cancellationToken)) + { + return; + } + + while (!cancellationToken.IsCancellationRequested) + { + // TODO: configurable? 5 minutes? + if (DateTimeOffset.Now - _lastAccess > TimeSpan.FromMinutes(2)) + { + _logger.LogInformation("Stopping idle HLS session for channel {Channel}", channelNumber); + return; + } + + var transcodedBuffer = TimeSpan.FromSeconds( + Math.Max(0, _transcodedUntil.Subtract(DateTimeOffset.Now).TotalSeconds)); + if (transcodedBuffer <= TimeSpan.FromMinutes(1)) + { + // only use realtime encoding when we're at least 30 seconds ahead + bool realtime = transcodedBuffer >= TimeSpan.FromSeconds(30); + if (!await Transcode(channelNumber, false, realtime, cancellationToken)) + { + return; + } + } + else + { + await TrimAndDelete(channelNumber, cancellationToken); + await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); + } + } + } + finally + { + _timer.Elapsed -= Cancel; + } + } + + private async Task Transcode(string channelNumber, bool firstProcess, bool realtime, CancellationToken cancellationToken) + { + try + { + using IServiceScope scope = _serviceScopeFactory.CreateScope(); + IMediator mediator = scope.ServiceProvider.GetRequiredService(); + + var request = new GetPlayoutItemProcessByChannelNumber( + channelNumber, + "segmenter", + firstProcess ? DateTimeOffset.Now : _transcodedUntil.AddSeconds(1), + !firstProcess, + realtime); + + // _logger.LogInformation("Request {@Request}", request); + + Either result = await mediator.Send(request, cancellationToken); + + // _logger.LogInformation("Result {Result}", result.ToString()); + + foreach (BaseError error in result.LeftAsEnumerable()) + { + _logger.LogWarning( + "Failed to create process for HLS session on channel {Channel}: {Error}", + channelNumber, + error.ToString()); + + return false; + } + + foreach (PlayoutItemProcessModel processModel in result.RightAsEnumerable()) + { + await TrimAndDelete(channelNumber, cancellationToken); + + Process process = processModel.Process; + + _logger.LogDebug( + "ffmpeg hls arguments {FFmpegArguments}", + string.Join(" ", process.StartInfo.ArgumentList)); + + process.Start(); + try + { + await process.WaitForExitAsync(cancellationToken); + process.WaitForExit(); + } + catch (TaskCanceledException) + { + _logger.LogInformation("Terminating HLS process for channel {Channel}", channelNumber); + process.Kill(); + process.WaitForExit(); + + return false; + } + + _logger.LogInformation("HLS process has completed for channel {Channel}", channelNumber); + + _transcodedUntil = processModel.Until; + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Error transcoding channel {Channel}", channelNumber); + return false; + } + + return true; + } + + private async Task TrimAndDelete(string channelNumber, CancellationToken cancellationToken) + { + string playlistFileName = Path.Combine( + FileSystemLayout.TranscodeFolder, + channelNumber, + "live.m3u8"); + + if (File.Exists(playlistFileName)) + { + // trim playlist and insert discontinuity before appending with new ffmpeg process + string[] lines = await File.ReadAllLinesAsync(playlistFileName, cancellationToken); + TrimPlaylistResult trimResult = HlsPlaylistFilter.TrimPlaylistWithDiscontinuity( + _playlistStart, + DateTimeOffset.Now.AddMinutes(-1), + lines); + await File.WriteAllTextAsync(playlistFileName, trimResult.Playlist, cancellationToken); + + // delete old segments + foreach (string file in Directory.GetFiles( + Path.Combine(FileSystemLayout.TranscodeFolder, channelNumber), + "*.ts")) + { + string fileName = Path.GetFileName(file); + if (fileName.StartsWith("live") && int.Parse(fileName.Replace("live", string.Empty).Split('.')[0]) < + int.Parse(trimResult.Sequence)) + { + File.Delete(file); + } + } + + _playlistStart = trimResult.PlaylistStart; + } + } + } +} diff --git a/ErsatzTV.Application/Streaming/PlayoutItemProcessModel.cs b/ErsatzTV.Application/Streaming/PlayoutItemProcessModel.cs new file mode 100644 index 000000000..281790e0a --- /dev/null +++ b/ErsatzTV.Application/Streaming/PlayoutItemProcessModel.cs @@ -0,0 +1,7 @@ +using System; +using System.Diagnostics; + +namespace ErsatzTV.Application.Streaming +{ + public record PlayoutItemProcessModel(Process Process, DateTimeOffset Until); +} diff --git a/ErsatzTV.Application/Streaming/Queries/FFmpegProcessHandler.cs b/ErsatzTV.Application/Streaming/Queries/FFmpegProcessHandler.cs index 6ee0d3517..130f504c1 100644 --- a/ErsatzTV.Application/Streaming/Queries/FFmpegProcessHandler.cs +++ b/ErsatzTV.Application/Streaming/Queries/FFmpegProcessHandler.cs @@ -1,5 +1,4 @@ using System; -using System.Diagnostics; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -14,7 +13,7 @@ using static LanguageExt.Prelude; namespace ErsatzTV.Application.Streaming.Queries { - public abstract class FFmpegProcessHandler : IRequestHandler> + public abstract class FFmpegProcessHandler : IRequestHandler> where T : FFmpegProcessRequest { private readonly IDbContextFactory _dbContextFactory; @@ -22,16 +21,16 @@ namespace ErsatzTV.Application.Streaming.Queries protected FFmpegProcessHandler(IDbContextFactory dbContextFactory) => _dbContextFactory = dbContextFactory; - public async Task> Handle(T request, CancellationToken cancellationToken) + public async Task> Handle(T request, CancellationToken cancellationToken) { await using TvContext dbContext = _dbContextFactory.CreateDbContext(); Validation> validation = await Validate(dbContext, request); return await validation.Match( tuple => GetProcess(dbContext, request, tuple.Item1, tuple.Item2), - error => Task.FromResult>(error.Join())); + error => Task.FromResult>(error.Join())); } - protected abstract Task> GetProcess( + protected abstract Task> GetProcess( TvContext dbContext, T request, Channel channel, diff --git a/ErsatzTV.Application/Streaming/Queries/FFmpegProcessRequest.cs b/ErsatzTV.Application/Streaming/Queries/FFmpegProcessRequest.cs index ce16acd38..4bdf63527 100644 --- a/ErsatzTV.Application/Streaming/Queries/FFmpegProcessRequest.cs +++ b/ErsatzTV.Application/Streaming/Queries/FFmpegProcessRequest.cs @@ -1,4 +1,4 @@ -using System.Diagnostics; +using System; using ErsatzTV.Core; using LanguageExt; using MediatR; @@ -6,5 +6,10 @@ using MediatR; namespace ErsatzTV.Application.Streaming.Queries { public record FFmpegProcessRequest - (string ChannelNumber, string Mode, bool StartAtZero) : IRequest>; + ( + string ChannelNumber, + string Mode, + DateTimeOffset Now, + bool StartAtZero, + bool HlsRealtime) : IRequest>; } diff --git a/ErsatzTV.Application/Streaming/Queries/GetConcatProcessByChannelNumber.cs b/ErsatzTV.Application/Streaming/Queries/GetConcatProcessByChannelNumber.cs index 2d6944cb7..7a526476e 100644 --- a/ErsatzTV.Application/Streaming/Queries/GetConcatProcessByChannelNumber.cs +++ b/ErsatzTV.Application/Streaming/Queries/GetConcatProcessByChannelNumber.cs @@ -1,11 +1,15 @@ -namespace ErsatzTV.Application.Streaming.Queries +using System; + +namespace ErsatzTV.Application.Streaming.Queries { public record GetConcatProcessByChannelNumber : FFmpegProcessRequest { public GetConcatProcessByChannelNumber(string scheme, string host, string channelNumber) : base( channelNumber, "ts", - false) + DateTimeOffset.Now, + false, + true) { Scheme = scheme; Host = host; diff --git a/ErsatzTV.Application/Streaming/Queries/GetConcatProcessByChannelNumberHandler.cs b/ErsatzTV.Application/Streaming/Queries/GetConcatProcessByChannelNumberHandler.cs index 0762056f0..8da833d19 100644 --- a/ErsatzTV.Application/Streaming/Queries/GetConcatProcessByChannelNumberHandler.cs +++ b/ErsatzTV.Application/Streaming/Queries/GetConcatProcessByChannelNumberHandler.cs @@ -1,4 +1,5 @@ -using System.Diagnostics; +using System; +using System.Diagnostics; using System.Runtime.InteropServices; using System.Threading.Tasks; using ErsatzTV.Core; @@ -27,7 +28,7 @@ namespace ErsatzTV.Application.Streaming.Queries _runtimeInfo = runtimeInfo; } - protected override async Task> GetProcess( + protected override async Task> GetProcess( TvContext dbContext, GetConcatProcessByChannelNumber request, Channel channel, @@ -37,12 +38,14 @@ namespace ErsatzTV.Application.Streaming.Queries .GetValue(ConfigElementKey.FFmpegSaveReports) .Map(result => result.IfNone(false)); - return _ffmpegProcessService.ConcatChannel( + Process process = _ffmpegProcessService.ConcatChannel( ffmpegPath, saveReports, channel, request.Scheme, request.Host); + + return new PlayoutItemProcessModel(process, DateTimeOffset.MaxValue); } } } diff --git a/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumber.cs b/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumber.cs index 50e0d8cb3..53606e173 100644 --- a/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumber.cs +++ b/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumber.cs @@ -1,11 +1,20 @@ -namespace ErsatzTV.Application.Streaming.Queries +using System; + +namespace ErsatzTV.Application.Streaming.Queries { public record GetPlayoutItemProcessByChannelNumber : FFmpegProcessRequest { - public GetPlayoutItemProcessByChannelNumber(string channelNumber, string mode, bool startAtZero) : base( + public GetPlayoutItemProcessByChannelNumber( + string channelNumber, + string mode, + DateTimeOffset now, + bool startAtZero, + bool hlsRealtime) : base( channelNumber, mode, - startAtZero) + now, + startAtZero, + hlsRealtime) { } } diff --git a/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs b/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs index 8c56ae21c..6fdc76e4d 100644 --- a/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs +++ b/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs @@ -48,17 +48,13 @@ namespace ErsatzTV.Application.Streaming.Queries _runtimeInfo = runtimeInfo; } - protected override async Task> GetProcess( + protected override async Task> GetProcess( TvContext dbContext, GetPlayoutItemProcessByChannelNumber request, Channel channel, string ffmpegPath) { - DateTimeOffset now = request.Mode switch - { - "segmenter" => DateTimeOffset.Now + FFmpegSegmenterService.SegmenterDelay, - _ => DateTimeOffset.Now - }; + DateTimeOffset now = request.Now; Either maybePlayoutItem = await dbContext.PlayoutItems .Include(i => i.MediaItem) @@ -108,18 +104,22 @@ namespace ErsatzTV.Application.Streaming.Queries .GetValue(ConfigElementKey.FFmpegVaapiDriver) .MapT(i => (VaapiDriver)i); - return Right( - await _ffmpegProcessService.ForPlayoutItem( - ffmpegPath, - saveReports, - channel, - version, - playoutItemWithPath.Path, - playoutItemWithPath.PlayoutItem.StartOffset, - request.StartAtZero ? playoutItemWithPath.PlayoutItem.StartOffset : now, - maybeGlobalWatermark, - maybeVaapiDriver, - request.StartAtZero)); + Process process = await _ffmpegProcessService.ForPlayoutItem( + ffmpegPath, + saveReports, + channel, + version, + playoutItemWithPath.Path, + playoutItemWithPath.PlayoutItem.StartOffset, + request.StartAtZero ? playoutItemWithPath.PlayoutItem.StartOffset : now, + maybeGlobalWatermark, + maybeVaapiDriver, + request.StartAtZero, + request.HlsRealtime); + + var result = new PlayoutItemProcessModel(process, playoutItemWithPath.PlayoutItem.FinishOffset); + + return Right(result); }, async error => { @@ -138,16 +138,21 @@ namespace ErsatzTV.Application.Streaming.Queries .MapT(pi => pi.StartOffset - now), () => Option.None.AsTask()); + DateTimeOffset finish = maybeDuration.Match(d => now.Add(d), () => now); + switch (error) { case UnableToLocatePlayoutItem: if (channel.FFmpegProfile.Transcode) { - return _ffmpegProcessService.ForError( + Process errorProcess = _ffmpegProcessService.ForError( ffmpegPath, channel, maybeDuration, "Channel is Offline"); + + + return new PlayoutItemProcessModel(errorProcess, finish); } else { @@ -159,7 +164,13 @@ namespace ErsatzTV.Application.Streaming.Queries case PlayoutItemDoesNotExistOnDisk: if (channel.FFmpegProfile.Transcode) { - return _ffmpegProcessService.ForError(ffmpegPath, channel, maybeDuration, error.Value); + Process errorProcess = _ffmpegProcessService.ForError( + ffmpegPath, + channel, + maybeDuration, + error.Value); + + return new PlayoutItemProcessModel(errorProcess, finish); } else { @@ -171,11 +182,13 @@ namespace ErsatzTV.Application.Streaming.Queries default: if (channel.FFmpegProfile.Transcode) { - return _ffmpegProcessService.ForError( + Process errorProcess = _ffmpegProcessService.ForError( ffmpegPath, channel, maybeDuration, "Channel is Offline"); + + return new PlayoutItemProcessModel(errorProcess, finish); } else { diff --git a/ErsatzTV.Core.Tests/ErsatzTV.Core.Tests.csproj b/ErsatzTV.Core.Tests/ErsatzTV.Core.Tests.csproj index 041fa74f0..01fda11e5 100644 --- a/ErsatzTV.Core.Tests/ErsatzTV.Core.Tests.csproj +++ b/ErsatzTV.Core.Tests/ErsatzTV.Core.Tests.csproj @@ -6,10 +6,6 @@ - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs b/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs index 315403c77..d938a3d40 100644 --- a/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs +++ b/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs @@ -185,6 +185,7 @@ namespace ErsatzTV.Core.Tests.FFmpeg now, None, None, + false, false); process.StartInfo.RedirectStandardError = true; diff --git a/ErsatzTV.Core/Errors/ChannelHasProcess.cs b/ErsatzTV.Core/Errors/ChannelHasProcess.cs deleted file mode 100644 index fcdb5aed9..000000000 --- a/ErsatzTV.Core/Errors/ChannelHasProcess.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace ErsatzTV.Core.Errors -{ - public class ChannelHasProcess : BaseError - { - public ChannelHasProcess() : base("Channel already has ffmpeg process") - { - } - } -} diff --git a/ErsatzTV.Core/Errors/ChannelSessionAlreadyActive.cs b/ErsatzTV.Core/Errors/ChannelSessionAlreadyActive.cs new file mode 100644 index 000000000..c0cfdbb08 --- /dev/null +++ b/ErsatzTV.Core/Errors/ChannelSessionAlreadyActive.cs @@ -0,0 +1,9 @@ +namespace ErsatzTV.Core.Errors +{ + public class ChannelSessionAlreadyActive : BaseError + { + public ChannelSessionAlreadyActive() : base("Channel already has HLS session") + { + } + } +} diff --git a/ErsatzTV.Core/ErsatzTV.Core.csproj b/ErsatzTV.Core/ErsatzTV.Core.csproj index f529efc9a..0900a1029 100644 --- a/ErsatzTV.Core/ErsatzTV.Core.csproj +++ b/ErsatzTV.Core/ErsatzTV.Core.csproj @@ -7,10 +7,6 @@ - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/ErsatzTV.Core/FFmpeg/FFmpegProcessBuilder.cs b/ErsatzTV.Core/FFmpeg/FFmpegProcessBuilder.cs index c16fc0891..110194e5a 100644 --- a/ErsatzTV.Core/FFmpeg/FFmpegProcessBuilder.cs +++ b/ErsatzTV.Core/FFmpeg/FFmpegProcessBuilder.cs @@ -109,7 +109,14 @@ namespace ErsatzTV.Core.FFmpeg { if (realtimeOutput) { - _arguments.Add("-re"); + if (!_arguments.Contains("-re")) + { + _arguments.Add("-re"); + } + } + else + { + _arguments.RemoveAll(s => s == "-re"); } return this; @@ -312,8 +319,8 @@ namespace ErsatzTV.Core.FFmpeg public FFmpegProcessBuilder WithHls(string channelNumber, MediaVersion mediaVersion, bool startAtZero) { - const int INITIAL_SEGMENT_SECONDS = 1; - const int SUBSEQUENT_SEGMENT_SECONDS = 2; + const int INITIAL_SEGMENT_SECONDS = 4; + const int SUBSEQUENT_SEGMENT_SECONDS = 4; if (!int.TryParse(mediaVersion.RFrameRate, out int frameRate)) { @@ -340,9 +347,9 @@ namespace ErsatzTV.Core.FFmpeg "-force_key_frames", $"expr:gte(t,n_forced*{segmentSeconds})", "-f", "hls", "-hls_time", $"{segmentSeconds}", - "-hls_list_size", "10", + "-hls_list_size", "0", "-segment_list_flags", "+live", - "-hls_flags", "delete_segments+program_date_time+append_list+discont_start+omit_endlist", + "-hls_flags", "program_date_time+append_list+omit_endlist+independent_segments", Path.Combine(FileSystemLayout.TranscodeFolder, channelNumber, "live.m3u8") }); diff --git a/ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs b/ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs index 63ec0aed6..ad136a77e 100644 --- a/ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs +++ b/ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs @@ -40,7 +40,8 @@ namespace ErsatzTV.Core.FFmpeg DateTimeOffset now, Option globalWatermark, Option maybeVaapiDriver, - bool startAtZero) + bool startAtZero, + bool hlsRealtime) { MediaStream videoStream = await _ffmpegStreamSelector.SelectVideoStream(channel, version); Option maybeAudioStream = await _ffmpegStreamSelector.SelectAudioStream(channel, version); @@ -120,6 +121,7 @@ namespace ErsatzTV.Core.FFmpeg // HLS needs to segment and generate playlist case StreamingMode.HttpLiveStreamingSegmenter: return builder.WithHls(channel.Number, version, startAtZero) + .WithRealtimeOutput(hlsRealtime) .Build(); default: return builder.WithFormat("mpegts") diff --git a/ErsatzTV.Core/FFmpeg/FFmpegSegmenterService.cs b/ErsatzTV.Core/FFmpeg/FFmpegSegmenterService.cs index cfd36beb9..c3229862e 100644 --- a/ErsatzTV.Core/FFmpeg/FFmpegSegmenterService.cs +++ b/ErsatzTV.Core/FFmpeg/FFmpegSegmenterService.cs @@ -1,98 +1,23 @@ -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Threading; +using System.Collections.Concurrent; using ErsatzTV.Core.Interfaces.FFmpeg; -using LanguageExt; -using Microsoft.Extensions.Logging; namespace ErsatzTV.Core.FFmpeg { public class FFmpegSegmenterService : IFFmpegSegmenterService { - public static readonly TimeSpan SegmenterDelay = TimeSpan.FromSeconds(3); - - private static readonly ConcurrentDictionary Processes = new(); - - private readonly ILogger _logger; - - public FFmpegSegmenterService(ILogger logger) => _logger = logger; - - public bool ProcessExistsForChannel(string channelNumber) + public FFmpegSegmenterService() { - if (Processes.TryGetValue(channelNumber, out ProcessAndToken processAndToken)) - { - if (!processAndToken.Process.HasExited || !Processes.TryRemove( - new KeyValuePair(channelNumber, processAndToken))) - { - return true; - } - } - - return false; + SessionWorkers = new ConcurrentDictionary(); } - public bool TryAdd(string channelNumber, Process process) - { - var cts = new CancellationTokenSource(); - var processAndToken = new ProcessAndToken(process, cts, DateTimeOffset.Now); - if (Processes.TryAdd(channelNumber, processAndToken)) - { - CancellationToken token = cts.Token; - token.Register(process.Kill); - return true; - } - - return false; - } + public ConcurrentDictionary SessionWorkers { get; } public void TouchChannel(string channelNumber) { - if (Processes.TryGetValue(channelNumber, out ProcessAndToken processAndToken)) + if (SessionWorkers.TryGetValue(channelNumber, out IHlsSessionWorker worker)) { - ProcessAndToken newValue = processAndToken with { LastAccess = DateTimeOffset.Now }; - if (!Processes.TryUpdate(channelNumber, newValue, processAndToken)) - { - _logger.LogWarning("Failed to update last access for channel {Channel}", channelNumber); - } + worker?.Touch(); } } - - public void CleanUpSessions() - { - foreach ((string key, (_, CancellationTokenSource cts, DateTimeOffset lastAccess)) in Processes.ToList()) - { - // TODO: configure this time span? 5 min? - if (DateTimeOffset.Now.Subtract(lastAccess) > TimeSpan.FromMinutes(2)) - { - _logger.LogDebug("Cleaning up ffmpeg session for channel {Channel}", key); - - cts.Cancel(); - Processes.TryRemove(key, out _); - } - } - } - - public Unit KillAll() - { - foreach ((string key, ProcessAndToken processAndToken) in Processes.ToList()) - { - try - { - processAndToken.TokenSource.Cancel(); - Processes.TryRemove(key, out _); - } - catch (Exception ex) - { - _logger.LogInformation(ex, "Error killing process"); - } - } - - return Unit.Default; - } - - private record ProcessAndToken(Process Process, CancellationTokenSource TokenSource, DateTimeOffset LastAccess); } } diff --git a/ErsatzTV.Core/FFmpeg/HlsPlaylistFilter.cs b/ErsatzTV.Core/FFmpeg/HlsPlaylistFilter.cs new file mode 100644 index 000000000..b7dde91f6 --- /dev/null +++ b/ErsatzTV.Core/FFmpeg/HlsPlaylistFilter.cs @@ -0,0 +1,111 @@ +using System; +using System.Text; + +namespace ErsatzTV.Core.FFmpeg +{ + public class HlsPlaylistFilter + { + public static TrimPlaylistResult TrimPlaylist( + DateTimeOffset playlistStart, + DateTimeOffset filterBefore, + string[] lines, + int maxSegments = 10, + bool endWithDiscontinuity = false) + { + DateTimeOffset currentTime = playlistStart; + DateTimeOffset nextPlaylistStart = DateTimeOffset.MaxValue; + + var discontinuitySequence = 0; + var startSequence = "0"; + var output = new StringBuilder(); + var started = false; + var i = 0; + var segments = 0; + while (!lines[i].StartsWith("#EXTINF:")) + { + if (lines[i].StartsWith("#EXT-X-DISCONTINUITY-SEQUENCE")) + { + discontinuitySequence = int.Parse(lines[i].Split(':')[1]); + } + + i++; + } + + while (i < lines.Length) + { + if (segments >= maxSegments) + { + break; + } + + string line = lines[i]; + // _logger.LogInformation("Line: {Line}", line); + if (line.StartsWith("#EXT-X-DISCONTINUITY")) + { + if (started) + { + output.AppendLine("#EXT-X-DISCONTINUITY"); + } + else + { + discontinuitySequence++; + } + + i++; + continue; + } + + var duration = TimeSpan.FromSeconds(double.Parse(lines[i].TrimEnd(',').Split(':')[1])); + if (currentTime < filterBefore) + { + currentTime += duration; + i += 3; + continue; + } + + nextPlaylistStart = currentTime < nextPlaylistStart ? currentTime : nextPlaylistStart; + + if (!started) + { + startSequence = lines[i + 2].Replace("live", string.Empty).Split('.')[0]; + + output.AppendLine("#EXTM3U"); + output.AppendLine("#EXT-X-VERSION:3"); + output.AppendLine("#EXT-X-TARGETDURATION:4"); + output.AppendLine($"#EXT-X-MEDIA-SEQUENCE:{startSequence}"); + output.AppendLine($"#EXT-X-DISCONTINUITY-SEQUENCE:{discontinuitySequence}"); + output.AppendLine("#EXT-X-INDEPENDENT-SEGMENTS"); + output.AppendLine("#EXT-X-DISCONTINUITY"); + + started = true; + } + + output.AppendLine(lines[i]); + string offset = currentTime.ToString("zzz").Replace(":", string.Empty); + output.AppendLine($"#EXT-X-PROGRAM-DATE-TIME:{currentTime:yyyy-MM-ddTHH:mm:ss.fff}{offset}"); + output.AppendLine(lines[i + 2]); + + currentTime += duration; + segments++; + i += 3; + } + + if (endWithDiscontinuity) + { + output.AppendLine("#EXT-X-DISCONTINUITY"); + } + + return new TrimPlaylistResult(nextPlaylistStart, startSequence, output.ToString()); + } + + public static TrimPlaylistResult TrimPlaylistWithDiscontinuity( + DateTimeOffset playlistStart, + DateTimeOffset filterBefore, + string[] lines) + { + return TrimPlaylist(playlistStart, filterBefore, lines, int.MaxValue, true); + } + } + + public record TrimPlaylistResult(DateTimeOffset PlaylistStart, string Sequence, string Playlist); +} diff --git a/ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegSegmenterService.cs b/ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegSegmenterService.cs index 3c518daec..32da42aa8 100644 --- a/ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegSegmenterService.cs +++ b/ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegSegmenterService.cs @@ -1,14 +1,11 @@ -using System.Diagnostics; -using LanguageExt; +using System.Collections.Concurrent; namespace ErsatzTV.Core.Interfaces.FFmpeg { public interface IFFmpegSegmenterService { - bool ProcessExistsForChannel(string channelNumber); - bool TryAdd(string channelNumber, Process process); + ConcurrentDictionary SessionWorkers { get; } + void TouchChannel(string channelNumber); - void CleanUpSessions(); - Unit KillAll(); } } diff --git a/ErsatzTV.Core/Interfaces/FFmpeg/IHlsSessionWorker.cs b/ErsatzTV.Core/Interfaces/FFmpeg/IHlsSessionWorker.cs new file mode 100644 index 000000000..cfd24b863 --- /dev/null +++ b/ErsatzTV.Core/Interfaces/FFmpeg/IHlsSessionWorker.cs @@ -0,0 +1,10 @@ +using System; + +namespace ErsatzTV.Core.Interfaces.FFmpeg +{ + public interface IHlsSessionWorker + { + DateTimeOffset PlaylistStart { get; } + void Touch(); + } +} diff --git a/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj b/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj index b286a0438..a547e4b8f 100644 --- a/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj +++ b/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj @@ -8,10 +8,6 @@ - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/ErsatzTV/Controllers/InternalController.cs b/ErsatzTV/Controllers/InternalController.cs index 834b3aa70..22a829314 100644 --- a/ErsatzTV/Controllers/InternalController.cs +++ b/ErsatzTV/Controllers/InternalController.cs @@ -1,4 +1,6 @@ -using System.Threading.Tasks; +using System; +using System.Diagnostics; +using System.Threading.Tasks; using ErsatzTV.Application.Streaming.Queries; using ErsatzTV.Extensions; using LanguageExt; @@ -31,25 +33,29 @@ namespace ErsatzTV.Controllers string channelNumber, [FromQuery] string mode = "mixed") => - _mediator.Send(new GetPlayoutItemProcessByChannelNumber(channelNumber, mode, false)).Map( - result => - result.Match( - process => - { - _logger.LogDebug( - "ffmpeg arguments {FFmpegArguments}", - string.Join(" ", process.StartInfo.ArgumentList)); - process.Start(); - return new FileStreamResult(process.StandardOutput.BaseStream, "video/mp2t"); - }, - error => - { - _logger.LogError( - "Failed to create stream for channel {ChannelNumber}: {Error}", - channelNumber, - error.Value); - return BadRequest(error.Value); - } - )); + _mediator.Send( + new GetPlayoutItemProcessByChannelNumber(channelNumber, mode, DateTimeOffset.Now, false, true)) + .Map( + result => + result.Match( + processModel => + { + Process process = processModel.Process; + + _logger.LogDebug( + "ffmpeg arguments {FFmpegArguments}", + string.Join(" ", process.StartInfo.ArgumentList)); + process.Start(); + return new FileStreamResult(process.StandardOutput.BaseStream, "video/mp2t"); + }, + error => + { + _logger.LogError( + "Failed to create stream for channel {ChannelNumber}: {Error}", + channelNumber, + error.Value); + return BadRequest(error.Value); + } + )); } } diff --git a/ErsatzTV/Controllers/IptvController.cs b/ErsatzTV/Controllers/IptvController.cs index 883f63f2e..3da4e2539 100644 --- a/ErsatzTV/Controllers/IptvController.cs +++ b/ErsatzTV/Controllers/IptvController.cs @@ -1,6 +1,7 @@ -using System.Threading.Channels; +using System; +using System.Diagnostics; +using System.IO; using System.Threading.Tasks; -using ErsatzTV.Application; using ErsatzTV.Application.Channels.Queries; using ErsatzTV.Application.Images; using ErsatzTV.Application.Images.Queries; @@ -9,6 +10,8 @@ using ErsatzTV.Application.Streaming.Queries; using ErsatzTV.Core; using ErsatzTV.Core.Domain; using ErsatzTV.Core.Errors; +using ErsatzTV.Core.FFmpeg; +using ErsatzTV.Core.Interfaces.FFmpeg; using ErsatzTV.Core.Iptv; using LanguageExt; using MediatR; @@ -22,18 +25,18 @@ namespace ErsatzTV.Controllers [ApiExplorerSettings(IgnoreApi = true)] public class IptvController : ControllerBase { - private readonly ChannelWriter _channel; + private readonly IFFmpegSegmenterService _ffmpegSegmenterService; private readonly ILogger _logger; private readonly IMediator _mediator; public IptvController( IMediator mediator, ILogger logger, - ChannelWriter channel) + IFFmpegSegmenterService ffmpegSegmenterService) { _mediator = mediator; _logger = logger; - _channel = channel; + _ffmpegSegmenterService = ffmpegSegmenterService; } [HttpGet("iptv/channels.m3u")] @@ -53,8 +56,10 @@ namespace ErsatzTV.Controllers _mediator.Send(new GetConcatProcessByChannelNumber(Request.Scheme, Request.Host.ToString(), channelNumber)) .Map( result => result.Match( - process => + processModel => { + Process process = processModel.Process; + _logger.LogInformation("Starting ts stream for channel {ChannelNumber}", channelNumber); // _logger.LogDebug( // "ffmpeg concat arguments {FFmpegArguments}", @@ -64,6 +69,26 @@ namespace ErsatzTV.Controllers }, error => BadRequest(error.Value))); + [HttpGet("iptv/session/{channelNumber}/hls.m3u8")] + public async Task GetLivePlaylist(string channelNumber) + { + if (_ffmpegSegmenterService.SessionWorkers.TryGetValue(channelNumber, out IHlsSessionWorker worker)) + { + DateTimeOffset now = DateTimeOffset.Now.AddSeconds(-30); + + string fileName = Path.Combine(FileSystemLayout.TranscodeFolder, channelNumber, "live.m3u8"); + if (System.IO.File.Exists(fileName)) + { + string[] input = await System.IO.File.ReadAllLinesAsync(fileName); + + TrimPlaylistResult result = HlsPlaylistFilter.TrimPlaylist(worker.PlaylistStart, now, input); + return Content(result.Playlist, "application/vnd.apple.mpegurl"); + } + } + + return NotFound(); + } + [HttpGet("iptv/channel/{channelNumber}.m3u8")] public async Task GetHttpLiveStreamingVideo( string channelNumber, @@ -75,13 +100,13 @@ namespace ErsatzTV.Controllers case "segmenter": Either result = await _mediator.Send(new StartFFmpegSession(channelNumber, false)); return result.Match( - _ => Redirect($"/iptv/session/{channelNumber}/live.m3u8"), + _ => Redirect($"/iptv/session/{channelNumber}/hls.m3u8"), error => { switch (error) { - case ChannelHasProcess: - return RedirectPreserveMethod($"/iptv/session/{channelNumber}/live.m3u8"); + case ChannelSessionAlreadyActive: + return RedirectPreserveMethod($"/iptv/session/{channelNumber}/hls.m3u8"); default: _logger.LogWarning( "Failed to start segmenter for channel {ChannelNumber}: {Error}", diff --git a/ErsatzTV/ErsatzTV.csproj b/ErsatzTV/ErsatzTV.csproj index 62e80ef1c..e3dbb9b61 100644 --- a/ErsatzTV/ErsatzTV.csproj +++ b/ErsatzTV/ErsatzTV.csproj @@ -13,10 +13,6 @@ - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/ErsatzTV/Services/FFmpegSchedulerService.cs b/ErsatzTV/Services/FFmpegSchedulerService.cs deleted file mode 100644 index fc59f847a..000000000 --- a/ErsatzTV/Services/FFmpegSchedulerService.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Channels; -using System.Threading.Tasks; -using ErsatzTV.Application; -using ErsatzTV.Application.Streaming.Commands; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; - -namespace ErsatzTV.Services -{ - public class FFmpegSchedulerService : BackgroundService - { - private readonly ILogger _logger; - private readonly ChannelWriter _workerChannel; - - public FFmpegSchedulerService( - ChannelWriter workerChannel, - ILogger logger) - { - _workerChannel = workerChannel; - _logger = logger; - } - - protected override async Task ExecuteAsync(CancellationToken cancellationToken) - { - while (!cancellationToken.IsCancellationRequested) - { - await _workerChannel.WriteAsync(new CleanUpFFmpegSessions(), cancellationToken); - await Task.Delay(TimeSpan.FromMinutes(1), cancellationToken); - } - } - } -} diff --git a/ErsatzTV/Services/FFmpegWorkerService.cs b/ErsatzTV/Services/FFmpegWorkerService.cs index 824b4fa29..deb407779 100644 --- a/ErsatzTV/Services/FFmpegWorkerService.cs +++ b/ErsatzTV/Services/FFmpegWorkerService.cs @@ -1,16 +1,11 @@ using System; -using System.Diagnostics; using System.IO; using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; using ErsatzTV.Application; using ErsatzTV.Application.Streaming.Commands; -using ErsatzTV.Application.Streaming.Queries; -using ErsatzTV.Core; using ErsatzTV.Core.Interfaces.FFmpeg; -using LanguageExt; -using MediatR; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -21,20 +16,17 @@ namespace ErsatzTV.Services public class FFmpegWorkerService : BackgroundService { private readonly ChannelReader _channel; - private readonly ChannelWriter _channelWriter; private readonly ILogger _logger; private readonly IFFmpegSegmenterService _ffmpegSegmenterService; private readonly IServiceScopeFactory _serviceScopeFactory; public FFmpegWorkerService( ChannelReader channel, - ChannelWriter channelWriter, IServiceScopeFactory serviceScopeFactory, ILogger logger, IFFmpegSegmenterService ffmpegSegmenterService) { _channel = channel; - _channelWriter = channelWriter; _serviceScopeFactory = serviceScopeFactory; _logger = logger; _ffmpegSegmenterService = ffmpegSegmenterService; @@ -49,7 +41,7 @@ namespace ErsatzTV.Services try { using IServiceScope scope = _serviceScopeFactory.CreateScope(); - IMediator mediator = scope.ServiceProvider.GetRequiredService(); + // IMediator mediator = scope.ServiceProvider.GetRequiredService(); switch (request) { @@ -58,52 +50,6 @@ namespace ErsatzTV.Services { _ffmpegSegmenterService.TouchChannel(parent.Name); } - break; - case CleanUpFFmpegSessions: - _ffmpegSegmenterService.CleanUpSessions(); - break; - case StartFFmpegSession startFFmpegSession: - _logger.LogInformation( - "Starting ffmpeg session for channel {Channel}", - startFFmpegSession.ChannelNumber); - - if (!_ffmpegSegmenterService.ProcessExistsForChannel(startFFmpegSession.ChannelNumber)) - { - var req = new GetPlayoutItemProcessByChannelNumber( - startFFmpegSession.ChannelNumber, - "segmenter", - startFFmpegSession.StartAtZero); - Either maybeProcess = await mediator.Send(req, cancellationToken); - maybeProcess.Match( - process => - { - if (_ffmpegSegmenterService.TryAdd(startFFmpegSession.ChannelNumber, process)) - { - _logger.LogDebug( - "ffmpeg hls arguments {FFmpegArguments}", - string.Join(" ", process.StartInfo.ArgumentList)); - - process.Start(); - process.EnableRaisingEvents = true; - process.Exited += (_, _) => - { - if (process.ExitCode == 0) - { - _channelWriter.TryWrite( - new StartFFmpegSession(startFFmpegSession.ChannelNumber, true)); - } - else - { - _logger.LogDebug( - "hls segmenter for channel {Channel} exited with code {ExitCode}", - startFFmpegSession.ChannelNumber, - process.ExitCode); - } - }; - } - }, - _ => { }); - } break; } @@ -113,9 +59,6 @@ namespace ErsatzTV.Services _logger.LogWarning(ex, "Failed to handle ffmpeg worker request"); } } - - // kill any running processes after cancellation - _ffmpegSegmenterService.KillAll(); } } } diff --git a/ErsatzTV/Startup.cs b/ErsatzTV/Startup.cs index fc0dc69bb..f043de580 100644 --- a/ErsatzTV/Startup.cs +++ b/ErsatzTV/Startup.cs @@ -7,6 +7,7 @@ using Blazored.LocalStorage; using Dapper; using ErsatzTV.Application; using ErsatzTV.Application.Channels.Queries; +using ErsatzTV.Application.Streaming; using ErsatzTV.Application.Streaming.Commands; using ErsatzTV.Core; using ErsatzTV.Core.Emby; @@ -300,6 +301,7 @@ namespace ErsatzTV services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped( _ => @@ -326,7 +328,6 @@ namespace ErsatzTV services.AddHostedService(); services.AddHostedService(); services.AddHostedService(); - services.AddHostedService(); } private void AddChannel(IServiceCollection services)