timothy/ersatztv
1
0
Fork 0
Code Issues 51 Pull Requests 5 Actions Packages Projects Releases 9 Wiki Activity

log problematic playlists (#640)

Browse Source
This commit is contained in:
Jason Dove
2022-02-16 08:18:22 -06:00
committed by GitHub
parent 66ab0b3990
commit 1d6279cee8
9 changed files with 181 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ErsatzTV.Application/Streaming/Commands/StartFFmpegSessionHandler.cs
+6 -3
View File
@@ -22,6 +22,7 @@ namespace ErsatzTV.Application.Streaming.Commands
private readonly IServiceScopeFactory _serviceScopeFactory;
private readonly IFFmpegSegmenterService _ffmpegSegmenterService;
private readonly IConfigElementRepository _configElementRepository;
private readonly IHlsPlaylistFilter _hlsPlaylistFilter;
private readonly ILocalFileSystem _localFileSystem;
public StartFFmpegSessionHandler(
@@ -29,13 +30,15 @@ namespace ErsatzTV.Application.Streaming.Commands
ILogger<StartFFmpegSessionHandler> logger,
IServiceScopeFactory serviceScopeFactory,
IFFmpegSegmenterService ffmpegSegmenterService,
IConfigElementRepository configElementRepository)
IConfigElementRepository configElementRepository,
IHlsPlaylistFilter hlsPlaylistFilter)
{
_localFileSystem = localFileSystem;
_logger = logger;
_serviceScopeFactory = serviceScopeFactory;
_ffmpegSegmenterService = ffmpegSegmenterService;
_configElementRepository = configElementRepository;
_hlsPlaylistFilter = hlsPlaylistFilter;
}
public Task<Either<BaseError, Unit>> Handle(StartFFmpegSession request, CancellationToken cancellationToken) =>
@@ -78,7 +81,7 @@ namespace ErsatzTV.Application.Streaming.Commands
return Unit.Default;
}
private static async Task WaitForPlaylistSegments(string playlistFileName, int initialSegmentCount, IHlsSessionWorker worker)
private async Task WaitForPlaylistSegments(string playlistFileName, int initialSegmentCount, IHlsSessionWorker worker)
{
while (!File.Exists(playlistFileName))
{
@@ -92,7 +95,7 @@ namespace ErsatzTV.Application.Streaming.Commands
DateTimeOffset now = DateTimeOffset.Now.AddSeconds(-30);
string[] input = await File.ReadAllLinesAsync(playlistFileName);
TrimPlaylistResult result = HlsPlaylistFilter.TrimPlaylist(worker.PlaylistStart, now, input);
TrimPlaylistResult result = _hlsPlaylistFilter.TrimPlaylist(worker.PlaylistStart, now, input);
segmentCount = result.SegmentCount;
}
}
ErsatzTV.Application/Streaming/HlsSessionWorker.cs
+17 -11
View File
@@ -24,6 +24,7 @@ namespace ErsatzTV.Application.Streaming
public class HlsSessionWorker : IHlsSessionWorker
{
private static int _workAheadCount;
private readonly IHlsPlaylistFilter _hlsPlaylistFilter;
private readonly IServiceScopeFactory _serviceScopeFactory;
private readonly ILogger<HlsSessionWorker> _logger;
private DateTimeOffset _lastAccess;
@@ -33,8 +34,9 @@ namespace ErsatzTV.Application.Streaming
private DateTimeOffset _playlistStart;
private Option<int> _targetFramerate;
public HlsSessionWorker(IServiceScopeFactory serviceScopeFactory, ILogger<HlsSessionWorker> logger)
public HlsSessionWorker(IHlsPlaylistFilter hlsPlaylistFilter, IServiceScopeFactory serviceScopeFactory, ILogger<HlsSessionWorker> logger)
{
_hlsPlaylistFilter = hlsPlaylistFilter;
_serviceScopeFactory = serviceScopeFactory;
_logger = logger;
}
@@ -123,7 +125,11 @@ namespace ErsatzTV.Application.Streaming
}
}
private async Task<bool> Transcode(string channelNumber, bool firstProcess, bool realtime, CancellationToken cancellationToken)
private async Task<bool> Transcode(
string channelNumber,
bool firstProcess,
bool realtime,
CancellationToken cancellationToken)
{
try
{
@@ -212,7 +218,7 @@ namespace ErsatzTV.Application.Streaming
return true;
}
private async Task TrimAndDelete(string channelNumber, CancellationToken cancellationToken)
{
string playlistFileName = Path.Combine(
@@ -224,7 +230,7 @@ namespace ErsatzTV.Application.Streaming
{
// trim playlist and insert discontinuity before appending with new ffmpeg process
string[] lines = await File.ReadAllLinesAsync(playlistFileName, cancellationToken);
TrimPlaylistResult trimResult = HlsPlaylistFilter.TrimPlaylistWithDiscontinuity(
TrimPlaylistResult trimResult = _hlsPlaylistFilter.TrimPlaylistWithDiscontinuity(
_playlistStart,
DateTimeOffset.Now.AddMinutes(-1),
lines);
@@ -246,13 +252,13 @@ namespace ErsatzTV.Application.Streaming
var toDelete = allSegments.Filter(s => s.SequenceNumber < trimResult.Sequence).ToList();
// if (toDelete.Count > 0)
// {
// _logger.LogInformation(
// "Deleting HLS segments {Min} to {Max} (less than {StartSequence})",
// toDelete.Map(s => s.SequenceNumber).Min(),
// toDelete.Map(s => s.SequenceNumber).Max(),
// trimResult.Sequence);
// _logger.LogInformation(
// "Deleting HLS segments {Min} to {Max} (less than {StartSequence})",
// toDelete.Map(s => s.SequenceNumber).Min(),
// toDelete.Map(s => s.SequenceNumber).Max(),
// trimResult.Sequence);
// }
foreach (Segment segment in toDelete)
{
File.Delete(segment.File);
@@ -261,7 +267,7 @@ namespace ErsatzTV.Application.Streaming
_playlistStart = trimResult.PlaylistStart;
}
}
private async Task<long> GetPtsOffset(IMediator mediator, string channelNumber, CancellationToken cancellationToken)
{
var directory = new DirectoryInfo(Path.Combine(FileSystemLayout.TranscodeFolder, channelNumber));
ErsatzTV.Core.Tests/ErsatzTV.Core.Tests.csproj
+1
View File
@@ -10,6 +10,7 @@
<PackageReference Include="LanguageExt.Core" Version="4.0.3" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="6.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
<PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.0.64">
ErsatzTV.Core.Tests/FFmpeg/HlsPlaylistFilterTests.cs
+24 -10
View File
@@ -1,6 +1,9 @@
using System;
using ErsatzTV.Core.FFmpeg;
using ErsatzTV.Core.Interfaces.FFmpeg;
using FluentAssertions;
using Microsoft.Extensions.Logging;
using Moq;
using NUnit.Framework;
namespace ErsatzTV.Core.Tests.FFmpeg
@@ -8,8 +11,19 @@ namespace ErsatzTV.Core.Tests.FFmpeg
[TestFixture]
public class HlsPlaylistFilterTests
{
private HlsPlaylistFilter _hlsPlaylistFilter;
[SetUp]
public void SetUp()
{
_hlsPlaylistFilter = new HlsPlaylistFilter(
new Mock<ITempFilePool>().Object,
new Mock<ILogger<HlsPlaylistFilter>>().Object
);
}
[Test]
public void HlsPlaylistFilter_ShouldRewriteProgramDateTime()
public void _hlsPlaylistFilter_ShouldRewriteProgramDateTime()
{
var start = new DateTimeOffset(2021, 10, 9, 8, 0, 0, TimeSpan.FromHours(-5));
string[] input = NormalizeLineEndings(@"#EXTM3U
@@ -28,7 +42,7 @@ live001138.ts
#EXT-X-PROGRAM-DATE-TIME:2021-10-08T08:34:57.320-0500
live001139.ts").Split(Environment.NewLine);
TrimPlaylistResult result = HlsPlaylistFilter.TrimPlaylist(start, start.AddSeconds(-30), input);
TrimPlaylistResult result = _hlsPlaylistFilter.TrimPlaylist(start, start.AddSeconds(-30), input);
result.PlaylistStart.Should().Be(start);
result.Sequence.Should().Be(1137);
@@ -53,7 +67,7 @@ live001139.ts
}
[Test]
public void HlsPlaylistFilter_ShouldLimitSegments()
public void _hlsPlaylistFilter_ShouldLimitSegments()
{
var start = new DateTimeOffset(2021, 10, 9, 8, 0, 0, TimeSpan.FromHours(-5));
string[] input = NormalizeLineEndings(@"#EXTM3U
@@ -72,7 +86,7 @@ live001138.ts
#EXT-X-PROGRAM-DATE-TIME:2021-10-08T08:34:57.320-0500
live001139.ts").Split(Environment.NewLine);
TrimPlaylistResult result = HlsPlaylistFilter.TrimPlaylist(start, start.AddSeconds(-30), input, 2);
TrimPlaylistResult result = _hlsPlaylistFilter.TrimPlaylist(start, start.AddSeconds(-30), input, 2);
result.PlaylistStart.Should().Be(start);
result.Sequence.Should().Be(1137);
@@ -94,7 +108,7 @@ live001138.ts
}
[Test]
public void HlsPlaylistFilter_ShouldAddDiscontinuity()
public void _hlsPlaylistFilter_ShouldAddDiscontinuity()
{
var start = new DateTimeOffset(2021, 10, 9, 8, 0, 0, TimeSpan.FromHours(-5));
string[] input = NormalizeLineEndings(@"#EXTM3U
@@ -113,7 +127,7 @@ live001138.ts
#EXT-X-PROGRAM-DATE-TIME:2021-10-08T08:34:57.320-0500
live001139.ts").Split(Environment.NewLine);
TrimPlaylistResult result = HlsPlaylistFilter.TrimPlaylist(
TrimPlaylistResult result = _hlsPlaylistFilter.TrimPlaylist(
start,
start.AddSeconds(-30),
input,
@@ -144,7 +158,7 @@ live001139.ts
}
[Test]
public void HlsPlaylistFilter_ShouldFilterOldSegments()
public void _hlsPlaylistFilter_ShouldFilterOldSegments()
{
var start = new DateTimeOffset(2021, 10, 9, 8, 0, 0, TimeSpan.FromHours(-5));
string[] input = NormalizeLineEndings(@"#EXTM3U
@@ -163,7 +177,7 @@ live001138.ts
#EXT-X-PROGRAM-DATE-TIME:2021-10-08T08:34:57.320-0500
live001139.ts").Split(Environment.NewLine);
TrimPlaylistResult result = HlsPlaylistFilter.TrimPlaylist(start, start.AddSeconds(6), input);
TrimPlaylistResult result = _hlsPlaylistFilter.TrimPlaylist(start, start.AddSeconds(6), input);
result.PlaylistStart.Should().Be(start.AddSeconds(8));
result.Sequence.Should().Be(1139);
@@ -182,7 +196,7 @@ live001139.ts
}
[Test]
public void HlsPlaylistFilter_ShouldFilterOldDiscontinuity()
public void _hlsPlaylistFilter_ShouldFilterOldDiscontinuity()
{
var start = new DateTimeOffset(2021, 10, 9, 8, 0, 0, TimeSpan.FromHours(-5));
string[] input = NormalizeLineEndings(@"#EXTM3U
@@ -202,7 +216,7 @@ live001138.ts
#EXT-X-PROGRAM-DATE-TIME:2021-10-08T08:34:57.320-0500
live001139.ts").Split(Environment.NewLine);
TrimPlaylistResult result = HlsPlaylistFilter.TrimPlaylist(start, start.AddSeconds(6), input);
TrimPlaylistResult result = _hlsPlaylistFilter.TrimPlaylist(start, start.AddSeconds(6), input);
result.PlaylistStart.Should().Be(start.AddSeconds(8));
result.Sequence.Should().Be(1139);
ErsatzTV.Core/FFmpeg/HlsPlaylistFilter.cs
+106 -72
View File
@@ -1,110 +1,144 @@
using System;
using System.Globalization;
using System.IO;
using System.Text;
using ErsatzTV.Core.Interfaces.FFmpeg;
using Microsoft.Extensions.Logging;
namespace ErsatzTV.Core.FFmpeg
{
public class HlsPlaylistFilter
public class HlsPlaylistFilter : IHlsPlaylistFilter
{
public static TrimPlaylistResult TrimPlaylist(
private readonly ITempFilePool _tempFilePool;
private readonly ILogger<HlsPlaylistFilter> _logger;
public HlsPlaylistFilter(ITempFilePool tempFilePool, ILogger<HlsPlaylistFilter> logger)
{
_tempFilePool = tempFilePool;
_logger = logger;
}
public TrimPlaylistResult TrimPlaylist(
DateTimeOffset playlistStart,
DateTimeOffset filterBefore,
string[] lines,
int maxSegments = 10,
bool endWithDiscontinuity = false)
{
DateTimeOffset currentTime = playlistStart;
DateTimeOffset nextPlaylistStart = DateTimeOffset.MaxValue;
var discontinuitySequence = 0;
var startSequence = 0;
var output = new StringBuilder();
var started = false;
var i = 0;
var segments = 0;
while (!lines[i].StartsWith("#EXTINF:"))
try
{
if (lines[i].StartsWith("#EXT-X-DISCONTINUITY-SEQUENCE"))
{
discontinuitySequence = int.Parse(lines[i].Split(':')[1]);
}
DateTimeOffset currentTime = playlistStart;
DateTimeOffset nextPlaylistStart = DateTimeOffset.MaxValue;
i++;
}
while (i < lines.Length)
{
if (segments >= maxSegments)
var discontinuitySequence = 0;
var startSequence = 0;
var output = new StringBuilder();
var started = false;
var i = 0;
var segments = 0;
while (!lines[i].StartsWith("#EXTINF:"))
{
break;
}
string line = lines[i];
// _logger.LogInformation("Line: {Line}", line);
if (line.StartsWith("#EXT-X-DISCONTINUITY"))
{
if (started)
if (lines[i].StartsWith("#EXT-X-DISCONTINUITY-SEQUENCE"))
{
output.AppendLine("#EXT-X-DISCONTINUITY");
}
else
{
discontinuitySequence++;
discontinuitySequence = int.Parse(lines[i].Split(':')[1]);
}
i++;
continue;
}
var duration = TimeSpan.FromSeconds(
double.Parse(
lines[i].TrimEnd(',').Split(':')[1],
NumberStyles.Number,
CultureInfo.InvariantCulture));
if (currentTime < filterBefore)
while (i < lines.Length)
{
if (segments >= maxSegments)
{
break;
}
string line = lines[i];
// _logger.LogInformation("Line: {Line}", line);
if (line.StartsWith("#EXT-X-DISCONTINUITY"))
{
if (started)
{
output.AppendLine("#EXT-X-DISCONTINUITY");
}
else
{
discontinuitySequence++;
}
i++;
continue;
}
var duration = TimeSpan.FromSeconds(
double.Parse(
lines[i].TrimEnd(',').Split(':')[1],
NumberStyles.Number,
CultureInfo.InvariantCulture));
if (currentTime < filterBefore)
{
currentTime += duration;
i += 3;
continue;
}
nextPlaylistStart = currentTime < nextPlaylistStart ? currentTime : nextPlaylistStart;
if (!started)
{
startSequence = int.Parse(lines[i + 2].Replace("live", string.Empty).Split('.')[0]);
output.AppendLine("#EXTM3U");
output.AppendLine("#EXT-X-VERSION:6");
output.AppendLine("#EXT-X-TARGETDURATION:4");
output.AppendLine($"#EXT-X-MEDIA-SEQUENCE:{startSequence}");
output.AppendLine($"#EXT-X-DISCONTINUITY-SEQUENCE:{discontinuitySequence}");
output.AppendLine("#EXT-X-INDEPENDENT-SEGMENTS");
output.AppendLine("#EXT-X-DISCONTINUITY");
started = true;
}
output.AppendLine(lines[i]);
string offset = currentTime.ToString("zzz").Replace(":", string.Empty);
output.AppendLine($"#EXT-X-PROGRAM-DATE-TIME:{currentTime:yyyy-MM-ddTHH:mm:ss.fff}{offset}");
output.AppendLine(lines[i + 2]);
currentTime += duration;
segments++;
i += 3;
continue;
}
nextPlaylistStart = currentTime < nextPlaylistStart ? currentTime : nextPlaylistStart;
if (!started)
var playlist = output.ToString();
if (endWithDiscontinuity && !playlist.EndsWith($"#EXT-X-DISCONTINUITY{Environment.NewLine}"))
{
startSequence = int.Parse(lines[i + 2].Replace("live", string.Empty).Split('.')[0]);
output.AppendLine("#EXTM3U");
output.AppendLine("#EXT-X-VERSION:6");
output.AppendLine("#EXT-X-TARGETDURATION:4");
output.AppendLine($"#EXT-X-MEDIA-SEQUENCE:{startSequence}");
output.AppendLine($"#EXT-X-DISCONTINUITY-SEQUENCE:{discontinuitySequence}");
output.AppendLine("#EXT-X-INDEPENDENT-SEGMENTS");
output.AppendLine("#EXT-X-DISCONTINUITY");
started = true;
playlist += "#EXT-X-DISCONTINUITY" + Environment.NewLine;
}
output.AppendLine(lines[i]);
string offset = currentTime.ToString("zzz").Replace(":", string.Empty);
output.AppendLine($"#EXT-X-PROGRAM-DATE-TIME:{currentTime:yyyy-MM-ddTHH:mm:ss.fff}{offset}");
output.AppendLine(lines[i + 2]);
currentTime += duration;
segments++;
i += 3;
return new TrimPlaylistResult(nextPlaylistStart, startSequence, playlist, segments);
}
var playlist = output.ToString();
if (endWithDiscontinuity && !playlist.EndsWith($"#EXT-X-DISCONTINUITY{Environment.NewLine}"))
catch (Exception ex)
{
playlist += "#EXT-X-DISCONTINUITY" + Environment.NewLine;
}
try
{
string file = _tempFilePool.GetNextTempFile(TempFileCategory.BadPlaylist);
File.WriteAllLines(file, lines);
return new TrimPlaylistResult(nextPlaylistStart, startSequence, playlist, segments);
_logger.LogError(ex, "Error filtering playlist. Bad playlist saved to {BadPlaylistFile}", file);
// TODO: better error result?
return new TrimPlaylistResult(playlistStart, 0, string.Empty, 0);
}
catch
{
// do nothing
}
throw;
}
}
public static TrimPlaylistResult TrimPlaylistWithDiscontinuity(
public TrimPlaylistResult TrimPlaylistWithDiscontinuity(
DateTimeOffset playlistStart,
DateTimeOffset filterBefore,
string[] lines)
ErsatzTV.Core/FFmpeg/IHlsPlaylistFilter.cs
+18
View File
@@ -0,0 +1,18 @@
using System;
namespace ErsatzTV.Core.FFmpeg;
public interface IHlsPlaylistFilter
{
TrimPlaylistResult TrimPlaylist(
DateTimeOffset playlistStart,
DateTimeOffset filterBefore,
string[] lines,
int maxSegments = 10,
bool endWithDiscontinuity = false);
TrimPlaylistResult TrimPlaylistWithDiscontinuity(
DateTimeOffset playlistStart,
DateTimeOffset filterBefore,
string[] lines);
}
ErsatzTV.Core/FFmpeg/TempFileCategory.cs
+3 -1
View File
@@ -5,6 +5,8 @@
Subtitle = 0,
SongBackground = 1,
CoverArt = 2,
CachedArtwork = 3
CachedArtwork = 3,
BadPlaylist = 99
}
}
ErsatzTV/Controllers/IptvController.cs
+5 -2
View File
@@ -26,17 +26,20 @@ namespace ErsatzTV.Controllers
public class IptvController : ControllerBase
{
private readonly IFFmpegSegmenterService _ffmpegSegmenterService;
private readonly IHlsPlaylistFilter _hlsPlaylistFilter;
private readonly ILogger<IptvController> _logger;
private readonly IMediator _mediator;
public IptvController(
IMediator mediator,
ILogger<IptvController> logger,
IFFmpegSegmenterService ffmpegSegmenterService)
IFFmpegSegmenterService ffmpegSegmenterService,
IHlsPlaylistFilter hlsPlaylistFilter)
{
_mediator = mediator;
_logger = logger;
_ffmpegSegmenterService = ffmpegSegmenterService;
_hlsPlaylistFilter = hlsPlaylistFilter;
}
[HttpGet("iptv/channels.m3u")]
@@ -97,7 +100,7 @@ namespace ErsatzTV.Controllers
// worker.PlaylistStart,
// now);
TrimPlaylistResult result = HlsPlaylistFilter.TrimPlaylist(worker.PlaylistStart, now, input);
TrimPlaylistResult result = _hlsPlaylistFilter.TrimPlaylist(worker.PlaylistStart, now, input);
return Content(result.Playlist, "application/vnd.apple.mpegurl");
}
}
ErsatzTV/Startup.cs
+1
View File
@@ -254,6 +254,7 @@ namespace ErsatzTV
services.AddSingleton<ISearchIndex, SearchIndex>();
services.AddSingleton<IFFmpegSegmenterService, FFmpegSegmenterService>();
services.AddSingleton<ITempFilePool, TempFilePool>();
services.AddSingleton<IHlsPlaylistFilter, HlsPlaylistFilter>();
AddChannel<IBackgroundServiceRequest>(services);
AddChannel<IPlexBackgroundServiceRequest>(services);
AddChannel<IJellyfinBackgroundServiceRequest>(services);