Tuning a channel intermittently hard-failed with ffmpeg exit 8 and
`Server returned 404 Not Found` on /media/jellyfin/{itemId}.
Root cause: ValidatePlayoutItemPath checked `File.Exists` on the local
branch, but the three media-server remote-stream branches returned
`http://localhost:{port}/media/{plex,jellyfin,emby}/{id}` unconditionally.
When the media was gone from the media server too, validation "succeeded"
and ffmpeg was launched against a URL that 404s.
That bypassed the good error path the handler already had
(PlayoutItemDoesNotExistOnDisk renders an error card sized to run until
the NEXT playout item, so the dead item is skipped) and instead landed in
HlsSessionWorker's generic ffmpeg-failure path, which sizes its error card
to the failed 44s work-ahead chunk and then re-selects the SAME broken
item -- a repeating error card for the item's whole slot (~22 min).
Restore the method's own invariant: every PlayoutItemWithPath it returns
has been checked for existence. A definitive 404 now returns the new
PlayoutItemNotAvailableFromMediaServer error, handled in the same switch
arm as PlayoutItemDoesNotExistOnDisk.
The probe is deliberately fail-open: only a 404 reports the media gone.
A timeout, 5xx, auth error or transport failure reports available, so a
probe that cannot answer can never break a tune that would have worked.
That contract is pinned by tests so a later refactor cannot invert it.
Rejected alternatives (see docs/decisions.md): resizing the
HlsSessionWorker retry loop (cannot distinguish a dead item from a
transient transcoder failure -- prod has live VAAPI hwupload -22 failures
that must keep retrying), and writing MediaItemState from the streaming
path (breaks scanner ownership, and would not have fixed this: the item
is RemoteOnly, which PlayoutBuilder's skip does not exclude).
Scanner-side follow-ups filed separately: #476 (FileNotFound does not
cascade show -> episodes, the reason dead items keep being scheduled),
fixes #473
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
245 lines
9.0 KiB
C#
245 lines
9.0 KiB
C#
using System.IO.Abstractions;
|
|
using CliWrap;
|
|
using ErsatzTV.Application.Streaming;
|
|
using ErsatzTV.Core;
|
|
using ErsatzTV.Core.Domain;
|
|
using ErsatzTV.Core.Interfaces.Emby;
|
|
using ErsatzTV.Core.Interfaces.FFmpeg;
|
|
using ErsatzTV.Core.Interfaces.Jellyfin;
|
|
using ErsatzTV.Core.Interfaces.Plex;
|
|
using ErsatzTV.Core.Interfaces.Repositories;
|
|
using ErsatzTV.Core.Interfaces.Streaming;
|
|
using ErsatzTV.FFmpeg;
|
|
using ErsatzTV.Infrastructure.Data;
|
|
using ErsatzTV.Tests.Support;
|
|
using LanguageExt;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using NSubstitute;
|
|
using NSubstitute.Core;
|
|
using NSubstitute.Extensions;
|
|
using NUnit.Framework;
|
|
using Shouldly;
|
|
using DomainChannel = ErsatzTV.Core.Domain.Channel;
|
|
|
|
namespace ErsatzTV.Tests.Application.Streaming;
|
|
|
|
/// <summary>
|
|
/// ersatztv#473: a media-server item that is missing from disk used to fall back to the
|
|
/// remote-stream URL unconditionally. It must now be probed first, and an unavailable
|
|
/// remote stream must render an error card instead of a real playout process.
|
|
/// </summary>
|
|
[TestFixture]
|
|
public class GetPlayoutItemProcessByChannelNumberHandlerTests
|
|
{
|
|
private const string ChannelNumber = "1";
|
|
private const string JellyfinItemId = "abc123";
|
|
|
|
private readonly List<string> _tempFiles = [];
|
|
private InMemoryTvContext _db = null!;
|
|
private IFFmpegProcessService _ffmpegProcessService = null!;
|
|
private IRemoteStreamProber _remoteStreamProber = null!;
|
|
|
|
[SetUp]
|
|
public async Task SetUp()
|
|
{
|
|
_db = await InMemoryTvContext.CreateAsync();
|
|
_remoteStreamProber = Substitute.For<IRemoteStreamProber>();
|
|
|
|
_ffmpegProcessService = Substitute.For<IFFmpegProcessService>();
|
|
|
|
// both ForError and ForPlayoutItem must hand back a real value; the handler dereferences them
|
|
_ffmpegProcessService.ReturnsForAll(Task.FromResult(Cli.Wrap("ffmpeg")));
|
|
_ffmpegProcessService.ReturnsForAll(
|
|
Task.FromResult(
|
|
new PlayoutItemResult(
|
|
Cli.Wrap("ffmpeg"),
|
|
Option<GraphicsEngineContext>.None,
|
|
Option<int>.None)));
|
|
}
|
|
|
|
[TearDown]
|
|
public async Task TearDown()
|
|
{
|
|
await _db.DisposeAsync();
|
|
foreach (string tempFile in _tempFiles)
|
|
{
|
|
if (File.Exists(tempFile))
|
|
{
|
|
File.Delete(tempFile);
|
|
}
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task Should_Render_Error_Card_When_Remote_Stream_Is_Unavailable()
|
|
{
|
|
DateTimeOffset now = await SeedAll();
|
|
_remoteStreamProber.IsAvailable(Arg.Any<string>(), Arg.Any<CancellationToken>()).Returns(false);
|
|
|
|
Either<BaseError, PlayoutItemProcessModel> result =
|
|
await CreateHandler().Handle(Request(now), CancellationToken.None);
|
|
|
|
result.IsRight.ShouldBeTrue();
|
|
|
|
await _remoteStreamProber.Received(1)
|
|
.IsAvailable($"http://localhost:{Settings.StreamingPort}/media/jellyfin/{JellyfinItemId}", Arg.Any<CancellationToken>());
|
|
|
|
CallsTo(nameof(IFFmpegProcessService.ForError)).Count.ShouldBe(1);
|
|
CallsTo(nameof(IFFmpegProcessService.ForPlayoutItem)).Count.ShouldBe(0);
|
|
}
|
|
|
|
[Test]
|
|
public async Task Should_Stream_Remotely_When_Remote_Stream_Is_Available()
|
|
{
|
|
DateTimeOffset now = await SeedAll();
|
|
_remoteStreamProber.IsAvailable(Arg.Any<string>(), Arg.Any<CancellationToken>()).Returns(true);
|
|
|
|
Either<BaseError, PlayoutItemProcessModel> result =
|
|
await CreateHandler().Handle(Request(now), CancellationToken.None);
|
|
|
|
result.IsRight.ShouldBeTrue();
|
|
|
|
CallsTo(nameof(IFFmpegProcessService.ForError)).Count.ShouldBe(0);
|
|
|
|
List<ICall> playoutCalls = CallsTo(nameof(IFFmpegProcessService.ForPlayoutItem));
|
|
playoutCalls.Count.ShouldBe(1);
|
|
|
|
// videoPath is the 7th parameter of ForPlayoutItem
|
|
object?[] arguments = playoutCalls[0].GetArguments();
|
|
arguments[6].ShouldBe($"http://localhost:{Settings.StreamingPort}/media/jellyfin/{JellyfinItemId}");
|
|
}
|
|
|
|
private List<ICall> CallsTo(string methodName) =>
|
|
_ffmpegProcessService.ReceivedCalls()
|
|
.Where(c => c.GetMethodInfo().Name == methodName)
|
|
.ToList();
|
|
|
|
private static GetPlayoutItemProcessByChannelNumber Request(DateTimeOffset now) =>
|
|
new(
|
|
ChannelNumber,
|
|
StreamingMode.TransportStream,
|
|
now,
|
|
StartAtZero: false,
|
|
HlsRealtime: true,
|
|
ChannelStart: now,
|
|
PtsOffset: TimeSpan.Zero,
|
|
TargetFramerate: Option<FrameRate>.None,
|
|
IsTroubleshooting: false,
|
|
FFmpegProfileId: Option<int>.None);
|
|
|
|
private GetPlayoutItemProcessByChannelNumberHandler CreateHandler()
|
|
{
|
|
var fileSystem = Substitute.For<IFileSystem>();
|
|
fileSystem.File.Exists(Arg.Any<string>()).Returns(false);
|
|
|
|
return new GetPlayoutItemProcessByChannelNumberHandler(
|
|
_db.Factory,
|
|
_ffmpegProcessService,
|
|
fileSystem,
|
|
Substitute.For<IExternalJsonPlayoutItemProvider>(),
|
|
Substitute.For<IPlexPathReplacementService>(),
|
|
Substitute.For<IJellyfinPathReplacementService>(),
|
|
Substitute.For<IEmbyPathReplacementService>(),
|
|
Substitute.For<IMediaCollectionRepository>(),
|
|
Substitute.For<ITelevisionRepository>(),
|
|
Substitute.For<IArtistRepository>(),
|
|
Substitute.For<ISongVideoGenerator>(),
|
|
Substitute.For<IMusicVideoCreditsGenerator>(),
|
|
Substitute.For<IWatermarkSelector>(),
|
|
Substitute.For<IGraphicsElementSelector>(),
|
|
Substitute.For<IDecoSelector>(),
|
|
_remoteStreamProber,
|
|
NullLogger<GetPlayoutItemProcessByChannelNumberHandler>.Instance);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Seeds ffmpeg/ffprobe config, an ffmpeg profile, a channel, a playout and a playout item
|
|
/// covering "now" whose media item is a jellyfin episode missing from disk.
|
|
/// </summary>
|
|
/// <returns>The "now" the request should use.</returns>
|
|
private async Task<DateTimeOffset> SeedAll()
|
|
{
|
|
string ffmpeg = Path.GetTempFileName();
|
|
string ffprobe = Path.GetTempFileName();
|
|
_tempFiles.Add(ffmpeg);
|
|
_tempFiles.Add(ffprobe);
|
|
|
|
var now = new DateTimeOffset(2026, 1, 1, 12, 0, 0, TimeSpan.Zero);
|
|
|
|
await using TvContext context = _db.CreateContext();
|
|
|
|
context.ConfigElements.Add(new ConfigElement { Key = ConfigElementKey.FFmpegPath.Key, Value = ffmpeg });
|
|
context.ConfigElements.Add(new ConfigElement { Key = ConfigElementKey.FFprobePath.Key, Value = ffprobe });
|
|
|
|
var profile = new FFmpegProfile
|
|
{
|
|
Name = "Test",
|
|
Resolution = new Resolution { Name = "1080p", Width = 1920, Height = 1080 }
|
|
};
|
|
context.FFmpegProfiles.Add(profile);
|
|
await context.SaveChangesAsync();
|
|
|
|
var channel = new DomainChannel(Guid.NewGuid())
|
|
{
|
|
Number = ChannelNumber,
|
|
Name = "Test",
|
|
Group = "ErsatzTV",
|
|
Categories = string.Empty,
|
|
FFmpegProfileId = profile.Id,
|
|
StreamSelector = string.Empty,
|
|
PreferredAudioLanguageCode = string.Empty,
|
|
PreferredAudioTitle = string.Empty,
|
|
PreferredSubtitleLanguageCode = string.Empty,
|
|
MusicVideoCreditsTemplate = string.Empty,
|
|
StreamingMode = StreamingMode.TransportStream,
|
|
PlayoutSource = ChannelPlayoutSource.Generated,
|
|
PlayoutMode = ChannelPlayoutMode.Continuous
|
|
};
|
|
context.Channels.Add(channel);
|
|
|
|
var episode = new JellyfinEpisode
|
|
{
|
|
ItemId = JellyfinItemId,
|
|
EpisodeMetadata = [new EpisodeMetadata { Title = "Missing", SortTitle = "Missing", Subtitles = [] }],
|
|
MediaVersions =
|
|
[
|
|
new MediaVersion
|
|
{
|
|
Name = "Main",
|
|
Duration = TimeSpan.FromMinutes(30),
|
|
MediaFiles = [new MediaFile { Path = "/gone/episode.mkv", PathHash = "gone" }],
|
|
Streams = []
|
|
}
|
|
]
|
|
};
|
|
context.AddRange(episode);
|
|
await context.SaveChangesAsync();
|
|
|
|
var playout = new Playout
|
|
{
|
|
ChannelId = channel.Id,
|
|
ScheduleKind = PlayoutScheduleKind.Classic
|
|
};
|
|
context.Playouts.Add(playout);
|
|
await context.SaveChangesAsync();
|
|
|
|
context.PlayoutItems.Add(
|
|
new PlayoutItem
|
|
{
|
|
PlayoutId = playout.Id,
|
|
MediaItemId = episode.Id,
|
|
Start = now.AddMinutes(-5).UtcDateTime,
|
|
Finish = now.AddMinutes(25).UtcDateTime,
|
|
InPoint = TimeSpan.FromMinutes(5),
|
|
OutPoint = TimeSpan.FromMinutes(30),
|
|
Watermarks = [],
|
|
PlayoutItemWatermarks = [],
|
|
GraphicsElements = [],
|
|
PlayoutItemGraphicsElements = []
|
|
});
|
|
await context.SaveChangesAsync();
|
|
|
|
return now;
|
|
}
|
|
}
|