Tuning a channel intermittently hard-failed with ffmpeg exit 8 and
`Server returned 404 Not Found` on /media/jellyfin/{itemId}.
Root cause: ValidatePlayoutItemPath checked `File.Exists` on the local
branch, but the three media-server remote-stream branches returned
`http://localhost:{port}/media/{plex,jellyfin,emby}/{id}` unconditionally.
When the media was gone from the media server too, validation "succeeded"
and ffmpeg was launched against a URL that 404s.
That bypassed the good error path the handler already had
(PlayoutItemDoesNotExistOnDisk renders an error card sized to run until
the NEXT playout item, so the dead item is skipped) and instead landed in
HlsSessionWorker's generic ffmpeg-failure path, which sizes its error card
to the failed 44s work-ahead chunk and then re-selects the SAME broken
item -- a repeating error card for the item's whole slot (~22 min).
Restore the method's own invariant: every PlayoutItemWithPath it returns
has been checked for existence. A definitive 404 now returns the new
PlayoutItemNotAvailableFromMediaServer error, handled in the same switch
arm as PlayoutItemDoesNotExistOnDisk.
The probe is deliberately fail-open: only a 404 reports the media gone.
A timeout, 5xx, auth error or transport failure reports available, so a
probe that cannot answer can never break a tune that would have worked.
That contract is pinned by tests so a later refactor cannot invert it.
Rejected alternatives (see docs/decisions.md): resizing the
HlsSessionWorker retry loop (cannot distinguish a dead item from a
transient transcoder failure -- prod has live VAAPI hwupload -22 failures
that must keep retrying), and writing MediaItemState from the streaming
path (breaks scanner ownership, and would not have fixed this: the item
is RemoteOnly, which PlayoutBuilder's skip does not exclude).
Scanner-side follow-ups filed separately: #476 (FileNotFound does not
cascade show -> episodes, the reason dead items keep being scheduled),
fixes #473
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
118 lines
4.0 KiB
C#
118 lines
4.0 KiB
C#
using System.Net;
|
|
using ErsatzTV.Infrastructure.Streaming;
|
|
using Microsoft.Extensions.Logging;
|
|
using NSubstitute;
|
|
using NUnit.Framework;
|
|
using Shouldly;
|
|
|
|
namespace ErsatzTV.Infrastructure.Tests.Streaming;
|
|
|
|
[TestFixture]
|
|
public class HttpRemoteStreamProberTests
|
|
{
|
|
private const string Url = "http://localhost:8409/media/jellyfin/abc123";
|
|
|
|
[Test]
|
|
public async Task Should_Report_Unavailable_On_404()
|
|
{
|
|
HttpRemoteStreamProber prober = ProberReturning(HttpStatusCode.NotFound);
|
|
|
|
bool result = await prober.IsAvailable(Url, CancellationToken.None);
|
|
|
|
result.ShouldBeFalse();
|
|
}
|
|
|
|
[TestCase(HttpStatusCode.OK)]
|
|
[TestCase(HttpStatusCode.PartialContent)]
|
|
[TestCase(HttpStatusCode.NoContent)]
|
|
public async Task Should_Report_Available_On_Success(HttpStatusCode statusCode)
|
|
{
|
|
HttpRemoteStreamProber prober = ProberReturning(statusCode);
|
|
|
|
bool result = await prober.IsAvailable(Url, CancellationToken.None);
|
|
|
|
result.ShouldBeTrue();
|
|
}
|
|
|
|
// the fail-open contract: a probe that cannot answer must never block a tune that would
|
|
// otherwise have worked. these cases exist so a future refactor can't silently invert it.
|
|
[TestCase(HttpStatusCode.InternalServerError)]
|
|
[TestCase(HttpStatusCode.BadGateway)]
|
|
[TestCase(HttpStatusCode.Unauthorized)]
|
|
[TestCase(HttpStatusCode.Forbidden)]
|
|
public async Task Should_Fail_Open_On_Other_Status_Codes(HttpStatusCode statusCode)
|
|
{
|
|
HttpRemoteStreamProber prober = ProberReturning(statusCode);
|
|
|
|
bool result = await prober.IsAvailable(Url, CancellationToken.None);
|
|
|
|
result.ShouldBeTrue();
|
|
}
|
|
|
|
[Test]
|
|
public async Task Should_Fail_Open_On_Transport_Failure()
|
|
{
|
|
var prober = new HttpRemoteStreamProber(
|
|
new StubHttpClientFactory(new ThrowingHttpMessageHandler(new HttpRequestException("no route to host"))),
|
|
Substitute.For<ILogger<HttpRemoteStreamProber>>());
|
|
|
|
bool result = await prober.IsAvailable(Url, CancellationToken.None);
|
|
|
|
result.ShouldBeTrue();
|
|
}
|
|
|
|
[Test]
|
|
public async Task Should_Fail_Open_On_Timeout()
|
|
{
|
|
var prober = new HttpRemoteStreamProber(
|
|
new StubHttpClientFactory(new ThrowingHttpMessageHandler(new TaskCanceledException("timed out"))),
|
|
Substitute.For<ILogger<HttpRemoteStreamProber>>());
|
|
|
|
bool result = await prober.IsAvailable(Url, CancellationToken.None);
|
|
|
|
result.ShouldBeTrue();
|
|
}
|
|
|
|
[Test]
|
|
public async Task Should_Fail_Open_When_Caller_Cancels()
|
|
{
|
|
HttpRemoteStreamProber prober = ProberReturning(HttpStatusCode.OK);
|
|
|
|
using var cts = new CancellationTokenSource();
|
|
await cts.CancelAsync();
|
|
|
|
bool result = await prober.IsAvailable(Url, cts.Token);
|
|
|
|
result.ShouldBeTrue();
|
|
}
|
|
|
|
private static HttpRemoteStreamProber ProberReturning(HttpStatusCode statusCode) =>
|
|
new(
|
|
new StubHttpClientFactory(new StatusCodeHttpMessageHandler(statusCode)),
|
|
Substitute.For<ILogger<HttpRemoteStreamProber>>());
|
|
|
|
private sealed class StubHttpClientFactory(HttpMessageHandler handler) : IHttpClientFactory
|
|
{
|
|
public HttpClient CreateClient(string name) => new(handler, disposeHandler: false);
|
|
}
|
|
|
|
private sealed class StatusCodeHttpMessageHandler(HttpStatusCode statusCode) : HttpMessageHandler
|
|
{
|
|
protected override Task<HttpResponseMessage> SendAsync(
|
|
HttpRequestMessage request,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
return Task.FromResult(new HttpResponseMessage(statusCode));
|
|
}
|
|
}
|
|
|
|
private sealed class ThrowingHttpMessageHandler(Exception exception) : HttpMessageHandler
|
|
{
|
|
protected override Task<HttpResponseMessage> SendAsync(
|
|
HttpRequestMessage request,
|
|
CancellationToken cancellationToken) =>
|
|
Task.FromException<HttpResponseMessage>(exception);
|
|
}
|
|
}
|