ersatztv/ErsatzTV/Controllers/Api/TroubleshootController.cs

using System.ComponentModel.DataAnnotations;
using System.IO.Abstractions;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading.Channels;
using ErsatzTV.Application;
using ErsatzTV.Application.MediaItems;
using ErsatzTV.Application.Troubleshooting;
using ErsatzTV.Application.Troubleshooting.Queries;
using ErsatzTV.Controllers.Api.Requests;
using ErsatzTV.Core;
using ErsatzTV.Core.Api.Troubleshooting;
using ErsatzTV.Application.Channels;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Interfaces.FFmpeg;
using ErsatzTV.Core.Interfaces.Locking;
using ErsatzTV.Core.Interfaces.Repositories;
using ErsatzTV.Core.Interfaces.Troubleshooting;
using ErsatzTV.Core.Troubleshooting;
using ErsatzTV.Extensions;
using MediatR;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Serilog.Context;

namespace ErsatzTV.Controllers.Api;

[ApiController]
public class TroubleshootController(
    ChannelWriter<IFFmpegWorkerRequest> channelWriter,
    IFileSystem fileSystem,
    IConfigElementRepository configElementRepository,
    ITroubleshootingNotifier notifier,
    IEntityLocker entityLocker,
    ITroubleshootingPlaybackStatusStore statusStore,
    IMediator mediator) : ControllerBase
{
    private static readonly JsonSerializerOptions GeneralJsonOptions = new()
    {
        Converters = { new JsonStringEnumConverter() },
        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
        WriteIndented = true
    };

    [HttpGet("api/troubleshoot/info", Name = "GetTroubleshootingInfo")]
    [Tags("Troubleshooting")]
    [EndpointSummary("Get troubleshooting diagnostic info")]
    [EndpointGroupName("general")]
    [ProducesResponseType(typeof(TroubleshootingInfoResponseModel), StatusCodes.Status200OK)]
    public async Task<TroubleshootingInfoResponseModel> GetInfo(CancellationToken cancellationToken)
    {
        TroubleshootingInfo info = await mediator.Send(new GetTroubleshootingInfo(), cancellationToken);

        // mirrors the "General" tab JSON blob built by Pages/Troubleshooting/Troubleshooting.razor
        string generalJson = JsonSerializer.Serialize(
            new
            {
                info.Version,
                Environment = info.Environment.OrderBy(x => x.Key).ToDictionary(x => x.Key, x => x.Value),
                info.Cpus,
                info.VideoControllers,
                info.Health,
                info.FFmpegSettings,
                AviSynth = new
                {
                    Demuxer = info.AviSynthDemuxer,
                    Installed = info.AviSynthInstalled
                },
                info.Channels,
                info.FFmpegProfiles
            },
            GeneralJsonOptions);

        return new TroubleshootingInfoResponseModel(
            generalJson,
            info.NvidiaCapabilities,
            info.QsvCapabilities,
            info.VaapiCapabilities,
            info.VideoToolboxCapabilities);
    }

    [HttpPost("api/troubleshoot/validate-schedule", Name = "ValidateSequentialSchedule")]
    [Tags("Troubleshooting")]
    [EndpointSummary("Validate a sequential schedule YAML document")]
    [EndpointDescription(
        "Validates a sequential-schedule YAML string against the full (or import) schema. Returns whether it is " +
        "valid, any validation messages, and the JSON conversion of the YAML. Parse/validator errors are reported " +
        "as messages (IsValid=false), never as a 500.")]
    [EndpointGroupName("general")]
    [ProducesResponseType(typeof(ValidateSequentialScheduleResponseModel), StatusCodes.Status200OK)]
    [ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status400BadRequest)]
    public async Task<IActionResult> ValidateSchedule(
        [Required] [FromBody] ValidateSequentialScheduleRequest request,
        CancellationToken cancellationToken)
    {
        if (string.IsNullOrWhiteSpace(request.Yaml))
        {
            return new BadRequestObjectResult(
                new ProblemDetails
                {
                    Status = StatusCodes.Status400BadRequest,
                    Title = "Validation failed",
                    Detail = "[Yaml] must not be empty"
                });
        }

        ValidateSequentialScheduleViewModel result = await mediator.Send(request.ToQuery(), cancellationToken);
        return new OkObjectResult(
            new ValidateSequentialScheduleResponseModel(result.IsValid, result.Messages, result.Json));
    }

    [HttpHead("api/troubleshoot/playback.m3u8")]
    [HttpGet("api/troubleshoot/playback.m3u8")]
    [Tags("Troubleshooting")]
    [EndpointSummary("Start a troubleshooting playback session")]
    [EndpointGroupName("general")]
    [ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status409Conflict)]
    public async Task<IActionResult> TroubleshootPlayback(
        [FromQuery]
        int mediaItem,
        [FromQuery]
        int channel,
        [FromQuery]
        int ffmpegProfile,
        [FromQuery]
        StreamingMode streamingMode,
        [FromQuery]
        List<int> watermark,
        [FromQuery]
        List<int> graphicsElement,
        [FromQuery]
        string streamSelector,
        [FromQuery]
        int? subtitleId,
        [FromQuery]
        int seekSeconds,
        [FromQuery]
        DateTimeOffset? start,
        CancellationToken cancellationToken)
    {
        var sessionId = Guid.NewGuid();
        using var logContext = LogContext.PushProperty(InMemoryLogService.CorrelationIdKey, sessionId);

        // acquiredLock: the handler took the troubleshooting lock on a successful Prepare.
        // startEnqueued: we handed off to StartTroubleshootingPlayback, which becomes the lock's releaser.
        var acquiredLock = false;
        var startEnqueued = false;

        try
        {
            // fast lock-conflict signal so the client gets a 409 (not a 404) while another
            // troubleshooting session is active; the atomic acquire in the handler is the real guard
            if (entityLocker.IsTroubleshootingPlaybackLocked())
            {
                return ApiResults.ConflictProblem(
                    "Troubleshooting playback in progress",
                    "Another troubleshooting playback session is currently running. Try again once it completes.");
            }

            Option<int> ss = seekSeconds > 0 ? seekSeconds : Option<int>.None;

            Either<BaseError, PlayoutItemResult> result = await mediator.Send(
                new PrepareTroubleshootingPlayback(
                    sessionId,
                    streamingMode,
                    mediaItem,
                    channel,
                    ffmpegProfile,
                    streamSelector,
                    watermark,
                    graphicsElement,
                    subtitleId,
                    ss,
                    Optional(start)),
                cancellationToken);

            if (result.IsLeft)
            {
                return NotFound();
            }

            // Prepare returned a process, so the handler holds the troubleshooting lock now
            acquiredLock = true;

            foreach (PlayoutItemResult playoutItemResult in result.RightToSeq())
            {
                Either<BaseError, MediaItemInfo> maybeMediaInfo =
                    await mediator.Send(
                        new GetMediaItemInfo(await playoutItemResult.MediaItemId.IfNoneAsync(0)),
                        cancellationToken);

                try
                {
                    TroubleshootingInfo troubleshootingInfo = await mediator.Send(
                        new GetTroubleshootingInfo(),
                        cancellationToken);

                    // filter ffmpeg profiles
                    troubleshootingInfo.FFmpegProfiles.RemoveAll(p => p.Id != ffmpegProfile);

                    // filter watermarks
                    troubleshootingInfo.Watermarks.RemoveAll(p => !watermark.Contains(p.Id));

                    await channelWriter.WriteAsync(
                        new StartTroubleshootingPlayback(
                            sessionId,
                            streamSelector,
                            playoutItemResult,
                            maybeMediaInfo.ToOption(),
                            troubleshootingInfo),
                        cancellationToken);

                    // StartTroubleshootingPlayback is now responsible for releasing the lock in its finally
                    startEnqueued = true;

                    string playlistFile = Path.Combine(FileSystemLayout.TranscodeTroubleshootingFolder, "live.m3u8");
                    while (!fileSystem.File.Exists(playlistFile))
                    {
                        await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);
                        if (cancellationToken.IsCancellationRequested || notifier.IsFailed(sessionId))
                        {
                            break;
                        }
                    }

                    int initialSegmentCount = await configElementRepository
                        .GetValue<int>(ConfigElementKey.FFmpegInitialSegmentCount, cancellationToken)
                        .Map(maybeCount => maybeCount.Match(c => c, () => 1));

                    initialSegmentCount = Math.Max(initialSegmentCount, 2);

                    bool hasSegments = false;
                    while (!hasSegments)
                    {
                        await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);

                        string[] segmentFiles = streamingMode switch
                        {
                            // StreamingMode.HttpLiveStreamingSegmenter => Directory.GetFiles(
                            //     FileSystemLayout.TranscodeTroubleshootingFolder,
                            //     "*.m4s"),
                            _ => Directory.GetFiles(FileSystemLayout.TranscodeTroubleshootingFolder, "*.ts")
                        };

                        if (segmentFiles.Length >= initialSegmentCount)
                        {
                            hasSegments = true;
                        }
                    }

                    if (!notifier.IsFailed(sessionId))
                    {
                        return Redirect("~/iptv/session/.troubleshooting/live.m3u8");
                    }
                }
                finally
                {
                    notifier.RemoveSession(sessionId);
                }
            }
        }
        catch (Exception)
        {
            // do nothing
        }
        finally
        {
            // the handler acquired the lock but we never handed off to the worker
            // (cancellation/exception in the window before enqueue) -> release it so it doesn't leak
            if (acquiredLock && !startEnqueued)
            {
                entityLocker.UnlockTroubleshootingPlayback();
            }
        }

        return NotFound();
    }

    [HttpHead("api/troubleshoot/playback/archive")]
    [HttpGet("api/troubleshoot/playback/archive")]
    [Tags("Troubleshooting")]
    [EndpointSummary("Download the last troubleshooting playback session archive")]
    [EndpointGroupName("general")]
    public async Task<IActionResult> TroubleshootPlaybackArchive(CancellationToken cancellationToken)
    {
        Option<string> maybeArchivePath = await mediator.Send(new ArchiveTroubleshootingResults(), cancellationToken);
        foreach (string archivePath in maybeArchivePath)
        {
            var fs = new FileStream(
                archivePath,
                FileMode.Open,
                FileAccess.Read,
                FileShare.Read,
                4096,
                FileOptions.DeleteOnClose);
            return File(
                fs,
                "application/zip",
                $"ersatztv-troubleshooting-{DateTimeOffset.Now.ToUnixTimeSeconds()}.zip");
        }

        return NotFound();
    }

    [HttpHead("api/troubleshoot/playback/sample/{mediaItemId:int}")]
    [HttpGet("api/troubleshoot/playback/sample/{mediaItemId:int}")]
    [Tags("Troubleshooting")]
    [EndpointSummary("Download a media sample archive for troubleshooting")]
    [EndpointGroupName("general")]
    public async Task<IActionResult> TroubleshootPlaybackSample(int mediaItemId, CancellationToken cancellationToken)
    {
        Option<string> maybeArchivePath = await mediator.Send(new ArchiveMediaSample(mediaItemId), cancellationToken);
        foreach (string archivePath in maybeArchivePath)
        {
            var fs = new FileStream(
                archivePath,
                FileMode.Open,
                FileAccess.Read,
                FileShare.Read,
                4096,
                FileOptions.DeleteOnClose);
            return File(
                fs,
                "application/zip",
                $"ersatztv-media-sample-{DateTimeOffset.Now.ToUnixTimeSeconds()}.zip");
        }

        return NotFound();
    }

    [HttpGet("api/troubleshoot/playback/stream-selectors", Name = "GetTroubleshootingStreamSelectors")]
    [Tags("Troubleshooting")]
    [EndpointSummary("List available channel stream selectors")]
    [EndpointGroupName("general")]
    [ProducesResponseType(typeof(List<string>), StatusCodes.Status200OK)]
    public async Task<List<string>> GetStreamSelectors(CancellationToken cancellationToken) =>
        await mediator.Send(new GetChannelStreamSelectors(), cancellationToken);

    [HttpGet("api/troubleshoot/playback/subtitles/{mediaItemId:int}", Name = "GetTroubleshootingSubtitles")]
    [Tags("Troubleshooting")]
    [EndpointSummary("List selectable subtitle streams for a media item")]
    [EndpointDescription(
        "Returns the subtitle streams that can be burned in for a troubleshooting playback. Each item's id is the " +
        "value to pass back as the playback.m3u8 endpoint's subtitleId query parameter.")]
    [EndpointGroupName("general")]
    [ProducesResponseType(typeof(List<TroubleshootingSubtitleResponseModel>), StatusCodes.Status200OK)]
    [ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status404NotFound)]
    public async Task<IActionResult> GetSubtitles(int mediaItemId, CancellationToken cancellationToken)
    {
        Either<BaseError, MediaItemInfo> maybeMediaItem =
            await mediator.Send(new GetMediaItemInfo(mediaItemId), cancellationToken);
        if (maybeMediaItem.IsLeft)
        {
            return ApiResults.NotFoundProblem();
        }

        List<SubtitleViewModel> subtitles =
            await mediator.Send(new GetTroubleshootingSubtitles(mediaItemId), cancellationToken);

        return new OkObjectResult(
            subtitles
                .Map(s => new TroubleshootingSubtitleResponseModel(s.Id, s.Language, s.Title, s.Codec))
                .ToList());
    }

    [HttpGet("api/troubleshoot/playback/status", Name = "GetTroubleshootingPlaybackStatus")]
    [Tags("Troubleshooting")]
    [EndpointSummary("Get the status of the current or last troubleshooting playback session")]
    [EndpointDescription(
        "Reports whether a troubleshooting playback is idle, running, completed, or failed, along with the last " +
        "session's ffmpeg exit code, playback speed, and a tail (last 500 lines) of its log output.")]
    [EndpointGroupName("general")]
    [ProducesResponseType(typeof(TroubleshootingPlaybackStatusResponseModel), StatusCodes.Status200OK)]
    public async Task<TroubleshootingPlaybackStatusResponseModel> GetPlaybackStatus(CancellationToken cancellationToken)
    {
        bool running = entityLocker.IsTroubleshootingPlaybackLocked();
        Option<TroubleshootingPlaybackResult> maybeResult = statusStore.CurrentResult;

        string state = "idle";
        int? exitCode = null;
        double? speed = null;

        if (running)
        {
            state = "running";
        }

        foreach (TroubleshootingPlaybackResult result in maybeResult)
        {
            exitCode = result.ExitCode;
            speed = result.Speed.MatchUnsafe(v => (double?)v, () => null);
            if (!running)
            {
                state = result.ExitCode == 0 ? "completed" : "failed";
            }
        }

        string logs = await ReadTroubleshootingLogTail(cancellationToken);

        return new TroubleshootingPlaybackStatusResponseModel(state, exitCode, speed, logs);
    }

    private async Task<string> ReadTroubleshootingLogTail(CancellationToken cancellationToken)
    {
        const int MaxLines = 500;
        string logFile = Path.Combine(FileSystemLayout.TranscodeTroubleshootingFolder, "logs.txt");
        if (!fileSystem.File.Exists(logFile))
        {
            return null;
        }

        string[] lines = await fileSystem.File.ReadAllLinesAsync(logFile, cancellationToken);
        if (lines.Length > MaxLines)
        {
            lines = lines[^MaxLines..];
        }

        return string.Join(Environment.NewLine, lines);
    }
}