ersatztv/ErsatzTV.Scanner/Core/Metadata/MediaServerMusicVideoLibraryScanner.cs

using System.Collections.Immutable;
using System.IO.Abstractions;
using ErsatzTV.Core;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Domain.MediaServer;
using ErsatzTV.Core.Errors;
using ErsatzTV.Core.Extensions;
using ErsatzTV.Core.Interfaces.Repositories;
using ErsatzTV.Core.Metadata;
using ErsatzTV.Scanner.Core.Interfaces;
using Microsoft.Extensions.Logging;

namespace ErsatzTV.Scanner.Core.Metadata;

// ersatztv#496: music videos used to be reconciled by a LIBRARY-SCOPED local-path diff plus hard delete, because
// they carried no server identity. They now carry one (JellyfinMusicVideo.ItemId/Etag), so this scanner is the
// music-video sibling of MediaServer{Movie,Television,OtherVideo}LibraryScanner: it diffs on the server item id
// and soft-trashes (FileNotFound) instead of deleting, which is reversible and EmptyTrash-governed.
public abstract class MediaServerMusicVideoLibraryScanner<TConnectionParameters, TLibrary, TMusicVideo, TEtag>
    where TConnectionParameters : MediaServerConnectionParameters
    where TLibrary : Library
    where TMusicVideo : MusicVideo
    where TEtag : MediaServerItemEtag
{
    private const string UnknownArtist = "Unknown Artist";

    private readonly IArtistRepository _artistRepository;
    private readonly IFileSystem _fileSystem;
    private readonly ILibraryRepository _libraryRepository;
    private readonly ILogger _logger;
    private readonly IMetadataRepository _metadataRepository;
    private readonly IScannerProxy _scannerProxy;

    protected MediaServerMusicVideoLibraryScanner(
        IScannerProxy scannerProxy,
        IFileSystem fileSystem,
        IArtistRepository artistRepository,
        ILibraryRepository libraryRepository,
        IMetadataRepository metadataRepository,
        ILogger logger)
    {
        _scannerProxy = scannerProxy;
        _fileSystem = fileSystem;
        _artistRepository = artistRepository;
        _libraryRepository = libraryRepository;
        _metadataRepository = metadataRepository;
        _logger = logger;
    }

    protected async Task<Either<BaseError, Unit>> ScanLibrary(
        IMediaServerMusicVideoRepository<TLibrary, TMusicVideo, TEtag> musicVideoRepository,
        TConnectionParameters connectionParameters,
        TLibrary library,
        LibraryPath libraryPath,
        Func<TMusicVideo, string> getLocalPath,
        bool deepScan,
        CancellationToken cancellationToken)
    {
        try
        {
            // ersatztv#484: one counter per enumeration, created here and read only after the enumeration
            // completes. It is never a field on the (singleton) api client, so concurrent scans of different
            // libraries cannot leak failures into each other's sweep decision.
            var projectionFailures = new MediaServerProjectionFailureCounter();

            return await ScanLibrary(
                musicVideoRepository,
                connectionParameters,
                library,
                libraryPath,
                getLocalPath,
                GetMusicVideoLibraryItems(connectionParameters, library, projectionFailures),
                projectionFailures,
                deepScan,
                cancellationToken);
        }
        catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException)
        {
            return new ScanCanceled();
        }
    }

    private async Task<Either<BaseError, Unit>> ScanLibrary(
        IMediaServerMusicVideoRepository<TLibrary, TMusicVideo, TEtag> musicVideoRepository,
        TConnectionParameters connectionParameters,
        TLibrary library,
        LibraryPath libraryPath,
        Func<TMusicVideo, string> getLocalPath,
        IAsyncEnumerable<Tuple<TMusicVideo, int>> musicVideoEntries,
        MediaServerProjectionFailureCounter projectionFailures,
        bool deepScan,
        CancellationToken cancellationToken)
    {
        var incomingItemIds = new List<string>();
        var incomingLocalPaths = new List<string>();
        var existingMusicVideos = (await musicVideoRepository.GetExistingMusicVideos(library))
            .ToImmutableDictionary(e => e.MediaServerItemId, e => e);

        await foreach ((TMusicVideo incoming, int totalMusicVideoCount) in musicVideoEntries.WithCancellation(
                           cancellationToken))
        {
            if (cancellationToken.IsCancellationRequested)
            {
                return new ScanCanceled();
            }

            incomingItemIds.Add(MediaServerItemId(incoming));

            decimal percentCompletion = totalMusicVideoCount == 0
                ? 1
                : Math.Clamp((decimal)incomingItemIds.Count / totalMusicVideoCount, 0, 1);
            if (!await _scannerProxy.UpdateProgress(percentCompletion, cancellationToken))
            {
                return new ScanCanceled();
            }

            string localPath = getLocalPath(incoming);
            incomingLocalPaths.Add(localPath);

            if (!await ShouldScanItem(
                    musicVideoRepository,
                    library,
                    existingMusicVideos,
                    incoming,
                    localPath,
                    deepScan))
            {
                continue;
            }

            Either<BaseError, MediaItemScanResult<TMusicVideo>> maybeMusicVideo =
                await ProcessMusicVideo(
                    musicVideoRepository,
                    connectionParameters,
                    library,
                    libraryPath,
                    incoming,
                    localPath,
                    deepScan,
                    cancellationToken);

            if (maybeMusicVideo.IsLeft)
            {
                foreach (BaseError error in maybeMusicVideo.LeftToSeq())
                {
                    _logger.LogWarning("Error processing music video: {Error}", error.Value);
                }

                continue;
            }

            foreach (MediaItemScanResult<TMusicVideo> result in maybeMusicVideo.RightToSeq())
            {
                await musicVideoRepository.SetEtag(result.Item, MediaServerEtag(incoming));

                if (_fileSystem.File.Exists(result.LocalPath))
                {
                    Option<int> flagResult = await musicVideoRepository.FlagNormal(library, result.Item);
                    if (flagResult.IsSome)
                    {
                        result.IsUpdated = true;
                    }
                }
                else
                {
                    Option<int> flagResult = await musicVideoRepository.FlagUnavailable(library, result.Item);
                    if (flagResult.IsSome)
                    {
                        result.IsUpdated = true;
                    }
                }

                if (result.IsAdded || result.IsUpdated)
                {
                    if (!await _scannerProxy.ReindexMediaItems([result.Item.Id], cancellationToken))
                    {
                        _logger.LogWarning("Failed to reindex media items from scanner process");
                    }
                }
            }
        }

        await TrashMissingMusicVideos(
            musicVideoRepository,
            library,
            libraryPath,
            incomingItemIds,
            incomingLocalPaths,
            existingMusicVideos,
            projectionFailures,
            cancellationToken);

        return Unit.Default;
    }

    // Soft-trash music videos the media server no longer reports. Identity is the SERVER ITEM ID, scoped to this
    // library by GetExistingMusicVideos/FlagFileNotFound (both join LibraryPath.LibraryId), so a file served by two
    // libraries with overlapping local paths is two independent identities and one library's sweep can no longer
    // reach the other's row — the ersatztv#496 cross-library false-trash this replaced.
    private async Task TrashMissingMusicVideos(
        IMediaServerMusicVideoRepository<TLibrary, TMusicVideo, TEtag> musicVideoRepository,
        TLibrary library,
        LibraryPath libraryPath,
        List<string> incomingItemIds,
        List<string> incomingLocalPaths,
        ImmutableDictionary<string, TEtag> existingMusicVideos,
        MediaServerProjectionFailureCounter projectionFailures,
        CancellationToken cancellationToken)
    {
        // Legacy rows (no identity yet) are reconciled by local path. They are counted into the guard's existing
        // set below, because on the FIRST scan after the identity landed they are the entire library — a guard
        // that only saw the (empty) identity set would let a zero-item fetch sweep every one of them.
        List<string> existingLegacyPaths = await musicVideoRepository.GetExistingLegacyMusicVideoPaths(library);

        // ersatztv#477: refuse the sweep when a successful fetch returned zero items but rows exist locally — an
        // empty incoming set is indistinguishable from a transient error and would otherwise flag a whole library.
        // ersatztv#484: a swallowed projection failure is indistinguishable from a deletion here, so it must
        // suppress the sweep. This gates the LEGACY path diff too — that one is if anything more exposed, since
        // a legacy row has no etag to fall back on.
        if (!MediaServerReconciliationGuard.ShouldFlagMissing(
                _logger,
                library.Name,
                incomingItemIds.Count,
                existingMusicVideos.Count + existingLegacyPaths.Count,
                projectionFailures.Count))
        {
            return;
        }

        var fileNotFoundItemIds = existingMusicVideos.Keys.Except(incomingItemIds).ToList();
        List<int> ids = await musicVideoRepository.FlagFileNotFound(library, fileNotFoundItemIds);

        // A legacy row whose file IS still reported was adopted during this scan's loop, so it is no longer
        // legacy; what remains here is only what the server has genuinely stopped reporting.
        var fileNotFoundPaths = existingLegacyPaths.Except(incomingLocalPaths).ToList();
        ids = ids.Concat(await musicVideoRepository.FlagFileNotFoundByPaths(library, fileNotFoundPaths)).ToList();

        if (ids.Count > 0 && !await _scannerProxy.ReindexMediaItems(ids.ToArray(), cancellationToken))
        {
            _logger.LogWarning("Failed to reindex media items from scanner process");
        }

        // Trashed music videos are still rows, so an artist only becomes empty once the trash is emptied (or its
        // items are removed some other way); this keeps ersatztv#494's empty-artist cleanup without the delete.
        List<int> artistIds = await _artistRepository.DeleteEmptyArtists(libraryPath);
        if (artistIds.Count > 0 && !await _scannerProxy.RemoveMediaItems(artistIds.ToArray(), cancellationToken))
        {
            _logger.LogWarning("Failed to remove empty artists from scanner process");
        }
    }

    private async Task<Either<BaseError, MediaItemScanResult<TMusicVideo>>> ProcessMusicVideo(
        IMediaServerMusicVideoRepository<TLibrary, TMusicVideo, TEtag> musicVideoRepository,
        TConnectionParameters connectionParameters,
        TLibrary library,
        LibraryPath libraryPath,
        TMusicVideo incoming,
        string localPath,
        bool deepScan,
        CancellationToken cancellationToken)
    {
        // ersatztv#488: resolve the owning folder from the DB, never from libraryPath.LibraryFolders (that
        // navigation is only eager-loaded on the local scan path and is null for media-server callers).
        string folder = Path.GetDirectoryName(localPath) ?? libraryPath.Path;
        Option<int> maybeParentFolder =
            await _libraryRepository.GetParentFolderId(libraryPath, folder, cancellationToken);
        LibraryFolder libraryFolder = await _libraryRepository.GetOrAddFolder(libraryPath, maybeParentFolder, folder);

        return await GetOrAddArtist(libraryPath, incoming)
            .BindT(artist => musicVideoRepository.GetOrAdd(
                library,
                artist,
                libraryFolder,
                incoming,
                localPath,
                deepScan,
                cancellationToken))
            .MapT(result =>
            {
                result.LocalPath = localPath;
                return result;
            })
            .BindT(result => UpdateMetadata(result, incoming, cancellationToken))
            .BindT(result => UpdateStatistics(connectionParameters, library, result, incoming, deepScan));
    }

    private async Task<Either<BaseError, Artist>> GetOrAddArtist(LibraryPath libraryPath, TMusicVideo musicVideo)
    {
        string artistName = musicVideo.MusicVideoMetadata
            .HeadOrNone()
            .Bind(metadata => Optional(metadata.Artists).Flatten().HeadOrNone())
            .Map(artist => artist.Name)
            .IfNone(UnknownArtist);

        var metadata = new ArtistMetadata
        {
            MetadataKind = MetadataKind.External,
            Title = artistName,
            SortTitle = SortTitle.GetSortTitle(artistName),
            DateAdded = DateTime.UtcNow,
            Genres = [],
            Styles = [],
            Moods = [],
            Artwork = [],
            Guids = [],
            Subtitles = []
        };

        Option<Artist> maybeArtist = await _artistRepository.GetArtistByMetadata(libraryPath.Id, metadata);
        foreach (Artist artist in maybeArtist)
        {
            return artist;
        }

        Either<BaseError, MediaItemScanResult<Artist>> result =
            await _artistRepository.AddArtist(libraryPath.Id, artistName, metadata);
        return result.Map(scanResult => scanResult.Item);
    }

    // ersatztv#924: the library-items projection carries NO streams and no width/height (the server only reports
    // those through its per-item playback info), so the projection's version must never be written as the
    // statistics — doing so stored a version FFmpegStreamSelector cannot play, and, because
    // IMetadataRepository.UpdateStatistics reconciles streams against the version it is given, removed every
    // stream an existing row already had. Statistics come from the media server, under the same condition
    // MediaServerMovieLibraryScanner.UpdateStatistics uses; when the server cannot supply them nothing is written,
    // so stored streams are kept rather than replaced by the projection's empty list. (`result.IsAdded` is implied
    // by the empty-streams arm while the projection carries no streams; it is kept so the condition stays the
    // movie scanner's.)
    private async Task<Either<BaseError, MediaItemScanResult<TMusicVideo>>> UpdateStatistics(
        TConnectionParameters connectionParameters,
        TLibrary library,
        MediaItemScanResult<TMusicVideo> result,
        TMusicVideo incoming,
        bool deepScan)
    {
        TMusicVideo existing = result.Item;

        if (deepScan || result.IsAdded || MediaServerEtag(existing) != MediaServerEtag(incoming) ||
            existing.GetHeadVersion().Streams is not { Count: > 0 })
        {
            Option<MediaVersion> maybeMediaVersion = await GetMediaServerStatistics(
                connectionParameters,
                library,
                result,
                incoming);

            foreach (MediaVersion mediaVersion in maybeMediaVersion)
            {
                if (await _metadataRepository.UpdateStatistics(result.Item, mediaVersion))
                {
                    result.IsUpdated = true;
                }
            }
        }

        return result;
    }

    private async Task<bool> ShouldScanItem(
        IMediaServerMusicVideoRepository<TLibrary, TMusicVideo, TEtag> musicVideoRepository,
        TLibrary library,
        ImmutableDictionary<string, TEtag> existingMusicVideos,
        TMusicVideo incoming,
        string localPath,
        bool deepScan)
    {
        // deep scan will always pull every music video
        if (deepScan)
        {
            return true;
        }

        string existingEtag = string.Empty;
        MediaItemState existingState = MediaItemState.Normal;
        if (existingMusicVideos.TryGetValue(MediaServerItemId(incoming), out TEtag? existingEntry))
        {
            existingEtag = existingEntry.Etag;
            existingState = existingEntry.State;
        }

        if (existingState is MediaItemState.Unavailable or MediaItemState.FileNotFound &&
            existingEtag == MediaServerEtag(incoming))
        {
            // skip scanning unavailable/file not found items that are unchanged and still don't exist locally
            if (!_fileSystem.File.Exists(localPath))
            {
                return false;
            }
        }
        else if (existingEtag == MediaServerEtag(incoming))
        {
            // item is unchanged, but file does not exist
            // don't scan, but mark as unavailable
            if (!_fileSystem.File.Exists(localPath))
            {
                if (existingState is not MediaItemState.Unavailable)
                {
                    foreach (int id in await musicVideoRepository.FlagUnavailable(library, incoming))
                    {
                        if (!await _scannerProxy.ReindexMediaItems([id], CancellationToken.None))
                        {
                            _logger.LogWarning("Failed to reindex media items from scanner process");
                        }
                    }
                }
            }

            return false;
        }

        if (existingEntry is null)
        {
            _logger.LogDebug("INSERT: new music video {Path}", localPath);
        }
        else
        {
            _logger.LogDebug("UPDATE: Etag has changed for music video {Path}", localPath);
        }

        return true;
    }

    protected abstract string MediaServerItemId(TMusicVideo musicVideo);
    protected abstract string MediaServerEtag(TMusicVideo musicVideo);

    // ersatztv#484: projectionFailures is the per-enumeration sink the api client reports swallowed
    // projection exceptions into; its Count gates the sweep below.
    protected abstract IAsyncEnumerable<Tuple<TMusicVideo, int>> GetMusicVideoLibraryItems(
        TConnectionParameters connectionParameters,
        TLibrary library,
        MediaServerProjectionFailureCounter projectionFailures);

    protected abstract Task<Either<BaseError, MediaItemScanResult<TMusicVideo>>> UpdateMetadata(
        MediaItemScanResult<TMusicVideo> result,
        TMusicVideo incoming,
        CancellationToken cancellationToken);

    // ersatztv#924: abstract rather than the movie scanner's virtual-returning-None, because a None default is
    // exactly the defect — an implementation that forgot to override it would scan music videos with no streams.
    // The returned version carries the chapters to store alongside the statistics.
    protected abstract Task<Option<MediaVersion>> GetMediaServerStatistics(
        TConnectionParameters connectionParameters,
        TLibrary library,
        MediaItemScanResult<TMusicVideo> result,
        TMusicVideo incoming);
}