using System.Collections.Immutable; using System.IO.Abstractions; using ErsatzTV.Core; using ErsatzTV.Core.Domain; using ErsatzTV.Core.Domain.MediaServer; using ErsatzTV.Core.Errors; using ErsatzTV.Core.Extensions; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Metadata; using ErsatzTV.Scanner.Core.Interfaces; using Microsoft.Extensions.Logging; namespace ErsatzTV.Scanner.Core.Metadata; // ersatztv#496: music videos used to be reconciled by a LIBRARY-SCOPED local-path diff plus hard delete, because // they carried no server identity. They now carry one (JellyfinMusicVideo.ItemId/Etag), so this scanner is the // music-video sibling of MediaServer{Movie,Television,OtherVideo}LibraryScanner: it diffs on the server item id // and soft-trashes (FileNotFound) instead of deleting, which is reversible and EmptyTrash-governed. public abstract class MediaServerMusicVideoLibraryScanner where TConnectionParameters : MediaServerConnectionParameters where TLibrary : Library where TMusicVideo : MusicVideo where TEtag : MediaServerItemEtag { private const string UnknownArtist = "Unknown Artist"; private readonly IArtistRepository _artistRepository; private readonly IFileSystem _fileSystem; private readonly ILibraryRepository _libraryRepository; private readonly ILogger _logger; private readonly IMetadataRepository _metadataRepository; private readonly IScannerProxy _scannerProxy; protected MediaServerMusicVideoLibraryScanner( IScannerProxy scannerProxy, IFileSystem fileSystem, IArtistRepository artistRepository, ILibraryRepository libraryRepository, IMetadataRepository metadataRepository, ILogger logger) { _scannerProxy = scannerProxy; _fileSystem = fileSystem; _artistRepository = artistRepository; _libraryRepository = libraryRepository; _metadataRepository = metadataRepository; _logger = logger; } protected async Task> ScanLibrary( IMediaServerMusicVideoRepository musicVideoRepository, TConnectionParameters connectionParameters, TLibrary library, LibraryPath libraryPath, Func getLocalPath, bool deepScan, CancellationToken cancellationToken) { try { // ersatztv#484: one counter per enumeration, created here and read only after the enumeration // completes. It is never a field on the (singleton) api client, so concurrent scans of different // libraries cannot leak failures into each other's sweep decision. var projectionFailures = new MediaServerProjectionFailureCounter(); return await ScanLibrary( musicVideoRepository, connectionParameters, library, libraryPath, getLocalPath, GetMusicVideoLibraryItems(connectionParameters, library, projectionFailures), projectionFailures, deepScan, cancellationToken); } catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException) { return new ScanCanceled(); } } private async Task> ScanLibrary( IMediaServerMusicVideoRepository musicVideoRepository, TConnectionParameters connectionParameters, TLibrary library, LibraryPath libraryPath, Func getLocalPath, IAsyncEnumerable> musicVideoEntries, MediaServerProjectionFailureCounter projectionFailures, bool deepScan, CancellationToken cancellationToken) { var incomingItemIds = new List(); var incomingLocalPaths = new List(); var existingMusicVideos = (await musicVideoRepository.GetExistingMusicVideos(library)) .ToImmutableDictionary(e => e.MediaServerItemId, e => e); await foreach ((TMusicVideo incoming, int totalMusicVideoCount) in musicVideoEntries.WithCancellation( cancellationToken)) { if (cancellationToken.IsCancellationRequested) { return new ScanCanceled(); } incomingItemIds.Add(MediaServerItemId(incoming)); decimal percentCompletion = totalMusicVideoCount == 0 ? 1 : Math.Clamp((decimal)incomingItemIds.Count / totalMusicVideoCount, 0, 1); if (!await _scannerProxy.UpdateProgress(percentCompletion, cancellationToken)) { return new ScanCanceled(); } string localPath = getLocalPath(incoming); incomingLocalPaths.Add(localPath); if (!await ShouldScanItem( musicVideoRepository, library, existingMusicVideos, incoming, localPath, deepScan)) { continue; } Either> maybeMusicVideo = await ProcessMusicVideo( musicVideoRepository, library, libraryPath, incoming, localPath, deepScan, cancellationToken); if (maybeMusicVideo.IsLeft) { foreach (BaseError error in maybeMusicVideo.LeftToSeq()) { _logger.LogWarning("Error processing music video: {Error}", error.Value); } continue; } foreach (MediaItemScanResult result in maybeMusicVideo.RightToSeq()) { await musicVideoRepository.SetEtag(result.Item, MediaServerEtag(incoming)); if (_fileSystem.File.Exists(result.LocalPath)) { Option flagResult = await musicVideoRepository.FlagNormal(library, result.Item); if (flagResult.IsSome) { result.IsUpdated = true; } } else { Option flagResult = await musicVideoRepository.FlagUnavailable(library, result.Item); if (flagResult.IsSome) { result.IsUpdated = true; } } if (result.IsAdded || result.IsUpdated) { if (!await _scannerProxy.ReindexMediaItems([result.Item.Id], cancellationToken)) { _logger.LogWarning("Failed to reindex media items from scanner process"); } } } } await TrashMissingMusicVideos( musicVideoRepository, library, libraryPath, incomingItemIds, incomingLocalPaths, existingMusicVideos, projectionFailures, cancellationToken); return Unit.Default; } // Soft-trash music videos the media server no longer reports. Identity is the SERVER ITEM ID, scoped to this // library by GetExistingMusicVideos/FlagFileNotFound (both join LibraryPath.LibraryId), so a file served by two // libraries with overlapping local paths is two independent identities and one library's sweep can no longer // reach the other's row — the ersatztv#496 cross-library false-trash this replaced. private async Task TrashMissingMusicVideos( IMediaServerMusicVideoRepository musicVideoRepository, TLibrary library, LibraryPath libraryPath, List incomingItemIds, List incomingLocalPaths, ImmutableDictionary existingMusicVideos, MediaServerProjectionFailureCounter projectionFailures, CancellationToken cancellationToken) { // Legacy rows (no identity yet) are reconciled by local path. They are counted into the guard's existing // set below, because on the FIRST scan after the identity landed they are the entire library — a guard // that only saw the (empty) identity set would let a zero-item fetch sweep every one of them. List existingLegacyPaths = await musicVideoRepository.GetExistingLegacyMusicVideoPaths(library); // ersatztv#477: refuse the sweep when a successful fetch returned zero items but rows exist locally — an // empty incoming set is indistinguishable from a transient error and would otherwise flag a whole library. // ersatztv#484: a swallowed projection failure is indistinguishable from a deletion here, so it must // suppress the sweep. This gates the LEGACY path diff too — that one is if anything more exposed, since // a legacy row has no etag to fall back on. if (!MediaServerReconciliationGuard.ShouldFlagMissing( _logger, library.Name, incomingItemIds.Count, existingMusicVideos.Count + existingLegacyPaths.Count, projectionFailures.Count)) { return; } var fileNotFoundItemIds = existingMusicVideos.Keys.Except(incomingItemIds).ToList(); List ids = await musicVideoRepository.FlagFileNotFound(library, fileNotFoundItemIds); // A legacy row whose file IS still reported was adopted during this scan's loop, so it is no longer // legacy; what remains here is only what the server has genuinely stopped reporting. var fileNotFoundPaths = existingLegacyPaths.Except(incomingLocalPaths).ToList(); ids = ids.Concat(await musicVideoRepository.FlagFileNotFoundByPaths(library, fileNotFoundPaths)).ToList(); if (ids.Count > 0 && !await _scannerProxy.ReindexMediaItems(ids.ToArray(), cancellationToken)) { _logger.LogWarning("Failed to reindex media items from scanner process"); } // Trashed music videos are still rows, so an artist only becomes empty once the trash is emptied (or its // items are removed some other way); this keeps ersatztv#494's empty-artist cleanup without the delete. List artistIds = await _artistRepository.DeleteEmptyArtists(libraryPath); if (artistIds.Count > 0 && !await _scannerProxy.RemoveMediaItems(artistIds.ToArray(), cancellationToken)) { _logger.LogWarning("Failed to remove empty artists from scanner process"); } } private async Task>> ProcessMusicVideo( IMediaServerMusicVideoRepository musicVideoRepository, TLibrary library, LibraryPath libraryPath, TMusicVideo incoming, string localPath, bool deepScan, CancellationToken cancellationToken) { // ersatztv#488: resolve the owning folder from the DB, never from libraryPath.LibraryFolders (that // navigation is only eager-loaded on the local scan path and is null for media-server callers). string folder = Path.GetDirectoryName(localPath) ?? libraryPath.Path; Option maybeParentFolder = await _libraryRepository.GetParentFolderId(libraryPath, folder, cancellationToken); LibraryFolder libraryFolder = await _libraryRepository.GetOrAddFolder(libraryPath, maybeParentFolder, folder); return await GetOrAddArtist(libraryPath, incoming) .BindT(artist => musicVideoRepository.GetOrAdd( library, artist, libraryFolder, incoming, localPath, deepScan, cancellationToken)) .MapT(result => { result.LocalPath = localPath; return result; }) .BindT(result => UpdateMetadata(result, incoming, cancellationToken)) .BindT(result => UpdateStatistics(result, incoming)); } private async Task> GetOrAddArtist(LibraryPath libraryPath, TMusicVideo musicVideo) { string artistName = musicVideo.MusicVideoMetadata .HeadOrNone() .Bind(metadata => Optional(metadata.Artists).Flatten().HeadOrNone()) .Map(artist => artist.Name) .IfNone(UnknownArtist); var metadata = new ArtistMetadata { MetadataKind = MetadataKind.External, Title = artistName, SortTitle = SortTitle.GetSortTitle(artistName), DateAdded = DateTime.UtcNow, Genres = [], Styles = [], Moods = [], Artwork = [], Guids = [], Subtitles = [] }; Option maybeArtist = await _artistRepository.GetArtistByMetadata(libraryPath.Id, metadata); foreach (Artist artist in maybeArtist) { return artist; } Either> result = await _artistRepository.AddArtist(libraryPath.Id, artistName, metadata); return result.Map(scanResult => scanResult.Item); } private async Task>> UpdateStatistics( MediaItemScanResult result, TMusicVideo incoming) { if (await _metadataRepository.UpdateStatistics(result.Item, incoming.GetHeadVersion())) { result.IsUpdated = true; } return result; } private async Task ShouldScanItem( IMediaServerMusicVideoRepository musicVideoRepository, TLibrary library, ImmutableDictionary existingMusicVideos, TMusicVideo incoming, string localPath, bool deepScan) { // deep scan will always pull every music video if (deepScan) { return true; } string existingEtag = string.Empty; MediaItemState existingState = MediaItemState.Normal; if (existingMusicVideos.TryGetValue(MediaServerItemId(incoming), out TEtag? existingEntry)) { existingEtag = existingEntry.Etag; existingState = existingEntry.State; } if (existingState is MediaItemState.Unavailable or MediaItemState.FileNotFound && existingEtag == MediaServerEtag(incoming)) { // skip scanning unavailable/file not found items that are unchanged and still don't exist locally if (!_fileSystem.File.Exists(localPath)) { return false; } } else if (existingEtag == MediaServerEtag(incoming)) { // item is unchanged, but file does not exist // don't scan, but mark as unavailable if (!_fileSystem.File.Exists(localPath)) { if (existingState is not MediaItemState.Unavailable) { foreach (int id in await musicVideoRepository.FlagUnavailable(library, incoming)) { if (!await _scannerProxy.ReindexMediaItems([id], CancellationToken.None)) { _logger.LogWarning("Failed to reindex media items from scanner process"); } } } } return false; } if (existingEntry is null) { _logger.LogDebug("INSERT: new music video {Path}", localPath); } else { _logger.LogDebug("UPDATE: Etag has changed for music video {Path}", localPath); } return true; } protected abstract string MediaServerItemId(TMusicVideo musicVideo); protected abstract string MediaServerEtag(TMusicVideo musicVideo); // ersatztv#484: projectionFailures is the per-enumeration sink the api client reports swallowed // projection exceptions into; its Count gates the sweep below. protected abstract IAsyncEnumerable> GetMusicVideoLibraryItems( TConnectionParameters connectionParameters, TLibrary library, MediaServerProjectionFailureCounter projectionFailures); protected abstract Task>> UpdateMetadata( MediaItemScanResult result, TMusicVideo incoming, CancellationToken cancellationToken); }