diff --git a/CHANGELOG.md b/CHANGELOG.md index 495668336..c4cb8cbb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Fix bug where replacing files in Plex would be missed by subsequent ETV library scans - This fix will require a one-time re-scan of each Plex library in full - After the initial full scan, incremental scans will behave as normal +- Fix edge case where some local episodes, music videos, other videos, songs, images would not automatically be restored from trash ### Changed - Log search index updates under scanner category at debug level, to indicate a potential cause for the UI being out of date diff --git a/ErsatzTV.Core/Interfaces/Repositories/ILibraryRepository.cs b/ErsatzTV.Core/Interfaces/Repositories/ILibraryRepository.cs index d7a5c170d..98568d865 100644 --- a/ErsatzTV.Core/Interfaces/Repositories/ILibraryRepository.cs +++ b/ErsatzTV.Core/Interfaces/Repositories/ILibraryRepository.cs @@ -18,5 +18,4 @@ public interface ILibraryRepository Task GetOrAddFolder(LibraryPath libraryPath, Option maybeParentFolder, string folder); Task UpdateLibraryFolderId(MediaFile mediaFile, int libraryFolderId); Task UpdatePath(LibraryPath libraryPath, string normalizedLibraryPath); - Task> FindAllMissingFiles(LibraryPath libraryPath); } diff --git a/ErsatzTV.Infrastructure/Data/Repositories/LibraryRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/LibraryRepository.cs index 5c704eaf1..4380a5f37 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/LibraryRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/LibraryRepository.cs @@ -1,6 +1,5 @@ using Dapper; using ErsatzTV.Core.Domain; -using ErsatzTV.Core.Extensions; using ErsatzTV.Core.Interfaces.Metadata; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Infrastructure.Extensions; @@ -212,41 +211,6 @@ public class LibraryRepository : ILibraryRepository new { Path = normalizedLibraryPath, libraryPath.Id }); } - public async Task> FindAllMissingFiles(LibraryPath libraryPath) - { - var result = new System.Collections.Generic.HashSet(); - - await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(); - IAsyncEnumerable items = dbContext.MediaItems - .AsNoTracking() - .Filter(mi => mi.LibraryPathId == libraryPath.Id) - .Filter(mi => mi.State == MediaItemState.FileNotFound) - .Include(mi => (mi as Movie).MediaVersions) - .ThenInclude(mv => mv.MediaFiles) - .Include(mi => (mi as Episode).MediaVersions) - .ThenInclude(mv => mv.MediaFiles) - .Include(mi => (mi as Song).MediaVersions) - .ThenInclude(mv => mv.MediaFiles) - .Include(mi => (mi as MusicVideo).MediaVersions) - .ThenInclude(mv => mv.MediaFiles) - .Include(mi => (mi as OtherVideo).MediaVersions) - .ThenInclude(mv => mv.MediaFiles) - .Include(mi => (mi as Image).MediaVersions) - .ThenInclude(mv => mv.MediaFiles) - .AsAsyncEnumerable(); - - await foreach (MediaItem item in items) - { - MediaVersion version = item.GetHeadVersion(); - foreach (MediaFile file in version.MediaFiles) - { - result.Add(file.Path); - } - } - - return result; - } - private static LibraryFolder CreateNewFolder(LibraryPath libraryPath, Option maybeParentFolder, string folder) { int? parentId = null; diff --git a/ErsatzTV.Infrastructure/Data/Repositories/MediaItemRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/MediaItemRepository.cs index b8b6c58b8..16a1500bc 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/MediaItemRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/MediaItemRepository.cs @@ -91,7 +91,7 @@ public class MediaItemRepository : IMediaItemRepository return await dbContext.Connection.QueryAsync( @"SELECT MF.Path FROM MediaItem M - INNER JOIN MediaVersion MV on M.Id = COALESCE(MovieId, MusicVideoId, OtherVideoId, SongId, EpisodeId) + INNER JOIN MediaVersion MV on M.Id = COALESCE(MovieId, MusicVideoId, OtherVideoId, SongId, EpisodeId, ImageId) INNER JOIN MediaFile MF on MV.Id = MF.MediaVersionId WHERE M.State IN (1,2) AND M.LibraryPathId = @LibraryPathId", new { LibraryPathId = libraryPath.Id }) diff --git a/ErsatzTV.Scanner/Core/Metadata/ImageFolderScanner.cs b/ErsatzTV.Scanner/Core/Metadata/ImageFolderScanner.cs index 633d6f9ee..378d7b890 100644 --- a/ErsatzTV.Scanner/Core/Metadata/ImageFolderScanner.cs +++ b/ErsatzTV.Scanner/Core/Metadata/ImageFolderScanner.cs @@ -1,4 +1,5 @@ -using Bugsnag; +using System.Collections.Immutable; +using Bugsnag; using ErsatzTV.Core; using ErsatzTV.Core.Domain; using ErsatzTV.Core.Errors; @@ -20,6 +21,7 @@ public class ImageFolderScanner : LocalFolderScanner, IImageFolderScanner private readonly IClient _client; private readonly IImageRepository _imageRepository; private readonly ILibraryRepository _libraryRepository; + private readonly IMediaItemRepository _mediaItemRepository; private readonly ILocalFileSystem _localFileSystem; private readonly ILocalMetadataProvider _localMetadataProvider; private readonly ILogger _logger; @@ -54,6 +56,7 @@ public class ImageFolderScanner : LocalFolderScanner, IImageFolderScanner _mediator = mediator; _imageRepository = imageRepository; _libraryRepository = libraryRepository; + _mediaItemRepository = mediaItemRepository; _client = client; _logger = logger; } @@ -83,6 +86,8 @@ public class ImageFolderScanner : LocalFolderScanner, IImageFolderScanner await _libraryRepository.UpdatePath(libraryPath, normalizedLibraryPath); } + ImmutableHashSet allTrashedItems = await _mediaItemRepository.GetAllTrashedItems(libraryPath); + if (ShouldIncludeFolder(libraryPath.Path) && allFolders.Add(libraryPath.Path)) { folderQueue.Enqueue(libraryPath.Path); @@ -139,10 +144,23 @@ public class ImageFolderScanner : LocalFolderScanner, IImageFolderScanner maybeParentFolder, imageFolder); - // skip folder if etag matches - if (allFiles.Count == 0 || knownFolder.Etag == etag) + if (knownFolder.Etag == etag) { - continue; + if (allFiles.Any(allTrashedItems.Contains)) + { + _logger.LogDebug("Previously trashed items are now present in folder {Folder}", imageFolder); + } + else + { + // etag matches and no trashed items are now present, continue to next folder + continue; + } + } + else + { + _logger.LogDebug( + "UPDATE: Etag has changed for folder {Folder}", + imageFolder); } // walk up to get duration, if needed diff --git a/ErsatzTV.Scanner/Core/Metadata/MusicVideoFolderScanner.cs b/ErsatzTV.Scanner/Core/Metadata/MusicVideoFolderScanner.cs index e2afe6303..6e7699612 100644 --- a/ErsatzTV.Scanner/Core/Metadata/MusicVideoFolderScanner.cs +++ b/ErsatzTV.Scanner/Core/Metadata/MusicVideoFolderScanner.cs @@ -1,4 +1,5 @@ -using Bugsnag; +using System.Collections.Immutable; +using Bugsnag; using ErsatzTV.Core; using ErsatzTV.Core.Domain; using ErsatzTV.Core.Errors; @@ -20,6 +21,7 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan private readonly IArtistRepository _artistRepository; private readonly IClient _client; private readonly ILibraryRepository _libraryRepository; + private readonly IMediaItemRepository _mediaItemRepository; private readonly ILocalFileSystem _localFileSystem; private readonly ILocalMetadataProvider _localMetadataProvider; private readonly ILocalSubtitlesProvider _localSubtitlesProvider; @@ -59,6 +61,7 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan _artistRepository = artistRepository; _musicVideoRepository = musicVideoRepository; _libraryRepository = libraryRepository; + _mediaItemRepository = mediaItemRepository; _mediator = mediator; _client = client; _logger = logger; @@ -84,6 +87,8 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan await _libraryRepository.UpdatePath(libraryPath, normalizedLibraryPath); } + ImmutableHashSet allTrashedItems = await _mediaItemRepository.GetAllTrashedItems(libraryPath); + var allArtistFolders = _localFileSystem.ListSubdirectories(libraryPath.Path) .Filter(ShouldIncludeFolder) .OrderBy(identity) @@ -151,6 +156,7 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan ffprobePath, result.Item, artistFolder, + allTrashedItems, cancellationToken); foreach (ScanCanceled error in scanResult.LeftToSeq().OfType()) @@ -312,6 +318,7 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan string ffprobePath, Artist artist, string artistFolder, + ImmutableHashSet allTrashedItems, CancellationToken cancellationToken) { var folderQueue = new Queue(); @@ -345,10 +352,23 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan maybeParentFolder, musicVideoFolder); - // skip folder if etag matches if (knownFolder.Etag == etag) { - continue; + if (allFiles.Any(allTrashedItems.Contains)) + { + _logger.LogDebug("Previously trashed items are now present in folder {Folder}", musicVideoFolder); + } + else + { + // etag matches and no trashed items are now present, continue to next folder + continue; + } + } + else + { + _logger.LogDebug( + "UPDATE: Etag has changed for folder {Folder}", + musicVideoFolder); } var hasErrors = false; diff --git a/ErsatzTV.Scanner/Core/Metadata/OtherVideoFolderScanner.cs b/ErsatzTV.Scanner/Core/Metadata/OtherVideoFolderScanner.cs index 6d4cdba55..90279d455 100644 --- a/ErsatzTV.Scanner/Core/Metadata/OtherVideoFolderScanner.cs +++ b/ErsatzTV.Scanner/Core/Metadata/OtherVideoFolderScanner.cs @@ -1,4 +1,5 @@ -using Bugsnag; +using System.Collections.Immutable; +using Bugsnag; using ErsatzTV.Core; using ErsatzTV.Core.Domain; using ErsatzTV.Core.Errors; @@ -19,6 +20,7 @@ public class OtherVideoFolderScanner : LocalFolderScanner, IOtherVideoFolderScan { private readonly IClient _client; private readonly ILibraryRepository _libraryRepository; + private readonly IMediaItemRepository _mediaItemRepository; private readonly ILocalFileSystem _localFileSystem; private readonly ILocalMetadataProvider _localMetadataProvider; private readonly ILocalSubtitlesProvider _localSubtitlesProvider; @@ -57,6 +59,7 @@ public class OtherVideoFolderScanner : LocalFolderScanner, IOtherVideoFolderScan _mediator = mediator; _otherVideoRepository = otherVideoRepository; _libraryRepository = libraryRepository; + _mediaItemRepository = mediaItemRepository; _client = client; _logger = logger; } @@ -78,16 +81,7 @@ public class OtherVideoFolderScanner : LocalFolderScanner, IOtherVideoFolderScan var allFolders = new System.Collections.Generic.HashSet(); var folderQueue = new Queue(); - System.Collections.Generic.HashSet allMissingFiles = - await _libraryRepository.FindAllMissingFiles(libraryPath); - - if (allMissingFiles.Count > 0) - { - _logger.LogDebug( - "Library path {Path} has {Count} missing files", - libraryPath.Path, - allMissingFiles.Count); - } + ImmutableHashSet allTrashedItems = await _mediaItemRepository.GetAllTrashedItems(libraryPath); string normalizedLibraryPath = libraryPath.Path.TrimEnd( Path.DirectorySeparatorChar, @@ -160,35 +154,25 @@ public class OtherVideoFolderScanner : LocalFolderScanner, IOtherVideoFolderScan maybeParentFolder, otherVideoFolder); - bool hasMissingFiles = allFiles.Any(allMissingFiles.Contains); - bool isSameEtag = !hasMissingFiles && knownFolder.Etag == etag; - - _logger.LogDebug( - "Scanning other video folder {Folder}; file count: {Count}, etag: {LastEtag} => {Etag}; has missing files: {HasMissingFiles}", - otherVideoFolder, - allFiles.Count, - etag, - knownFolder.Etag, - hasMissingFiles); - - // skip empty folder - if (allFiles.Count == 0) + if (knownFolder.Etag == etag) { - _logger.LogDebug("Skipping empty other videos folder"); - continue; + if (allFiles.Any(allTrashedItems.Contains)) + { + _logger.LogDebug("Previously trashed items are now present in folder {Folder}", otherVideoFolder); + } + else + { + // etag matches and no trashed items are now present, continue to next folder + continue; + } } - - // skip folder if etag matches - if (isSameEtag) + else { - _logger.LogDebug("Skipping unchanged other videos folder, that contains no missing items"); - continue; + _logger.LogDebug( + "UPDATE: Etag has changed for folder {Folder}", + otherVideoFolder); } - _logger.LogDebug( - "UPDATE: Etag has changed for folder {Folder}", - otherVideoFolder); - var hasErrors = false; foreach (string file in allFiles.OrderBy(identity)) diff --git a/ErsatzTV.Scanner/Core/Metadata/SongFolderScanner.cs b/ErsatzTV.Scanner/Core/Metadata/SongFolderScanner.cs index 7b87ad14a..8ea58e11d 100644 --- a/ErsatzTV.Scanner/Core/Metadata/SongFolderScanner.cs +++ b/ErsatzTV.Scanner/Core/Metadata/SongFolderScanner.cs @@ -1,4 +1,5 @@ -using Bugsnag; +using System.Collections.Immutable; +using Bugsnag; using ErsatzTV.Core; using ErsatzTV.Core.Domain; using ErsatzTV.Core.Errors; @@ -19,6 +20,7 @@ public class SongFolderScanner : LocalFolderScanner, ISongFolderScanner { private readonly IClient _client; private readonly ILibraryRepository _libraryRepository; + private readonly IMediaItemRepository _mediaItemRepository; private readonly ILocalFileSystem _localFileSystem; private readonly ILocalMetadataProvider _localMetadataProvider; private readonly ILogger _logger; @@ -54,6 +56,7 @@ public class SongFolderScanner : LocalFolderScanner, ISongFolderScanner _mediator = mediator; _songRepository = songRepository; _libraryRepository = libraryRepository; + _mediaItemRepository = mediaItemRepository; _client = client; _logger = logger; } @@ -82,6 +85,8 @@ public class SongFolderScanner : LocalFolderScanner, ISongFolderScanner await _libraryRepository.UpdatePath(libraryPath, normalizedLibraryPath); } + ImmutableHashSet allTrashedItems = await _mediaItemRepository.GetAllTrashedItems(libraryPath); + if (ShouldIncludeFolder(libraryPath.Path)) { folderQueue.Enqueue(libraryPath.Path); @@ -135,11 +140,24 @@ public class SongFolderScanner : LocalFolderScanner, ISongFolderScanner libraryPath, maybeParentFolder, songFolder); - - // skip folder if etag matches - if (allFiles.Count == 0 || knownFolder.Etag == etag) + + if (knownFolder.Etag == etag) { - continue; + if (allFiles.Any(allTrashedItems.Contains)) + { + _logger.LogDebug("Previously trashed items are now present in folder {Folder}", songFolder); + } + else + { + // etag matches and no trashed items are now present, continue to next folder + continue; + } + } + else + { + _logger.LogDebug( + "UPDATE: Etag has changed for folder {Folder}", + songFolder); } _logger.LogDebug( diff --git a/ErsatzTV.Scanner/Core/Metadata/TelevisionFolderScanner.cs b/ErsatzTV.Scanner/Core/Metadata/TelevisionFolderScanner.cs index 42fd978b2..6fbe96129 100644 --- a/ErsatzTV.Scanner/Core/Metadata/TelevisionFolderScanner.cs +++ b/ErsatzTV.Scanner/Core/Metadata/TelevisionFolderScanner.cs @@ -1,4 +1,5 @@ -using Bugsnag; +using System.Collections.Immutable; +using Bugsnag; using ErsatzTV.Core; using ErsatzTV.Core.Domain; using ErsatzTV.Core.Errors; @@ -20,6 +21,7 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan private readonly IClient _client; private readonly IFallbackMetadataProvider _fallbackMetadataProvider; private readonly ILibraryRepository _libraryRepository; + private readonly IMediaItemRepository _mediaItemRepository; private readonly ILocalFileSystem _localFileSystem; private readonly ILocalMetadataProvider _localMetadataProvider; private readonly ILocalSubtitlesProvider _localSubtitlesProvider; @@ -60,6 +62,7 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan _localSubtitlesProvider = localSubtitlesProvider; _metadataRepository = metadataRepository; _libraryRepository = libraryRepository; + _mediaItemRepository = mediaItemRepository; _mediator = mediator; _client = client; _fallbackMetadataProvider = fallbackMetadataProvider; @@ -85,6 +88,8 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan { await _libraryRepository.UpdatePath(libraryPath, normalizedLibraryPath); } + + ImmutableHashSet allTrashedItems = await _mediaItemRepository.GetAllTrashedItems(libraryPath); var allShowFolders = _localFileSystem.ListSubdirectories(libraryPath.Path) .Filter(ShouldIncludeFolder) @@ -153,6 +158,7 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan ffprobePath, result.Item, showFolder, + allTrashedItems, cancellationToken); foreach (ScanCanceled error in scanResult.LeftToSeq().OfType()) @@ -227,6 +233,7 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan string ffprobePath, Show show, string showFolder, + ImmutableHashSet allTrashedItems, CancellationToken cancellationToken) { foreach (string seasonFolder in _localFileSystem.ListSubdirectories(showFolder).Filter(ShouldIncludeFolder) @@ -248,7 +255,15 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan // skip folder if etag matches if (knownFolder.Etag == etag) { - continue; + if (allTrashedItems.Any(f => f.StartsWith(seasonFolder, StringComparison.OrdinalIgnoreCase))) + { + _logger.LogDebug("Previously trashed items are now present in folder {Folder}", seasonFolder); + } + else + { + // etag matches and no trashed items are now present, continue to next folder + continue; + } } Option maybeSeasonNumber = _fallbackMetadataProvider.GetSeasonNumberForFolder(seasonFolder);