timothy/ersatztv
1
0
Fork 0
Code Issues 51 Pull Requests 5 Actions Packages Projects Releases 9 Wiki Activity

properly restore all local library items from trash during scans (#1660)

Browse Source
This commit is contained in:
Jason Dove
2024-03-29 20:03:19 -05:00
committed by GitHub
parent 14a707a4e2
commit 7702999b9a
9 changed files with 106 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -55,6 +55,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Fix bug where replacing files in Plex would be missed by subsequent ETV library scans
- This fix will require a one-time re-scan of each Plex library in full
- After the initial full scan, incremental scans will behave as normal
- Fix edge case where some local episodes, music videos, other videos, songs, images would not automatically be restored from trash
### Changed
- Log search index updates under scanner category at debug level, to indicate a potential cause for the UI being out of date
ErsatzTV.Core/Interfaces/Repositories/ILibraryRepository.cs
-1
View File
@@ -18,5 +18,4 @@ public interface ILibraryRepository
Task<LibraryFolder> GetOrAddFolder(LibraryPath libraryPath, Option<int> maybeParentFolder, string folder);
Task UpdateLibraryFolderId(MediaFile mediaFile, int libraryFolderId);
Task UpdatePath(LibraryPath libraryPath, string normalizedLibraryPath);
Task<System.Collections.Generic.HashSet<string>> FindAllMissingFiles(LibraryPath libraryPath);
}
ErsatzTV.Infrastructure/Data/Repositories/LibraryRepository.cs
-36
View File
@@ -1,6 +1,5 @@
using Dapper;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Extensions;
using ErsatzTV.Core.Interfaces.Metadata;
using ErsatzTV.Core.Interfaces.Repositories;
using ErsatzTV.Infrastructure.Extensions;
@@ -212,41 +211,6 @@ public class LibraryRepository : ILibraryRepository
new { Path = normalizedLibraryPath, libraryPath.Id });
}
public async Task<System.Collections.Generic.HashSet<string>> FindAllMissingFiles(LibraryPath libraryPath)
{
var result = new System.Collections.Generic.HashSet<string>();
await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync();
IAsyncEnumerable<MediaItem> items = dbContext.MediaItems
.AsNoTracking()
.Filter(mi => mi.LibraryPathId == libraryPath.Id)
.Filter(mi => mi.State == MediaItemState.FileNotFound)
.Include(mi => (mi as Movie).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(mi => (mi as Episode).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(mi => (mi as Song).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(mi => (mi as MusicVideo).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(mi => (mi as OtherVideo).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.Include(mi => (mi as Image).MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
.AsAsyncEnumerable();
await foreach (MediaItem item in items)
{
MediaVersion version = item.GetHeadVersion();
foreach (MediaFile file in version.MediaFiles)
{
result.Add(file.Path);
}
}
return result;
}
private static LibraryFolder CreateNewFolder(LibraryPath libraryPath, Option<int> maybeParentFolder, string folder)
{
int? parentId = null;
ErsatzTV.Infrastructure/Data/Repositories/MediaItemRepository.cs
+1 -1
View File
@@ -91,7 +91,7 @@ public class MediaItemRepository : IMediaItemRepository
return await dbContext.Connection.QueryAsync<string>(
@"SELECT MF.Path
FROM MediaItem M
INNER JOIN MediaVersion MV on M.Id = COALESCE(MovieId, MusicVideoId, OtherVideoId, SongId, EpisodeId)
INNER JOIN MediaVersion MV on M.Id = COALESCE(MovieId, MusicVideoId, OtherVideoId, SongId, EpisodeId, ImageId)
INNER JOIN MediaFile MF on MV.Id = MF.MediaVersionId
WHERE M.State IN (1,2) AND M.LibraryPathId = @LibraryPathId",
new { LibraryPathId = libraryPath.Id })
ErsatzTV.Scanner/Core/Metadata/ImageFolderScanner.cs
+22 -4
View File
@@ -1,4 +1,5 @@
using Bugsnag;
using System.Collections.Immutable;
using Bugsnag;
using ErsatzTV.Core;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Errors;
@@ -20,6 +21,7 @@ public class ImageFolderScanner : LocalFolderScanner, IImageFolderScanner
private readonly IClient _client;
private readonly IImageRepository _imageRepository;
private readonly ILibraryRepository _libraryRepository;
private readonly IMediaItemRepository _mediaItemRepository;
private readonly ILocalFileSystem _localFileSystem;
private readonly ILocalMetadataProvider _localMetadataProvider;
private readonly ILogger<ImageFolderScanner> _logger;
@@ -54,6 +56,7 @@ public class ImageFolderScanner : LocalFolderScanner, IImageFolderScanner
_mediator = mediator;
_imageRepository = imageRepository;
_libraryRepository = libraryRepository;
_mediaItemRepository = mediaItemRepository;
_client = client;
_logger = logger;
}
@@ -83,6 +86,8 @@ public class ImageFolderScanner : LocalFolderScanner, IImageFolderScanner
await _libraryRepository.UpdatePath(libraryPath, normalizedLibraryPath);
}
ImmutableHashSet<string> allTrashedItems = await _mediaItemRepository.GetAllTrashedItems(libraryPath);
if (ShouldIncludeFolder(libraryPath.Path) && allFolders.Add(libraryPath.Path))
{
folderQueue.Enqueue(libraryPath.Path);
@@ -139,10 +144,23 @@ public class ImageFolderScanner : LocalFolderScanner, IImageFolderScanner
maybeParentFolder,
imageFolder);
// skip folder if etag matches
if (allFiles.Count == 0 || knownFolder.Etag == etag)
if (knownFolder.Etag == etag)
{
continue;
if (allFiles.Any(allTrashedItems.Contains))
{
_logger.LogDebug("Previously trashed items are now present in folder {Folder}", imageFolder);
}
else
{
// etag matches and no trashed items are now present, continue to next folder
continue;
}
}
else
{
_logger.LogDebug(
"UPDATE: Etag has changed for folder {Folder}",
imageFolder);
}
// walk up to get duration, if needed
ErsatzTV.Scanner/Core/Metadata/MusicVideoFolderScanner.cs
+23 -3
View File
@@ -1,4 +1,5 @@
using Bugsnag;
using System.Collections.Immutable;
using Bugsnag;
using ErsatzTV.Core;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Errors;
@@ -20,6 +21,7 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan
private readonly IArtistRepository _artistRepository;
private readonly IClient _client;
private readonly ILibraryRepository _libraryRepository;
private readonly IMediaItemRepository _mediaItemRepository;
private readonly ILocalFileSystem _localFileSystem;
private readonly ILocalMetadataProvider _localMetadataProvider;
private readonly ILocalSubtitlesProvider _localSubtitlesProvider;
@@ -59,6 +61,7 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan
_artistRepository = artistRepository;
_musicVideoRepository = musicVideoRepository;
_libraryRepository = libraryRepository;
_mediaItemRepository = mediaItemRepository;
_mediator = mediator;
_client = client;
_logger = logger;
@@ -84,6 +87,8 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan
await _libraryRepository.UpdatePath(libraryPath, normalizedLibraryPath);
}
ImmutableHashSet<string> allTrashedItems = await _mediaItemRepository.GetAllTrashedItems(libraryPath);
var allArtistFolders = _localFileSystem.ListSubdirectories(libraryPath.Path)
.Filter(ShouldIncludeFolder)
.OrderBy(identity)
@@ -151,6 +156,7 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan
ffprobePath,
result.Item,
artistFolder,
allTrashedItems,
cancellationToken);
foreach (ScanCanceled error in scanResult.LeftToSeq().OfType<ScanCanceled>())
@@ -312,6 +318,7 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan
string ffprobePath,
Artist artist,
string artistFolder,
ImmutableHashSet<string> allTrashedItems,
CancellationToken cancellationToken)
{
var folderQueue = new Queue<string>();
@@ -345,10 +352,23 @@ public class MusicVideoFolderScanner : LocalFolderScanner, IMusicVideoFolderScan
maybeParentFolder,
musicVideoFolder);
// skip folder if etag matches
if (knownFolder.Etag == etag)
{
continue;
if (allFiles.Any(allTrashedItems.Contains))
{
_logger.LogDebug("Previously trashed items are now present in folder {Folder}", musicVideoFolder);
}
else
{
// etag matches and no trashed items are now present, continue to next folder
continue;
}
}
else
{
_logger.LogDebug(
"UPDATE: Etag has changed for folder {Folder}",
musicVideoFolder);
}
var hasErrors = false;
ErsatzTV.Scanner/Core/Metadata/OtherVideoFolderScanner.cs
+19 -35
View File
@@ -1,4 +1,5 @@
using Bugsnag;
using System.Collections.Immutable;
using Bugsnag;
using ErsatzTV.Core;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Errors;
@@ -19,6 +20,7 @@ public class OtherVideoFolderScanner : LocalFolderScanner, IOtherVideoFolderScan
{
private readonly IClient _client;
private readonly ILibraryRepository _libraryRepository;
private readonly IMediaItemRepository _mediaItemRepository;
private readonly ILocalFileSystem _localFileSystem;
private readonly ILocalMetadataProvider _localMetadataProvider;
private readonly ILocalSubtitlesProvider _localSubtitlesProvider;
@@ -57,6 +59,7 @@ public class OtherVideoFolderScanner : LocalFolderScanner, IOtherVideoFolderScan
_mediator = mediator;
_otherVideoRepository = otherVideoRepository;
_libraryRepository = libraryRepository;
_mediaItemRepository = mediaItemRepository;
_client = client;
_logger = logger;
}
@@ -78,16 +81,7 @@ public class OtherVideoFolderScanner : LocalFolderScanner, IOtherVideoFolderScan
var allFolders = new System.Collections.Generic.HashSet<string>();
var folderQueue = new Queue<string>();
System.Collections.Generic.HashSet<string> allMissingFiles =
await _libraryRepository.FindAllMissingFiles(libraryPath);
if (allMissingFiles.Count > 0)
{
_logger.LogDebug(
"Library path {Path} has {Count} missing files",
libraryPath.Path,
allMissingFiles.Count);
}
ImmutableHashSet<string> allTrashedItems = await _mediaItemRepository.GetAllTrashedItems(libraryPath);
string normalizedLibraryPath = libraryPath.Path.TrimEnd(
Path.DirectorySeparatorChar,
@@ -160,35 +154,25 @@ public class OtherVideoFolderScanner : LocalFolderScanner, IOtherVideoFolderScan
maybeParentFolder,
otherVideoFolder);
bool hasMissingFiles = allFiles.Any(allMissingFiles.Contains);
bool isSameEtag = !hasMissingFiles && knownFolder.Etag == etag;
_logger.LogDebug(
"Scanning other video folder {Folder}; file count: {Count}, etag: {LastEtag} => {Etag}; has missing files: {HasMissingFiles}",
otherVideoFolder,
allFiles.Count,
etag,
knownFolder.Etag,
hasMissingFiles);
// skip empty folder
if (allFiles.Count == 0)
if (knownFolder.Etag == etag)
{
_logger.LogDebug("Skipping empty other videos folder");
continue;
if (allFiles.Any(allTrashedItems.Contains))
{
_logger.LogDebug("Previously trashed items are now present in folder {Folder}", otherVideoFolder);
}
else
{
// etag matches and no trashed items are now present, continue to next folder
continue;
}
}
// skip folder if etag matches
if (isSameEtag)
else
{
_logger.LogDebug("Skipping unchanged other videos folder, that contains no missing items");
continue;
_logger.LogDebug(
"UPDATE: Etag has changed for folder {Folder}",
otherVideoFolder);
}
_logger.LogDebug(
"UPDATE: Etag has changed for folder {Folder}",
otherVideoFolder);
var hasErrors = false;
foreach (string file in allFiles.OrderBy(identity))
ErsatzTV.Scanner/Core/Metadata/SongFolderScanner.cs
+23 -5
View File
@@ -1,4 +1,5 @@
using Bugsnag;
using System.Collections.Immutable;
using Bugsnag;
using ErsatzTV.Core;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Errors;
@@ -19,6 +20,7 @@ public class SongFolderScanner : LocalFolderScanner, ISongFolderScanner
{
private readonly IClient _client;
private readonly ILibraryRepository _libraryRepository;
private readonly IMediaItemRepository _mediaItemRepository;
private readonly ILocalFileSystem _localFileSystem;
private readonly ILocalMetadataProvider _localMetadataProvider;
private readonly ILogger<SongFolderScanner> _logger;
@@ -54,6 +56,7 @@ public class SongFolderScanner : LocalFolderScanner, ISongFolderScanner
_mediator = mediator;
_songRepository = songRepository;
_libraryRepository = libraryRepository;
_mediaItemRepository = mediaItemRepository;
_client = client;
_logger = logger;
}
@@ -82,6 +85,8 @@ public class SongFolderScanner : LocalFolderScanner, ISongFolderScanner
await _libraryRepository.UpdatePath(libraryPath, normalizedLibraryPath);
}
ImmutableHashSet<string> allTrashedItems = await _mediaItemRepository.GetAllTrashedItems(libraryPath);
if (ShouldIncludeFolder(libraryPath.Path))
{
folderQueue.Enqueue(libraryPath.Path);
@@ -135,11 +140,24 @@ public class SongFolderScanner : LocalFolderScanner, ISongFolderScanner
libraryPath,
maybeParentFolder,
songFolder);
// skip folder if etag matches
if (allFiles.Count == 0 || knownFolder.Etag == etag)
if (knownFolder.Etag == etag)
{
continue;
if (allFiles.Any(allTrashedItems.Contains))
{
_logger.LogDebug("Previously trashed items are now present in folder {Folder}", songFolder);
}
else
{
// etag matches and no trashed items are now present, continue to next folder
continue;
}
}
else
{
_logger.LogDebug(
"UPDATE: Etag has changed for folder {Folder}",
songFolder);
}
_logger.LogDebug(
ErsatzTV.Scanner/Core/Metadata/TelevisionFolderScanner.cs
+17 -2
View File
@@ -1,4 +1,5 @@
using Bugsnag;
using System.Collections.Immutable;
using Bugsnag;
using ErsatzTV.Core;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Errors;
@@ -20,6 +21,7 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan
private readonly IClient _client;
private readonly IFallbackMetadataProvider _fallbackMetadataProvider;
private readonly ILibraryRepository _libraryRepository;
private readonly IMediaItemRepository _mediaItemRepository;
private readonly ILocalFileSystem _localFileSystem;
private readonly ILocalMetadataProvider _localMetadataProvider;
private readonly ILocalSubtitlesProvider _localSubtitlesProvider;
@@ -60,6 +62,7 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan
_localSubtitlesProvider = localSubtitlesProvider;
_metadataRepository = metadataRepository;
_libraryRepository = libraryRepository;
_mediaItemRepository = mediaItemRepository;
_mediator = mediator;
_client = client;
_fallbackMetadataProvider = fallbackMetadataProvider;
@@ -85,6 +88,8 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan
{
await _libraryRepository.UpdatePath(libraryPath, normalizedLibraryPath);
}
ImmutableHashSet<string> allTrashedItems = await _mediaItemRepository.GetAllTrashedItems(libraryPath);
var allShowFolders = _localFileSystem.ListSubdirectories(libraryPath.Path)
.Filter(ShouldIncludeFolder)
@@ -153,6 +158,7 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan
ffprobePath,
result.Item,
showFolder,
allTrashedItems,
cancellationToken);
foreach (ScanCanceled error in scanResult.LeftToSeq().OfType<ScanCanceled>())
@@ -227,6 +233,7 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan
string ffprobePath,
Show show,
string showFolder,
ImmutableHashSet<string> allTrashedItems,
CancellationToken cancellationToken)
{
foreach (string seasonFolder in _localFileSystem.ListSubdirectories(showFolder).Filter(ShouldIncludeFolder)
@@ -248,7 +255,15 @@ public class TelevisionFolderScanner : LocalFolderScanner, ITelevisionFolderScan
// skip folder if etag matches
if (knownFolder.Etag == etag)
{
continue;
if (allTrashedItems.Any(f => f.StartsWith(seasonFolder, StringComparison.OrdinalIgnoreCase)))
{
_logger.LogDebug("Previously trashed items are now present in folder {Folder}", seasonFolder);
}
else
{
// etag matches and no trashed items are now present, continue to next folder
continue;
}
}
Option<int> maybeSeasonNumber = _fallbackMetadataProvider.GetSeasonNumberForFolder(seasonFolder);