timothy/ersatztv
1
0
Fork 0
Code Issues 51 Pull Requests 5 Actions Packages Projects Releases 9 Wiki Activity

optimize local scanning (#84)

Browse Source 
* optimize local scanning

* fix artwork updates

* fix adding genres and tags

* fix movie fallback metadata
This commit is contained in:
Jason Dove
2021-03-19 00:45:38 +00:00
committed by GitHub
parent c5c28cb92d
commit 739d074bc6
17 changed files with 296 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ErsatzTV.Core/Interfaces/Repositories/IMetadataRepository.cs
+4 -1
View File
@@ -7,7 +7,10 @@ namespace ErsatzTV.Core.Interfaces.Repositories
public interface IMetadataRepository
{
Task<Unit> RemoveGenre(Genre genre);
Task<Unit> UpdateStatistics(MediaVersion mediaVersion);
Task<bool> Update(Domain.Metadata metadata);
Task<bool> Add(Domain.Metadata metadata);
Task<bool> UpdateLocalStatistics(MediaVersion mediaVersion);
Task<Unit> UpdatePlexStatistics(MediaVersion mediaVersion);
Task<Unit> UpdateArtworkPath(Artwork artwork);
Task<Unit> AddArtwork(Domain.Metadata metadata, Artwork artwork);
Task<Unit> RemoveArtwork(Domain.Metadata metadata, ArtworkKind artworkKind);
ErsatzTV.Core/Interfaces/Repositories/IMovieRepository.cs
-1
View File
@@ -11,7 +11,6 @@ namespace ErsatzTV.Core.Interfaces.Repositories
Task<Option<Movie>> GetMovie(int movieId);
Task<Either<BaseError, Movie>> GetOrAdd(LibraryPath libraryPath, string path);
Task<Either<BaseError, PlexMovie>> GetOrAdd(PlexLibrary library, PlexMovie item);
Task<bool> Update(Movie movie);
Task<int> GetMovieCount();
Task<List<MovieMetadata>> GetPagedMovies(int pageNumber, int pageSize);
Task<IEnumerable<string>> FindMoviePaths(LibraryPath libraryPath);
ErsatzTV.Core/Interfaces/Repositories/ITelevisionRepository.cs
+1 -3
View File
@@ -8,9 +8,6 @@ namespace ErsatzTV.Core.Interfaces.Repositories
public interface ITelevisionRepository
{
Task<bool> AllShowsExist(List<int> showIds);
Task<bool> Update(Show show);
Task<bool> Update(Season season);
Task<bool> Update(Episode episode);
Task<List<Show>> GetAllShows();
Task<Option<Show>> GetShow(int showId);
Task<int> GetShowCount();
@@ -39,5 +36,6 @@ namespace ErsatzTV.Core.Interfaces.Repositories
Task<Unit> RemoveMissingPlexShows(PlexLibrary library, List<string> showKeys);
Task<Unit> RemoveMissingPlexSeasons(string showKey, List<string> seasonKeys);
Task<Unit> RemoveMissingPlexEpisodes(string seasonKey, List<string> episodeKeys);
Task<Unit> SetEpisodeNumber(Episode episode, int episodeNumber);
}
}
ErsatzTV.Core/Metadata/FallbackMetadataProvider.cs
+3
View File
@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text.RegularExpressions;
using ErsatzTV.Core.Domain;
@@ -86,6 +87,8 @@ namespace ErsatzTV.Core.Metadata
metadata.Title = match.Groups[1].Value;
metadata.Year = int.Parse(match.Groups[2].Value);
metadata.ReleaseDate = new DateTime(int.Parse(match.Groups[2].Value), 1, 1);
metadata.Genres = new List<Genre>();
metadata.Tags = new List<Tag>();
}
}
catch (Exception)
ErsatzTV.Core/Metadata/LocalFolderScanner.cs
+12 -9
View File
@@ -7,9 +7,9 @@ using System.Threading.Tasks;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Interfaces.Images;
using ErsatzTV.Core.Interfaces.Metadata;
using ErsatzTV.Core.Interfaces.Repositories;
using LanguageExt;
using Microsoft.Extensions.Logging;
using static LanguageExt.Prelude;
namespace ErsatzTV.Core.Metadata
{
@@ -48,17 +48,20 @@ namespace ErsatzTV.Core.Metadata
private readonly ILocalFileSystem _localFileSystem;
private readonly ILocalStatisticsProvider _localStatisticsProvider;
private readonly ILogger _logger;
private readonly IMetadataRepository _metadataRepository;
static LocalFolderScanner() => Crypto = new SHA1CryptoServiceProvider();
protected LocalFolderScanner(
ILocalFileSystem localFileSystem,
ILocalStatisticsProvider localStatisticsProvider,
IMetadataRepository metadataRepository,
IImageCache imageCache,
ILogger logger)
{
_localFileSystem = localFileSystem;
_localStatisticsProvider = localStatisticsProvider;
_metadataRepository = metadataRepository;
_imageCache = imageCache;
_logger = logger;
}
@@ -99,17 +102,16 @@ namespace ErsatzTV.Core.Metadata
}
}
protected bool RefreshArtwork(string artworkFile, Domain.Metadata metadata, ArtworkKind artworkKind)
protected async Task<bool> RefreshArtwork(string artworkFile, Domain.Metadata metadata, ArtworkKind artworkKind)
{
DateTime lastWriteTime = _localFileSystem.GetLastWriteTime(artworkFile);
metadata.Artwork ??= new List<Artwork>();
Option<Artwork> maybeArtwork =
Optional(metadata.Artwork).Flatten().FirstOrDefault(a => a.ArtworkKind == artworkKind);
Option<Artwork> maybeArtwork = metadata.Artwork.FirstOrDefault(a => a.ArtworkKind == artworkKind);
bool shouldRefresh = maybeArtwork.Match(
artwork => artwork.DateUpdated < lastWriteTime,
artwork => lastWriteTime.Subtract(artwork.DateUpdated) > TimeSpan.FromSeconds(1),
true);
if (shouldRefresh)
@@ -117,13 +119,14 @@ namespace ErsatzTV.Core.Metadata
_logger.LogDebug("Refreshing {Attribute} from {Path}", artworkKind, artworkFile);
string cacheName = _imageCache.CopyArtworkToCache(artworkFile, artworkKind);
maybeArtwork.Match(
artwork =>
await maybeArtwork.Match(
async artwork =>
{
artwork.Path = cacheName;
artwork.DateUpdated = lastWriteTime;
await _metadataRepository.UpdateArtworkPath(artwork);
},
() =>
async () =>
{
var artwork = new Artwork
{
@@ -132,8 +135,8 @@ namespace ErsatzTV.Core.Metadata
DateUpdated = lastWriteTime,
ArtworkKind = artworkKind
};
metadata.Artwork.Add(artwork);
await _metadataRepository.AddArtwork(metadata, artwork);
});
return true;
ErsatzTV.Core/Metadata/LocalMetadataProvider.cs
+25 -14
View File
@@ -23,16 +23,19 @@ namespace ErsatzTV.Core.Metadata
private readonly ILogger<LocalMetadataProvider> _logger;
private readonly IMediaItemRepository _mediaItemRepository;
private readonly IMetadataRepository _metadataRepository;
private readonly ITelevisionRepository _televisionRepository;
public LocalMetadataProvider(
IMediaItemRepository mediaItemRepository,
IMetadataRepository metadataRepository,
ITelevisionRepository televisionRepository,
IFallbackMetadataProvider fallbackMetadataProvider,
ILocalFileSystem localFileSystem,
ILogger<LocalMetadataProvider> logger)
{
_mediaItemRepository = mediaItemRepository;
_metadataRepository = metadataRepository;
_televisionRepository = televisionRepository;
_fallbackMetadataProvider = fallbackMetadataProvider;
_localFileSystem = localFileSystem;
@@ -92,8 +95,11 @@ namespace ErsatzTV.Core.Metadata
private async Task ApplyMetadataUpdate(Episode episode, Tuple<EpisodeMetadata, int> metadataEpisodeNumber)
{
(EpisodeMetadata metadata, int episodeNumber) = metadataEpisodeNumber;
episode.EpisodeNumber = episodeNumber;
Optional(episode.EpisodeMetadata).Flatten().HeadOrNone().Match(
if (episode.EpisodeNumber != episodeNumber)
{
await _televisionRepository.SetEpisodeNumber(episode, episodeNumber);
}
await Optional(episode.EpisodeMetadata).Flatten().HeadOrNone().Match(
existing =>
{
existing.Outline = metadata.Outline;
@@ -109,20 +115,22 @@ namespace ErsatzTV.Core.Metadata
existing.SortTitle = string.IsNullOrWhiteSpace(metadata.SortTitle)
? _fallbackMetadataProvider.GetSortTitle(metadata.Title)
: metadata.SortTitle;
return _metadataRepository.Update(existing);
},
() =>
{
metadata.SortTitle = string.IsNullOrWhiteSpace(metadata.SortTitle)
? _fallbackMetadataProvider.GetSortTitle(metadata.Title)
: metadata.SortTitle;
metadata.EpisodeId = episode.Id;
episode.EpisodeMetadata = new List<EpisodeMetadata> { metadata };
});
await _televisionRepository.Update(episode);
return _metadataRepository.Add(metadata);
});
}
private async Task ApplyMetadataUpdate(Movie movie, MovieMetadata metadata)
{
private Task ApplyMetadataUpdate(Movie movie, MovieMetadata metadata) =>
Optional(movie.MovieMetadata).Flatten().HeadOrNone().Match(
existing =>
{
@@ -163,20 +171,21 @@ namespace ErsatzTV.Core.Metadata
{
existing.Tags.Add(tag);
}
return _metadataRepository.Update(existing);
},
() =>
{
metadata.SortTitle = string.IsNullOrWhiteSpace(metadata.SortTitle)
? _fallbackMetadataProvider.GetSortTitle(metadata.Title)
: metadata.SortTitle;
metadata.MovieId = movie.Id;
movie.MovieMetadata = new List<MovieMetadata> { metadata };
return _metadataRepository.Add(metadata);
});
await _mediaItemRepository.Update(movie);
}
private async Task ApplyMetadataUpdate(Show show, ShowMetadata metadata)
{
private Task ApplyMetadataUpdate(Show show, ShowMetadata metadata) =>
Optional(show.ShowMetadata).Flatten().HeadOrNone().Match(
existing =>
{
@@ -217,17 +226,19 @@ namespace ErsatzTV.Core.Metadata
{
existing.Tags.Add(tag);
}
return _metadataRepository.Update(existing);
},
() =>
{
metadata.SortTitle = string.IsNullOrWhiteSpace(metadata.SortTitle)
? _fallbackMetadataProvider.GetSortTitle(metadata.Title)
: metadata.SortTitle;
metadata.ShowId = show.Id;
show.ShowMetadata = new List<ShowMetadata> { metadata };
});
await _televisionRepository.Update(show);
}
return _metadataRepository.Add(metadata);
});
private async Task<Option<MovieMetadata>> LoadMetadata(Movie mediaItem, string nfoFileName)
{
ErsatzTV.Core/Metadata/LocalStatisticsProvider.cs
+4 -1
View File
@@ -18,13 +18,16 @@ namespace ErsatzTV.Core.Metadata
private readonly ILocalFileSystem _localFileSystem;
private readonly ILogger<LocalStatisticsProvider> _logger;
private readonly IMediaItemRepository _mediaItemRepository;
private readonly IMetadataRepository _metadataRepository;
public LocalStatisticsProvider(
IMediaItemRepository mediaItemRepository,
IMetadataRepository metadataRepository,
ILocalFileSystem localFileSystem,
ILogger<LocalStatisticsProvider> logger)
{
_mediaItemRepository = mediaItemRepository;
_metadataRepository = metadataRepository;
_localFileSystem = localFileSystem;
_logger = logger;
}
@@ -79,7 +82,7 @@ namespace ErsatzTV.Core.Metadata
mediaItemVersion.VideoProfile = version.VideoProfile;
mediaItemVersion.VideoScanKind = version.VideoScanKind;
return await _mediaItemRepository.Update(mediaItem) && durationChange;
return await _metadataRepository.UpdateLocalStatistics(mediaItemVersion) && durationChange;
}
private Task<Either<BaseError, FFprobe>> GetProbeOutput(string ffprobePath, string filePath)
ErsatzTV.Core/Metadata/MovieFolderScanner.cs
+3 -6
View File
@@ -27,9 +27,10 @@ namespace ErsatzTV.Core.Metadata
IMovieRepository movieRepository,
ILocalStatisticsProvider localStatisticsProvider,
ILocalMetadataProvider localMetadataProvider,
IMetadataRepository metadataRepository,
IImageCache imageCache,
ILogger<MovieFolderScanner> logger)
: base(localFileSystem, localStatisticsProvider, imageCache, logger)
: base(localFileSystem, localStatisticsProvider, metadataRepository, imageCache, logger)
{
_localFileSystem = localFileSystem;
_movieRepository = movieRepository;
@@ -74,7 +75,6 @@ namespace ErsatzTV.Core.Metadata
foreach (string file in allFiles.OrderBy(identity))
{
// TODO: optimize dbcontext use here, do we need tracking? can we make partial updates with dapper?
// TODO: figure out how to rebuild playlists
Either<BaseError, Movie> maybeMovie = await _movieRepository
.GetOrAdd(libraryPath, file)
@@ -144,10 +144,7 @@ namespace ErsatzTV.Core.Metadata
async posterFile =>
{
MovieMetadata metadata = movie.MovieMetadata.Head();
if (RefreshArtwork(posterFile, metadata, artworkKind))
{
await _movieRepository.Update(movie);
}
await RefreshArtwork(posterFile, metadata, artworkKind);
});
return movie;
ErsatzTV.Core/Metadata/TelevisionFolderScanner.cs
+5 -19
View File
@@ -26,10 +26,12 @@ namespace ErsatzTV.Core.Metadata
ITelevisionRepository televisionRepository,
ILocalStatisticsProvider localStatisticsProvider,
ILocalMetadataProvider localMetadataProvider,
IMetadataRepository metadataRepository,
IImageCache imageCache,
ILogger<TelevisionFolderScanner> logger) : base(
localFileSystem,
localStatisticsProvider,
metadataRepository,
imageCache,
logger)
{
@@ -224,10 +226,7 @@ namespace ErsatzTV.Core.Metadata
async posterFile =>
{
ShowMetadata metadata = show.ShowMetadata.Head();
if (RefreshArtwork(posterFile, metadata, artworkKind))
{
await _televisionRepository.Update(show);
}
await RefreshArtwork(posterFile, metadata, artworkKind);
});
return show;
@@ -245,18 +244,8 @@ namespace ErsatzTV.Core.Metadata
await LocatePoster(season, seasonFolder).IfSomeAsync(
async posterFile =>
{
season.SeasonMetadata ??= new List<SeasonMetadata>();
if (!season.SeasonMetadata.Any())
{
season.SeasonMetadata.Add(new SeasonMetadata { SeasonId = season.Id });
}
SeasonMetadata metadata = season.SeasonMetadata.Head();
if (RefreshArtwork(posterFile, metadata, ArtworkKind.Poster))
{
await _televisionRepository.Update(season);
}
await RefreshArtwork(posterFile, metadata, ArtworkKind.Poster);
});
return season;
@@ -275,10 +264,7 @@ namespace ErsatzTV.Core.Metadata
async posterFile =>
{
EpisodeMetadata metadata = episode.EpisodeMetadata.Head();
if (RefreshArtwork(posterFile, metadata, ArtworkKind.Thumbnail))
{
await _televisionRepository.Update(episode);
}
await RefreshArtwork(posterFile, metadata, ArtworkKind.Thumbnail);
});
return episode;
ErsatzTV.Core/Plex/PlexMovieLibraryScanner.cs
+3 -1
View File
@@ -97,7 +97,7 @@ namespace ErsatzTV.Core.Plex
existingVersion.VideoScanKind = mediaVersion.VideoScanKind;
existingVersion.DateUpdated = incomingVersion.DateUpdated;
await _metadataRepository.UpdateStatistics(existingVersion);
await _metadataRepository.UpdatePlexStatistics(existingVersion);
},
_ => Task.CompletedTask);
}
@@ -127,6 +127,8 @@ namespace ErsatzTV.Core.Plex
existingMetadata.Genres.Add(genre);
await _movieRepository.AddGenre(existingMetadata, genre);
}
// TODO: update other metadata?
}
return existing;
ErsatzTV.Core/Plex/PlexTelevisionLibraryScanner.cs
+1 -1
View File
@@ -257,7 +257,7 @@ namespace ErsatzTV.Core.Plex
existingVersion.VideoScanKind = mediaVersion.VideoScanKind;
existingVersion.DateUpdated = incomingVersion.DateUpdated;
await _metadataRepository.UpdateStatistics(existingVersion);
await _metadataRepository.UpdatePlexStatistics(existingVersion);
},
_ => Task.CompletedTask);
}