timothy/ersatztv
1
0
Fork 0
Code Issues 51 Pull Requests 5 Actions Packages Projects Releases 9 Wiki Activity

scanning and poster improvements (#24)

Browse Source 
* first pass at refresh-all-metadata by source

* add version to startup logs

* lock media source during refresh

* fix local media source "name" in collection editor

* optimize scanning so playouts only rebuild when necessary

* support more poster file types

* more scanning improvements; check for missing posters during scans
This commit is contained in:
Jason Dove
2021-02-14 17:04:50 +00:00
committed by GitHub
parent 2c9d4d796a
commit 1aac2f13c9
28 changed files with 378 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ErsatzTV.Core/Metadata/LocalFileSystem.cs
+64
View File
@@ -0,0 +1,64 @@
using System;
using System.IO;
using System.Linq;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Interfaces.Metadata;
using LanguageExt;
using static LanguageExt.Prelude;
namespace ErsatzTV.Core.Metadata
{
public class LocalFileSystem : ILocalFileSystem
{
public bool IsMediaSourceAccessible(LocalMediaSource localMediaSource) =>
Directory.Exists(localMediaSource.Folder);
public Seq<string> FindRelevantVideos(LocalMediaSource localMediaSource)
{
Seq<string> allDirectories = Directory
.GetDirectories(localMediaSource.Folder, "*", SearchOption.AllDirectories)
.ToSeq()
.Add(localMediaSource.Folder);
// remove any directories with an .etvignore file locally, or in any parent directory
Seq<string> excluded = allDirectories.Filter(ShouldExcludeDirectory);
Seq<string> relevantDirectories = allDirectories
.Filter(d => !excluded.Any(d.StartsWith))
.Filter(d => localMediaSource.MediaType == MediaType.Other || !IsExtrasFolder(d));
return relevantDirectories
.Collect(d => Directory.GetFiles(d, "*", SearchOption.TopDirectoryOnly))
.Filter(file => KnownExtensions.Contains(Path.GetExtension(file)))
.OrderBy(identity)
.ToSeq();
}
public bool ShouldRefreshMetadata(LocalMediaSource localMediaSource, MediaItem mediaItem)
{
DateTime lastWrite = File.GetLastWriteTimeUtc(mediaItem.Path);
bool modified = lastWrite > mediaItem.LastWriteTime.IfNone(DateTime.MinValue);
return modified // media item has been modified
|| mediaItem.Metadata == null // media item has no metadata
|| mediaItem.Metadata.MediaType != localMediaSource.MediaType; // media item is typed incorrectly
}
public bool ShouldRefreshPoster(MediaItem mediaItem) =>
string.IsNullOrWhiteSpace(mediaItem.Poster);
private static bool ShouldExcludeDirectory(string path) => File.Exists(Path.Combine(path, ".etvignore"));
// see https://support.emby.media/support/solutions/articles/44001159102-movie-naming
private static bool IsExtrasFolder(string path) =>
ExtraFolderNames.Contains(Path.GetFileName(path)?.ToLowerInvariant());
// @formatter:off
private static readonly Seq<string> KnownExtensions = Seq(
".mpg", ".mp2", ".mpeg", ".mpe", ".mpv", ".ogg", ".mp4",
".m4p", ".m4v", ".avi", ".wmv", ".mov", ".mkv", ".ts");
private static readonly Seq<string> ExtraFolderNames = Seq(
"extras", "specials", "shorts", "scenes", "featurettes",
"behind the scenes", "deleted scenes", "interviews", "trailers");
// @formatter:on
}
}
ErsatzTV.Core/Metadata/LocalMediaScanner.cs
+86 -62
View File
@@ -1,5 +1,4 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
@@ -15,6 +14,7 @@ namespace ErsatzTV.Core.Metadata
{
public class LocalMediaScanner : ILocalMediaScanner
{
private readonly ILocalFileSystem _localFileSystem;
private readonly ILocalMetadataProvider _localMetadataProvider;
private readonly ILocalPosterProvider _localPosterProvider;
private readonly ILocalStatisticsProvider _localStatisticsProvider;
@@ -32,7 +32,8 @@ namespace ErsatzTV.Core.Metadata
ILocalPosterProvider localPosterProvider,
ISmartCollectionBuilder smartCollectionBuilder,
IPlayoutBuilder playoutBuilder,
ILogger<LocalMediaScanner> logger)
ILogger<LocalMediaScanner> logger,
ILocalFileSystem localFileSystem)
{
_mediaItemRepository = mediaItemRepository;
_playoutRepository = playoutRepository;
@@ -42,78 +43,45 @@ namespace ErsatzTV.Core.Metadata
_smartCollectionBuilder = smartCollectionBuilder;
_playoutBuilder = playoutBuilder;
_logger = logger;
_localFileSystem = localFileSystem;
}
public async Task<Unit> ScanLocalMediaSource(LocalMediaSource localMediaSource, string ffprobePath)
public async Task<Unit> ScanLocalMediaSource(
LocalMediaSource localMediaSource,
string ffprobePath,
ScanningMode scanningMode)
{
if (!Directory.Exists(localMediaSource.Folder))
if (!_localFileSystem.IsMediaSourceAccessible(localMediaSource))
{
_logger.LogWarning(
"Media source folder {Folder} does not exist; skipping scan",
"Media source folder {Folder} does not exist or is inaccessible; skipping scan",
localMediaSource.Folder);
return Unit.Default;
return unit;
}
List<MediaItem> knownMediaItems = await _mediaItemRepository.GetAllByMediaSourceId(localMediaSource.Id);
var modifiedPlayoutIds = new List<int>();
// remove files that no longer exist
// add new files
// refresh metadata for any files where it is missing
var knownExtensions = new List<string>
{
".mpg", ".mp2", ".mpeg", ".mpe", ".mpv", ".ogg", ".mp4", ".m4p", ".m4v",
".avi", ".wmv", ".mov", ".mkv", ".ts"
};
Seq<string> allDirectories = Directory
.GetDirectories(localMediaSource.Folder, "*", SearchOption.AllDirectories)
.ToSeq().Add(localMediaSource.Folder);
// remove any directories with an .etvignore file locally, or in any parent directory
Seq<string> excluded = allDirectories.Filter(ShouldExcludeDirectory);
Seq<string> relevantDirectories = allDirectories
.Filter(d => !excluded.Any(d.StartsWith));
var allFiles = relevantDirectories
.Collect(d => Directory.GetFiles(d, "*", SearchOption.TopDirectoryOnly))
.Filter(file => knownExtensions.Contains(Path.GetExtension(file)))
.OrderBy(identity)
.ToSeq();
Seq<string> allFiles = _localFileSystem.FindRelevantVideos(localMediaSource);
// check if the media item exists
(Seq<string> newFiles, Seq<MediaItem> existingMediaItems) = allFiles.Map(
s => Optional(knownMediaItems.Find(i => i.Path == s)).ToEither(s))
.Partition();
// TODO: flag as missing? delete after some period of time?
var removedMediaItems = knownMediaItems.Filter(i => !allFiles.Contains(i.Path)).ToSeq();
modifiedPlayoutIds.AddRange(await _playoutRepository.GetPlayoutIdsForMediaItems(removedMediaItems));
foreach (MediaItem mediaItem in removedMediaItems)
{
_logger.LogDebug("Removing missing local media item {MediaItem}", mediaItem.Path);
await _mediaItemRepository.Delete(mediaItem.Id);
}
// remove media items that no longer exist
var missingMediaItems = knownMediaItems.Filter(i => !allFiles.Contains(i.Path)).ToSeq();
await RemoveMissingItems(missingMediaItems);
modifiedPlayoutIds.AddRange(await _playoutRepository.GetPlayoutIdsForMediaItems(missingMediaItems));
// if exists, check if the file was modified
// also, try to re-categorize incorrect media types by refreshing metadata
Seq<MediaItem> modifiedMediaItems = existingMediaItems.Filter(
mediaItem =>
{
DateTime lastWrite = File.GetLastWriteTimeUtc(mediaItem.Path);
bool modified = lastWrite > mediaItem.LastWriteTime.IfNone(DateTime.MinValue);
return modified || mediaItem.Metadata == null ||
mediaItem.Metadata.MediaType != localMediaSource.MediaType;
});
Seq<MediaItem> staleMetadataMediaItems = scanningMode == ScanningMode.RescanAll
? existingMediaItems
: existingMediaItems.Filter(i => _localFileSystem.ShouldRefreshMetadata(localMediaSource, i));
Seq<MediaItem> modifiedMediaItems = await RefreshMetadataForItems(ffprobePath, staleMetadataMediaItems);
modifiedPlayoutIds.AddRange(await _playoutRepository.GetPlayoutIdsForMediaItems(modifiedMediaItems));
foreach (MediaItem mediaItem in modifiedMediaItems)
{
_logger.LogDebug("Refreshing metadata for media item {MediaItem}", mediaItem.Path);
await RefreshMetadata(mediaItem, ffprobePath);
}
// if new, add and store mtime, refresh metadata
var addedMediaItems = new Seq<MediaItem>();
var addedMediaItems = new List<MediaItem>();
foreach (string path in newFiles)
{
_logger.LogDebug("Adding new media item {MediaItem}", path);
@@ -129,7 +97,12 @@ namespace ErsatzTV.Core.Metadata
addedMediaItems.Add(mediaItem);
}
modifiedPlayoutIds.AddRange(await _playoutRepository.GetPlayoutIdsForMediaItems(addedMediaItems));
modifiedPlayoutIds.AddRange(await _playoutRepository.GetPlayoutIdsForMediaItems(addedMediaItems.ToSeq()));
Seq<MediaItem> stalePosterMediaItems = existingMediaItems
.Filter(_localFileSystem.ShouldRefreshPoster)
.Concat(addedMediaItems);
await RefreshPosterForItems(stalePosterMediaItems);
foreach (int playoutId in modifiedPlayoutIds.Distinct())
{
@@ -143,17 +116,68 @@ namespace ErsatzTV.Core.Metadata
Task.CompletedTask);
}
return Unit.Default;
return unit;
}
private async Task RefreshMetadata(MediaItem mediaItem, string ffprobePath)
private async Task<Seq<MediaItem>> RefreshMetadataForItems(
string ffprobePath,
Seq<MediaItem> staleMetadataMediaItems)
{
await _localStatisticsProvider.RefreshStatistics(ffprobePath, mediaItem);
await _localMetadataProvider.RefreshMetadata(mediaItem);
await _localPosterProvider.RefreshPoster(mediaItem);
await _smartCollectionBuilder.RefreshSmartCollections(mediaItem);
var modifiedMediaItems = new List<MediaItem>();
foreach (MediaItem mediaItem in staleMetadataMediaItems)
{
_logger.LogDebug("Refreshing metadata for media item {MediaItem}", mediaItem.Path);
if (await RefreshMetadata(mediaItem, ffprobePath))
{
// only queue playout rebuilds for media items
// where the duration or collections have changed
modifiedMediaItems.Add(mediaItem);
}
}
return modifiedMediaItems.ToSeq();
}
private static bool ShouldExcludeDirectory(string path) => File.Exists(Path.Combine(path, ".etvignore"));
private async Task RefreshPosterForItems(Seq<MediaItem> stalePosterMediaItems)
{
(Seq<MediaItem> movies, Seq<MediaItem> episodes) = stalePosterMediaItems
.Map(i => Optional(i).Filter(i2 => i2.Metadata?.MediaType == MediaType.TvShow).ToEither(i))
.Partition();
// there's a 1:1 movie:poster, so refresh all
foreach (MediaItem movie in movies)
{
_logger.LogDebug("Refreshing poster for media item {MediaItem}", movie.Path);
await _localPosterProvider.RefreshPoster(movie);
}
// we currently have 1 poster per series, so pick the first from each group
IEnumerable<MediaItem> episodesToRefresh = episodes.GroupBy(e => e.Metadata.Title)
.SelectMany(g => (Option<MediaItem>) g.FirstOrDefault());
foreach (MediaItem episode in episodesToRefresh)
{
_logger.LogDebug("Refreshing poster for media item {MediaItem}", episode.Path);
await _localPosterProvider.RefreshPoster(episode);
}
}
private async Task RemoveMissingItems(Seq<MediaItem> removedMediaItems)
{
// TODO: flag as missing? delete after some period of time?
foreach (MediaItem mediaItem in removedMediaItems)
{
_logger.LogDebug("Removing missing local media item {MediaItem}", mediaItem.Path);
await _mediaItemRepository.Delete(mediaItem.Id);
}
}
private async Task<bool> RefreshMetadata(MediaItem mediaItem, string ffprobePath)
{
bool durationChange = await _localStatisticsProvider.RefreshStatistics(ffprobePath, mediaItem);
await _localMetadataProvider.RefreshMetadata(mediaItem);
bool collectionChange = await _smartCollectionBuilder.RefreshSmartCollections(mediaItem);
return durationChange || collectionChange;
}
}
}
ErsatzTV.Core/Metadata/LocalPosterProvider.cs
+7 -4
View File
@@ -1,4 +1,5 @@
using System.IO;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using ErsatzTV.Core.Domain;
@@ -13,6 +14,8 @@ namespace ErsatzTV.Core.Metadata
{
public class LocalPosterProvider : ILocalPosterProvider
{
private static readonly string[] SupportedExtensions = { "jpg", "jpeg", "png", "gif", "tbn" };
private readonly IImageCache _imageCache;
private readonly ILogger<LocalPosterProvider> _logger;
private readonly IMediaItemRepository _mediaItemRepository;
@@ -46,8 +49,8 @@ namespace ErsatzTV.Core.Metadata
string folder = Path.GetDirectoryName(mediaItem.Path);
if (folder != null)
{
string[] possiblePaths =
{ "poster.jpg", Path.GetFileNameWithoutExtension(mediaItem.Path) + "-poster.jpg" };
IEnumerable<string> possiblePaths = SupportedExtensions.Collect(
e => new[] { $"poster.{e}", Path.GetFileNameWithoutExtension(mediaItem.Path) + $"-poster.{e}" });
Option<string> maybePoster =
possiblePaths.Map(p => Path.Combine(folder, p)).FirstOrDefault(File.Exists);
return maybePoster;
@@ -61,7 +64,7 @@ namespace ErsatzTV.Core.Metadata
string folder = Directory.GetParent(Path.GetDirectoryName(mediaItem.Path) ?? string.Empty)?.FullName;
if (folder != null)
{
string[] possiblePaths = { "poster.jpg" };
IEnumerable<string> possiblePaths = SupportedExtensions.Collect(e => new[] { $"poster.{e}" });
Option<string> maybePoster =
possiblePaths.Map(p => Path.Combine(folder, p)).FirstOrDefault(File.Exists);
return maybePoster;
ErsatzTV.Core/Metadata/LocalStatisticsProvider.cs
+7 -4
View File
@@ -26,21 +26,22 @@ namespace ErsatzTV.Core.Metadata
_logger = logger;
}
public async Task RefreshStatistics(string ffprobePath, MediaItem mediaItem)
public async Task<bool> RefreshStatistics(string ffprobePath, MediaItem mediaItem)
{
try
{
FFprobe ffprobe = await GetProbeOutput(ffprobePath, mediaItem);
MediaMetadata metadata = ProjectToMediaMetadata(ffprobe);
await ApplyStatisticsUpdate(mediaItem, metadata);
return await ApplyStatisticsUpdate(mediaItem, metadata);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to refresh statistics for media item at {Path}", mediaItem.Path);
return false;
}
}
private async Task ApplyStatisticsUpdate(
private async Task<bool> ApplyStatisticsUpdate(
MediaItem mediaItem,
MediaMetadata metadata)
{
@@ -49,6 +50,8 @@ namespace ErsatzTV.Core.Metadata
mediaItem.Metadata = new MediaMetadata();
}
bool durationChange = mediaItem.Metadata.Duration != metadata.Duration;
mediaItem.Metadata.Duration = metadata.Duration;
mediaItem.Metadata.AudioCodec = metadata.AudioCodec;
mediaItem.Metadata.SampleAspectRatio = metadata.SampleAspectRatio;
@@ -58,7 +61,7 @@ namespace ErsatzTV.Core.Metadata
mediaItem.Metadata.VideoCodec = metadata.VideoCodec;
mediaItem.Metadata.VideoScanType = metadata.VideoScanType;
await _mediaItemRepository.Update(mediaItem);
return await _mediaItemRepository.Update(mediaItem) && durationChange;
}
private Task<FFprobe> GetProbeOutput(string ffprobePath, MediaItem mediaItem)
ErsatzTV.Core/Metadata/ScanningMode.cs
+8
View File
@@ -0,0 +1,8 @@
namespace ErsatzTV.Core.Metadata
{
public enum ScanningMode
{
Default = 0,
RescanAll = 1
}
}
ErsatzTV.Core/Metadata/SmartCollectionBuilder.cs
+6 -2
View File
@@ -16,12 +16,16 @@ namespace ErsatzTV.Core.Metadata
public SmartCollectionBuilder(IMediaCollectionRepository mediaCollectionRepository) =>
_mediaCollectionRepository = mediaCollectionRepository;
public async Task RefreshSmartCollections(MediaItem mediaItem)
public async Task<bool> RefreshSmartCollections(MediaItem mediaItem)
{
var results = new List<bool>();
foreach (TelevisionMediaCollection collection in GetTelevisionCollections(mediaItem))
{
await _mediaCollectionRepository.InsertOrIgnore(collection);
results.Add(await _mediaCollectionRepository.InsertOrIgnore(collection));
}
return results.Any(identity);
}
private IEnumerable<TelevisionMediaCollection> GetTelevisionCollections(MediaItem mediaItem)