timothy/ersatztv
1
0
Fork 0
Code Issues 51 Pull Requests 5 Actions Packages Projects Releases 9 Wiki Activity

metadata improvements (#95)

Browse Source 
* fix episode fallback metadata processing, fix show fallback metadata year parsing

* fix sort title for "a" and "an"

* add and index studio metadata

* minimize circular logging with search index errors

* update plex movie sort titles as needed

* properly escape search links

* force refreshing all movie/show metadata
This commit is contained in:
Jason Dove
2021-03-21 18:43:08 +00:00
committed by GitHub
parent 00fdc272e9
commit 4c70d61d48
35 changed files with 3989 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ErsatzTV.Core/Domain/Metadata/Metadata.cs
+1
View File
@@ -17,5 +17,6 @@ namespace ErsatzTV.Core.Domain
public List<Artwork> Artwork { get; set; }
public List<Genre> Genres { get; set; }
public List<Tag> Tags { get; set; }
public List<Studio> Studios { get; set; }
}
}
ErsatzTV.Core/Domain/Metadata/Studio.cs
+8
View File
@@ -0,0 +1,8 @@
namespace ErsatzTV.Core.Domain
{
public class Studio
{
public int Id { get; set; }
public string Name { get; set; }
}
}
ErsatzTV.Core/Interfaces/Repositories/IMetadataRepository.cs
+2
View File
@@ -7,6 +7,8 @@ namespace ErsatzTV.Core.Interfaces.Repositories
public interface IMetadataRepository
{
Task<bool> RemoveGenre(Genre genre);
Task<bool> RemoveTag(Tag tag);
Task<bool> RemoveStudio(Studio studio);
Task<bool> Update(Domain.Metadata metadata);
Task<bool> Add(Domain.Metadata metadata);
Task<bool> UpdateLocalStatistics(MediaVersion mediaVersion);
ErsatzTV.Core/Interfaces/Repositories/IMovieRepository.cs
+3
View File
@@ -18,6 +18,9 @@ namespace ErsatzTV.Core.Interfaces.Repositories
Task<IEnumerable<string>> FindMoviePaths(LibraryPath libraryPath);
Task<List<int>> DeleteByPath(LibraryPath libraryPath, string path);
Task<bool> AddGenre(MovieMetadata metadata, Genre genre);
Task<bool> AddTag(MovieMetadata metadata, Tag tag);
Task<bool> AddStudio(MovieMetadata metadata, Studio studio);
Task<List<int>> RemoveMissingPlexMovies(PlexLibrary library, List<string> movieKeys);
Task<bool> UpdateSortTitle(MovieMetadata movieMetadata);
}
}
ErsatzTV.Core/Interfaces/Repositories/ITelevisionRepository.cs
+2
View File
@@ -40,6 +40,8 @@ namespace ErsatzTV.Core.Interfaces.Repositories
Task<Either<BaseError, PlexSeason>> GetOrAddPlexSeason(PlexLibrary library, PlexSeason item);
Task<Either<BaseError, PlexEpisode>> GetOrAddPlexEpisode(PlexLibrary library, PlexEpisode item);
Task<bool> AddGenre(ShowMetadata metadata, Genre genre);
Task<bool> AddTag(ShowMetadata metadata, Tag tag);
Task<bool> AddStudio(ShowMetadata metadata, Studio studio);
Task<List<int>> RemoveMissingPlexShows(PlexLibrary library, List<string> showKeys);
Task<Unit> RemoveMissingPlexSeasons(string showKey, List<string> seasonKeys);
Task<Unit> RemoveMissingPlexEpisodes(string seasonKey, List<string> episodeKeys);
ErsatzTV.Core/Metadata/FallbackMetadataProvider.cs
+15 -1
View File
@@ -23,7 +23,7 @@ namespace ErsatzTV.Core.Metadata
string path = episode.MediaVersions.Head().MediaFiles.Head().Path;
string fileName = Path.GetFileName(path);
var metadata = new EpisodeMetadata
{ MetadataKind = MetadataKind.Fallback, Title = fileName ?? path };
{ MetadataKind = MetadataKind.Fallback, Title = fileName ?? path, DateAdded = DateTime.UtcNow };
return fileName != null ? GetEpisodeMetadata(fileName, metadata) : Tuple(metadata, 0);
}
@@ -48,6 +48,16 @@ namespace ErsatzTV.Core.Metadata
return title.Substring(4);
}
if (title.StartsWith("a ", StringComparison.OrdinalIgnoreCase))
{
return title.Substring(2);
}
if (title.StartsWith("an ", StringComparison.OrdinalIgnoreCase))
{
return title.Substring(3);
}
if (title.StartsWith("Æ"))
{
return title.Replace("Æ", "E");
@@ -65,6 +75,7 @@ namespace ErsatzTV.Core.Metadata
if (match.Success)
{
metadata.Title = match.Groups[1].Value;
metadata.DateUpdated = DateTime.UtcNow;
return Tuple(metadata, int.Parse(match.Groups[3].Value));
}
}
@@ -89,6 +100,8 @@ namespace ErsatzTV.Core.Metadata
metadata.ReleaseDate = new DateTime(int.Parse(match.Groups[2].Value), 1, 1);
metadata.Genres = new List<Genre>();
metadata.Tags = new List<Tag>();
metadata.Studios = new List<Studio>();
metadata.DateUpdated = DateTime.UtcNow;
}
}
catch (Exception)
@@ -110,6 +123,7 @@ namespace ErsatzTV.Core.Metadata
metadata.Title = match.Groups[1].Value;
metadata.Year = int.Parse(match.Groups[2].Value);
metadata.ReleaseDate = new DateTime(int.Parse(match.Groups[2].Value), 1, 1);
metadata.DateUpdated = DateTime.UtcNow;
}
}
catch (Exception)
ErsatzTV.Core/Metadata/LocalMetadataProvider.cs
+85 -17
View File
@@ -23,16 +23,19 @@ namespace ErsatzTV.Core.Metadata
private readonly ILogger<LocalMetadataProvider> _logger;
private readonly IMetadataRepository _metadataRepository;
private readonly IMovieRepository _movieRepository;
private readonly ITelevisionRepository _televisionRepository;
public LocalMetadataProvider(
IMetadataRepository metadataRepository,
IMovieRepository movieRepository,
ITelevisionRepository televisionRepository,
IFallbackMetadataProvider fallbackMetadataProvider,
ILocalFileSystem localFileSystem,
ILogger<LocalMetadataProvider> logger)
{
_metadataRepository = metadataRepository;
_movieRepository = movieRepository;
_televisionRepository = televisionRepository;
_fallbackMetadataProvider = fallbackMetadataProvider;
_localFileSystem = localFileSystem;
@@ -138,7 +141,7 @@ namespace ErsatzTV.Core.Metadata
private Task<bool> ApplyMetadataUpdate(Movie movie, MovieMetadata metadata) =>
Optional(movie.MovieMetadata).Flatten().HeadOrNone().Match(
existing =>
async existing =>
{
existing.Outline = metadata.Outline;
existing.Plot = metadata.Plot;
@@ -158,29 +161,49 @@ namespace ErsatzTV.Core.Metadata
.ToList())
{
existing.Genres.Remove(genre);
await _metadataRepository.RemoveGenre(genre);
}
foreach (Genre genre in metadata.Genres.Filter(g => existing.Genres.All(g2 => g2.Name != g.Name))
.ToList())
{
existing.Genres.Add(genre);
await _movieRepository.AddGenre(existing, genre);
}
foreach (Tag tag in existing.Tags.Filter(t => metadata.Tags.All(t2 => t2.Name != t.Name))
.ToList())
{
existing.Tags.Remove(tag);
await _metadataRepository.RemoveTag(tag);
}
foreach (Tag tag in metadata.Tags.Filter(t => existing.Tags.All(t2 => t2.Name != t.Name))
.ToList())
{
existing.Tags.Add(tag);
await _movieRepository.AddTag(existing, tag);
}
return _metadataRepository.Update(existing);
foreach (Studio studio in existing.Studios
.Filter(s => metadata.Studios.All(s2 => s2.Name != s.Name))
.ToList())
{
existing.Studios.Remove(studio);
await _metadataRepository.RemoveStudio(studio);
}
foreach (Studio studio in metadata.Studios
.Filter(s => existing.Studios.All(s2 => s2.Name != s.Name))
.ToList())
{
existing.Studios.Add(studio);
await _movieRepository.AddStudio(existing, studio);
}
return await _metadataRepository.Update(existing);
},
() =>
async () =>
{
metadata.SortTitle = string.IsNullOrWhiteSpace(metadata.SortTitle)
? _fallbackMetadataProvider.GetSortTitle(metadata.Title)
@@ -188,12 +211,12 @@ namespace ErsatzTV.Core.Metadata
metadata.MovieId = movie.Id;
movie.MovieMetadata = new List<MovieMetadata> { metadata };
return _metadataRepository.Add(metadata);
return await _metadataRepository.Add(metadata);
});
private Task<bool> ApplyMetadataUpdate(Show show, ShowMetadata metadata) =>
Optional(show.ShowMetadata).Flatten().HeadOrNone().Match(
existing =>
async existing =>
{
existing.Outline = metadata.Outline;
existing.Plot = metadata.Plot;
@@ -213,29 +236,49 @@ namespace ErsatzTV.Core.Metadata
.ToList())
{
existing.Genres.Remove(genre);
await _metadataRepository.RemoveGenre(genre);
}
foreach (Genre genre in metadata.Genres.Filter(g => existing.Genres.All(g2 => g2.Name != g.Name))
.ToList())
{
existing.Genres.Add(genre);
await _televisionRepository.AddGenre(existing, genre);
}
foreach (Tag tag in existing.Tags.Filter(t => metadata.Tags.All(t2 => t2.Name != t.Name))
.ToList())
{
existing.Tags.Remove(tag);
await _metadataRepository.RemoveTag(tag);
}
foreach (Tag tag in metadata.Tags.Filter(t => existing.Tags.All(t2 => t2.Name != t.Name))
.ToList())
{
existing.Tags.Add(tag);
await _televisionRepository.AddTag(existing, tag);
}
return _metadataRepository.Update(existing);
foreach (Studio studio in existing.Studios
.Filter(s => metadata.Studios.All(s2 => s2.Name != s.Name))
.ToList())
{
existing.Studios.Remove(studio);
await _metadataRepository.RemoveStudio(studio);
}
foreach (Studio studio in metadata.Studios
.Filter(s => existing.Studios.All(s2 => s2.Name != s.Name))
.ToList())
{
existing.Studios.Add(studio);
await _televisionRepository.AddStudio(existing, studio);
}
return await _metadataRepository.Update(existing);
},
() =>
async () =>
{
metadata.SortTitle = string.IsNullOrWhiteSpace(metadata.SortTitle)
? _fallbackMetadataProvider.GetSortTitle(metadata.Title)
@@ -243,7 +286,7 @@ namespace ErsatzTV.Core.Metadata
metadata.ShowId = show.Id;
show.ShowMetadata = new List<ShowMetadata> { metadata };
return _metadataRepository.Add(metadata);
return await _metadataRepository.Add(metadata);
});
private async Task<Option<MovieMetadata>> LoadMetadata(Movie mediaItem, string nfoFileName)
@@ -294,10 +337,11 @@ namespace ErsatzTV.Core.Metadata
Plot = nfo.Plot,
Outline = nfo.Outline,
Tagline = nfo.Tagline,
Year = nfo.Year,
ReleaseDate = GetAired(nfo.Premiered) ?? new DateTime(nfo.Year, 1, 1),
Year = GetYear(nfo.Year, nfo.Premiered),
ReleaseDate = GetAired(nfo.Year, nfo.Premiered),
Genres = nfo.Genres.Map(g => new Genre { Name = g }).ToList(),
Tags = nfo.Tags.Map(t => new Tag { Name = t }).ToList()
Tags = nfo.Tags.Map(t => new Tag { Name = t }).ToList(),
Studios = nfo.Studios.Map(s => new Studio { Name = s }).ToList()
},
None);
}
@@ -322,7 +366,7 @@ namespace ErsatzTV.Core.Metadata
MetadataKind = MetadataKind.Sidecar,
DateUpdated = File.GetLastWriteTimeUtc(nfoFileName),
Title = nfo.Title,
ReleaseDate = GetAired(nfo.Aired),
ReleaseDate = GetAired(0, nfo.Aired),
Plot = nfo.Plot
};
return Tuple(metadata, nfo.Episode);
@@ -354,7 +398,8 @@ namespace ErsatzTV.Core.Metadata
Outline = nfo.Outline,
Tagline = nfo.Tagline,
Genres = nfo.Genres.Map(g => new Genre { Name = g }).ToList(),
Tags = nfo.Tags.Map(t => new Tag { Name = t }).ToList()
Tags = nfo.Tags.Map(t => new Tag { Name = t }).ToList(),
Studios = nfo.Studios.Map(s => new Studio { Name = s }).ToList()
},
None);
}
@@ -365,21 +410,38 @@ namespace ErsatzTV.Core.Metadata
}
}
private static DateTime? GetAired(string aired)
private static int? GetYear(int year, string premiered)
{
if (string.IsNullOrWhiteSpace(aired))
if (year > 1000)
{
return year;
}
if (string.IsNullOrWhiteSpace(premiered))
{
return null;
}
if (DateTime.TryParse(aired, out DateTime parsed))
if (DateTime.TryParse(premiered, out DateTime parsed))
{
return parsed;
return parsed.Year;
}
return null;
}
private static DateTime? GetAired(int year, string aired)
{
DateTime? fallback = year > 1000 ? new DateTime(year, 1, 1) : null;
if (string.IsNullOrWhiteSpace(aired))
{
return fallback;
}
return DateTime.TryParse(aired, out DateTime parsed) ? parsed : fallback;
}
[XmlRoot("movie")]
public class MovieNfo
{
@@ -409,6 +471,9 @@ namespace ErsatzTV.Core.Metadata
[XmlElement("tag")]
public List<string> Tags { get; set; }
[XmlElement("studio")]
public List<string> Studios { get; set; }
}
[XmlRoot("tvshow")]
@@ -437,6 +502,9 @@ namespace ErsatzTV.Core.Metadata
[XmlElement("tag")]
public List<string> Tags { get; set; }
[XmlElement("studio")]
public List<string> Studios { get; set; }
}
[XmlRoot("episodedetails")]
ErsatzTV.Core/Metadata/MovieFolderScanner.cs
+2 -2
View File
@@ -159,7 +159,7 @@ namespace ErsatzTV.Core.Metadata
}
catch (Exception ex)
{
return BaseError.New(ex.Message);
return BaseError.New(ex.ToString());
}
}
@@ -181,7 +181,7 @@ namespace ErsatzTV.Core.Metadata
}
catch (Exception ex)
{
return BaseError.New(ex.Message);
return BaseError.New(ex.ToString());
}
}
ErsatzTV.Core/Metadata/TelevisionFolderScanner.cs
+10 -6
View File
@@ -200,7 +200,7 @@ namespace ErsatzTV.Core.Metadata
}
catch (Exception ex)
{
return BaseError.New(ex.Message);
return BaseError.New(ex.ToString());
}
}
@@ -225,7 +225,11 @@ namespace ErsatzTV.Core.Metadata
},
async () =>
{
if (!Optional(episode.EpisodeMetadata).Flatten().Any())
bool shouldUpdate = Optional(episode.EpisodeMetadata).Flatten().HeadOrNone().Match(
m => m.DateUpdated == DateTime.MinValue,
true);
if (shouldUpdate)
{
string path = episode.MediaVersions.Head().MediaFiles.Head().Path;
_logger.LogDebug("Refreshing {Attribute} for {Path}", "Fallback Metadata", path);
@@ -237,7 +241,7 @@ namespace ErsatzTV.Core.Metadata
}
catch (Exception ex)
{
return BaseError.New(ex.Message);
return BaseError.New(ex.ToString());
}
}
@@ -260,7 +264,7 @@ namespace ErsatzTV.Core.Metadata
}
catch (Exception ex)
{
return BaseError.New(ex.Message);
return BaseError.New(ex.ToString());
}
}
@@ -279,7 +283,7 @@ namespace ErsatzTV.Core.Metadata
}
catch (Exception ex)
{
return BaseError.New(ex.Message);
return BaseError.New(ex.ToString());
}
}
@@ -298,7 +302,7 @@ namespace ErsatzTV.Core.Metadata
}
catch (Exception ex)
{
return BaseError.New(ex.Message);
return BaseError.New(ex.ToString());
}
}
ErsatzTV.Core/Plex/PlexMovieLibraryScanner.cs
+31
View File
@@ -158,6 +158,37 @@ namespace ErsatzTV.Core.Plex
}
}
foreach (Studio studio in existingMetadata.Studios
.Filter(s => incomingMetadata.Studios.All(s2 => s2.Name != s.Name))
.ToList())
{
existingMetadata.Studios.Remove(studio);
if (await _metadataRepository.RemoveStudio(studio))
{
result.IsUpdated = true;
}
}
foreach (Studio studio in incomingMetadata.Studios
.Filter(s => existingMetadata.Studios.All(s2 => s2.Name != s.Name))
.ToList())
{
existingMetadata.Studios.Add(studio);
if (await _movieRepository.AddStudio(existingMetadata, studio))
{
result.IsUpdated = true;
}
}
if (incomingMetadata.SortTitle != existingMetadata.SortTitle)
{
existingMetadata.SortTitle = incomingMetadata.SortTitle;
if (await _movieRepository.UpdateSortTitle(existingMetadata))
{
result.IsUpdated = true;
}
}
// TODO: update other metadata?
}
ErsatzTV.Core/Plex/PlexTelevisionLibraryScanner.cs
+22
View File
@@ -131,6 +131,28 @@ namespace ErsatzTV.Core.Plex
result.IsUpdated = true;
}
}
foreach (Studio studio in existingMetadata.Studios
.Filter(s => incomingMetadata.Studios.All(s2 => s2.Name != s.Name))
.ToList())
{
existingMetadata.Studios.Remove(studio);
if (await _metadataRepository.RemoveStudio(studio))
{
result.IsUpdated = true;
}
}
foreach (Studio studio in incomingMetadata.Studios
.Filter(s => existingMetadata.Studios.All(s2 => s2.Name != s.Name))
.ToList())
{
existingMetadata.Studios.Add(studio);
if (await _televisionRepository.AddStudio(existingMetadata, studio))
{
result.IsUpdated = true;
}
}
}
return result;