timothy/ersatztv
1
0
Fork 0
Code Issues 51 Pull Requests 5 Actions Packages Projects Releases 9 Wiki Activity

metadata improvements (#95)

Browse Source 
* fix episode fallback metadata processing, fix show fallback metadata year parsing

* fix sort title for "a" and "an"

* add and index studio metadata

* minimize circular logging with search index errors

* update plex movie sort titles as needed

* properly escape search links

* force refreshing all movie/show metadata
This commit is contained in:
Jason Dove
2021-03-21 18:43:08 +00:00
committed by GitHub
parent 00fdc272e9
commit 4c70d61d48
35 changed files with 3989 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ErsatzTV.Infrastructure/Data/Configurations/Metadata/MovieMetadataConfiguration.cs
+4
View File
@@ -21,6 +21,10 @@ namespace ErsatzTV.Infrastructure.Data.Configurations
builder.HasMany(mm => mm.Tags)
.WithOne()
.OnDelete(DeleteBehavior.Cascade);
builder.HasMany(mm => mm.Studios)
.WithOne()
.OnDelete(DeleteBehavior.Cascade);
}
}
}
ErsatzTV.Infrastructure/Data/Configurations/Metadata/ShowMetadataConfiguration.cs
+6 -2
View File
@@ -14,11 +14,15 @@ namespace ErsatzTV.Infrastructure.Data.Configurations
.WithOne()
.OnDelete(DeleteBehavior.Cascade);
builder.HasMany(mm => mm.Genres)
builder.HasMany(sm => sm.Genres)
.WithOne()
.OnDelete(DeleteBehavior.Cascade);
builder.HasMany(mm => mm.Tags)
builder.HasMany(sm => sm.Tags)
.WithOne()
.OnDelete(DeleteBehavior.Cascade);
builder.HasMany(sm => sm.Studios)
.WithOne()
.OnDelete(DeleteBehavior.Cascade);
}
ErsatzTV.Infrastructure/Data/Configurations/Metadata/StudioConfiguration.cs
+11
View File
@@ -0,0 +1,11 @@
using ErsatzTV.Core.Domain;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Metadata.Builders;
namespace ErsatzTV.Infrastructure.Data.Configurations
{
public class StudioConfiguration : IEntityTypeConfiguration<Studio>
{
public void Configure(EntityTypeBuilder<Studio> builder) => builder.ToTable("Studio");
}
}
ErsatzTV.Infrastructure/Data/Repositories/MetadataRepository.cs
+8
View File
@@ -112,5 +112,13 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
public Task<bool> RemoveGenre(Genre genre) =>
_dbConnection.ExecuteAsync("DELETE FROM Genre WHERE Id = @GenreId", new { GenreId = genre.Id })
.Map(result => result > 0);
public Task<bool> RemoveTag(Tag tag) =>
_dbConnection.ExecuteAsync("DELETE FROM Tag WHERE Id = @TagId", new { TagId = tag.Id })
.Map(result => result > 0);
public Task<bool> RemoveStudio(Studio studio) =>
_dbConnection.ExecuteAsync("DELETE FROM Studio WHERE Id = @StudioId", new { StudioId = studio.Id })
.Map(result => result > 0);
}
}
ErsatzTV.Infrastructure/Data/Repositories/MovieRepository.cs
+21
View File
@@ -41,6 +41,8 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
.ThenInclude(m => m.Genres)
.Include(m => m.MovieMetadata)
.ThenInclude(m => m.Tags)
.Include(m => m.MovieMetadata)
.ThenInclude(m => m.Studios)
.OrderBy(m => m.Id)
.SingleOrDefaultAsync(m => m.Id == movieId)
.Map(Optional);
@@ -56,6 +58,8 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
.ThenInclude(mm => mm.Genres)
.Include(i => i.MovieMetadata)
.ThenInclude(mm => mm.Tags)
.Include(i => i.MovieMetadata)
.ThenInclude(mm => mm.Studios)
.Include(i => i.LibraryPath)
.ThenInclude(lp => lp.Library)
.Include(i => i.MediaVersions)
@@ -82,6 +86,8 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
.Include(i => i.MovieMetadata)
.ThenInclude(mm => mm.Tags)
.Include(i => i.MovieMetadata)
.ThenInclude(mm => mm.Studios)
.Include(i => i.MovieMetadata)
.ThenInclude(mm => mm.Artwork)
.Include(i => i.MediaVersions)
.ThenInclude(mv => mv.MediaFiles)
@@ -165,6 +171,16 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
"INSERT INTO Genre (Name, MovieMetadataId) VALUES (@Name, @MetadataId)",
new { genre.Name, MetadataId = metadata.Id }).Map(result => result > 0);
public Task<bool> AddTag(MovieMetadata metadata, Tag tag) =>
_dbConnection.ExecuteAsync(
"INSERT INTO Tag (Name, MovieMetadataId) VALUES (@Name, @MetadataId)",
new { tag.Name, MetadataId = metadata.Id }).Map(result => result > 0);
public Task<bool> AddStudio(MovieMetadata metadata, Studio studio) =>
_dbConnection.ExecuteAsync(
"INSERT INTO Studio (Name, MovieMetadataId) VALUES (@Name, @MetadataId)",
new { studio.Name, MetadataId = metadata.Id }).Map(result => result > 0);
public async Task<List<int>> RemoveMissingPlexMovies(PlexLibrary library, List<string> movieKeys)
{
List<int> ids = await _dbConnection.QueryAsync<int>(
@@ -185,6 +201,11 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
return ids;
}
public Task<bool> UpdateSortTitle(MovieMetadata movieMetadata) =>
_dbConnection.ExecuteAsync(
@"UPDATE MovieMetadata SET SortTitle = @SortTitle WHERE Id = @Id",
new { movieMetadata.SortTitle, movieMetadata.Id }).Map(result => result > 0);
private static async Task<Either<BaseError, MediaItemScanResult<Movie>>> AddMovie(
TvContext dbContext,
int libraryPathId,
ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs
+4
View File
@@ -37,10 +37,14 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
.ThenInclude(mm => mm.Genres)
.Include(mi => (mi as Movie).MovieMetadata)
.ThenInclude(mm => mm.Tags)
.Include(mi => (mi as Movie).MovieMetadata)
.ThenInclude(mm => mm.Studios)
.Include(mi => (mi as Show).ShowMetadata)
.ThenInclude(mm => mm.Genres)
.Include(mi => (mi as Show).ShowMetadata)
.ThenInclude(mm => mm.Tags)
.Include(mi => (mi as Show).ShowMetadata)
.ThenInclude(mm => mm.Studios)
.OrderBy(mi => mi.Id)
.SingleOrDefaultAsync(mi => mi.Id == id)
.Map(Optional);
ErsatzTV.Infrastructure/Data/Repositories/TelevisionRepository.cs
+32 -4
View File
@@ -53,6 +53,8 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
.ThenInclude(sm => sm.Genres)
.Include(s => s.ShowMetadata)
.ThenInclude(sm => sm.Tags)
.Include(s => s.ShowMetadata)
.ThenInclude(sm => sm.Studios)
.OrderBy(s => s.Id)
.SingleOrDefaultAsync()
.Map(Optional);
@@ -354,11 +356,13 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
Option<PlexShow> maybeExisting = await dbContext.PlexShows
.AsNoTracking()
.Include(i => i.ShowMetadata)
.ThenInclude(mm => mm.Genres)
.ThenInclude(sm => sm.Genres)
.Include(i => i.ShowMetadata)
.ThenInclude(mm => mm.Tags)
.ThenInclude(sm => sm.Tags)
.Include(i => i.ShowMetadata)
.ThenInclude(mm => mm.Artwork)
.ThenInclude(sm => sm.Studios)
.Include(i => i.ShowMetadata)
.ThenInclude(sm => sm.Artwork)
.Include(i => i.LibraryPath)
.ThenInclude(lp => lp.Library)
.OrderBy(i => i.Key)
@@ -464,9 +468,19 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
public Task<bool> AddGenre(ShowMetadata metadata, Genre genre) =>
_dbConnection.ExecuteAsync(
"INSERT INTO Genre (Name, SeasonMetadataId) VALUES (@Name, @MetadataId)",
"INSERT INTO Genre (Name, ShowMetadataId) VALUES (@Name, @MetadataId)",
new { genre.Name, MetadataId = metadata.Id }).Map(result => result > 0);
public Task<bool> AddTag(ShowMetadata metadata, Tag tag) =>
_dbConnection.ExecuteAsync(
"INSERT INTO Tag (Name, ShowMetadataId) VALUES (@Name, @MetadataId)",
new { tag.Name, MetadataId = metadata.Id }).Map(result => result > 0);
public Task<bool> AddStudio(ShowMetadata metadata, Studio studio) =>
_dbConnection.ExecuteAsync(
"INSERT INTO Studio (Name, ShowMetadataId) VALUES (@Name, @MetadataId)",
new { studio.Name, MetadataId = metadata.Id }).Map(result => result > 0);
public async Task<List<int>> RemoveMissingPlexShows(PlexLibrary library, List<string> showKeys)
{
List<int> ids = await _dbConnection.QueryAsync<int>(
@@ -504,6 +518,9 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
SeasonMetadata = new List<SeasonMetadata>
{
new()
{
DateAdded = DateTime.UtcNow
}
}
};
await dbContext.Seasons.AddAsync(season);
@@ -524,6 +541,11 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
{
try
{
if (dbContext.MediaFiles.Any(mf => mf.Path == path))
{
return BaseError.New("Multi-episode files are not yet supported");
}
var episode = new Episode
{
LibraryPathId = libraryPathId,
@@ -532,6 +554,7 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
{
new()
{
DateAdded = DateTime.UtcNow,
DateUpdated = DateTime.MinValue,
MetadataKind = MetadataKind.Fallback
}
@@ -605,6 +628,11 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
{
try
{
if (dbContext.MediaFiles.Any(mf => mf.Path == item.MediaVersions.Head().MediaFiles.Head().Path))
{
return BaseError.New("Multi-episode files are not yet supported");
}
item.LibraryPathId = library.Paths.Head().Id;
await dbContext.PlexEpisodes.AddAsync(item);
ErsatzTV.Infrastructure/Migrations/20210321162257_Add_Studio.Designer.cs
Generated
+1749
View File
File diff suppressed because it is too large Load Diff
ErsatzTV.Infrastructure/Migrations/20210321162257_Add_Studio.cs
+75
View File
@@ -0,0 +1,75 @@
using Microsoft.EntityFrameworkCore.Migrations;
namespace ErsatzTV.Infrastructure.Migrations
{
public partial class Add_Studio : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
"Studio",
table => new
{
Id = table.Column<int>("INTEGER", nullable: false)
.Annotation("Sqlite:Autoincrement", true),
Name = table.Column<string>("TEXT", nullable: true),
EpisodeMetadataId = table.Column<int>("INTEGER", nullable: true),
MovieMetadataId = table.Column<int>("INTEGER", nullable: true),
SeasonMetadataId = table.Column<int>("INTEGER", nullable: true),
ShowMetadataId = table.Column<int>("INTEGER", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("PK_Studio", x => x.Id);
table.ForeignKey(
"FK_Studio_EpisodeMetadata_EpisodeMetadataId",
x => x.EpisodeMetadataId,
"EpisodeMetadata",
"Id",
onDelete: ReferentialAction.Restrict);
table.ForeignKey(
"FK_Studio_MovieMetadata_MovieMetadataId",
x => x.MovieMetadataId,
"MovieMetadata",
"Id",
onDelete: ReferentialAction.Cascade);
table.ForeignKey(
"FK_Studio_SeasonMetadata_SeasonMetadataId",
x => x.SeasonMetadataId,
"SeasonMetadata",
"Id",
onDelete: ReferentialAction.Restrict);
table.ForeignKey(
"FK_Studio_ShowMetadata_ShowMetadataId",
x => x.ShowMetadataId,
"ShowMetadata",
"Id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex(
"IX_Studio_EpisodeMetadataId",
"Studio",
"EpisodeMetadataId");
migrationBuilder.CreateIndex(
"IX_Studio_MovieMetadataId",
"Studio",
"MovieMetadataId");
migrationBuilder.CreateIndex(
"IX_Studio_SeasonMetadataId",
"Studio",
"SeasonMetadataId");
migrationBuilder.CreateIndex(
"IX_Studio_ShowMetadataId",
"Studio",
"ShowMetadataId");
}
protected override void Down(MigrationBuilder migrationBuilder) =>
migrationBuilder.DropTable(
"Studio");
}
}
ErsatzTV.Infrastructure/Migrations/20210321182241_Reset_MetadataDateUpdated_Studio.Designer.cs
Generated
+1749
View File
File diff suppressed because it is too large Load Diff
ErsatzTV.Infrastructure/Migrations/20210321182241_Reset_MetadataDateUpdated_Studio.cs
+18
View File
@@ -0,0 +1,18 @@
using Microsoft.EntityFrameworkCore.Migrations;
namespace ErsatzTV.Infrastructure.Migrations
{
public partial class Reset_MetadataDateUpdated_Studio : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.Sql(@"UPDATE MovieMetadata SET DateUpdated = '0001-01-01 00:00:00'");
migrationBuilder.Sql(@"UPDATE ShowMetadata SET DateUpdated = '0001-01-01 00:00:00'");
migrationBuilder.Sql(@"UPDATE Library SET LastScan = '0001-01-01 00:00:00'");
}
protected override void Down(MigrationBuilder migrationBuilder)
{
}
}
}
ErsatzTV.Infrastructure/Migrations/TvContextModelSnapshot.cs
+67
View File
@@ -845,6 +845,42 @@ namespace ErsatzTV.Infrastructure.Migrations
b.ToTable("ShowMetadata");
});
modelBuilder.Entity(
"ErsatzTV.Core.Domain.Studio",
b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<int?>("EpisodeMetadataId")
.HasColumnType("INTEGER");
b.Property<int?>("MovieMetadataId")
.HasColumnType("INTEGER");
b.Property<string>("Name")
.HasColumnType("TEXT");
b.Property<int?>("SeasonMetadataId")
.HasColumnType("INTEGER");
b.Property<int?>("ShowMetadataId")
.HasColumnType("INTEGER");
b.HasKey("Id");
b.HasIndex("EpisodeMetadataId");
b.HasIndex("MovieMetadataId");
b.HasIndex("SeasonMetadataId");
b.HasIndex("ShowMetadataId");
b.ToTable("Studio");
});
modelBuilder.Entity(
"ErsatzTV.Core.Domain.Tag",
b =>
@@ -1499,6 +1535,29 @@ namespace ErsatzTV.Infrastructure.Migrations
b.Navigation("Show");
});
modelBuilder.Entity(
"ErsatzTV.Core.Domain.Studio",
b =>
{
b.HasOne("ErsatzTV.Core.Domain.EpisodeMetadata", null)
.WithMany("Studios")
.HasForeignKey("EpisodeMetadataId");
b.HasOne("ErsatzTV.Core.Domain.MovieMetadata", null)
.WithMany("Studios")
.HasForeignKey("MovieMetadataId")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("ErsatzTV.Core.Domain.SeasonMetadata", null)
.WithMany("Studios")
.HasForeignKey("SeasonMetadataId");
b.HasOne("ErsatzTV.Core.Domain.ShowMetadata", null)
.WithMany("Studios")
.HasForeignKey("ShowMetadataId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity(
"ErsatzTV.Core.Domain.Tag",
b =>
@@ -1744,6 +1803,8 @@ namespace ErsatzTV.Infrastructure.Migrations
b.Navigation("Genres");
b.Navigation("Studios");
b.Navigation("Tags");
});
@@ -1765,6 +1826,8 @@ namespace ErsatzTV.Infrastructure.Migrations
b.Navigation("Genres");
b.Navigation("Studios");
b.Navigation("Tags");
});
@@ -1794,6 +1857,8 @@ namespace ErsatzTV.Infrastructure.Migrations
b.Navigation("Genres");
b.Navigation("Studios");
b.Navigation("Tags");
});
@@ -1805,6 +1870,8 @@ namespace ErsatzTV.Infrastructure.Migrations
b.Navigation("Genres");
b.Navigation("Studios");
b.Navigation("Tags");
});
ErsatzTV.Infrastructure/Plex/Models/PlexMetadataResponse.cs
+1
View File
@@ -15,6 +15,7 @@ namespace ErsatzTV.Infrastructure.Plex.Models
public int AddedAt { get; set; }
public int UpdatedAt { get; set; }
public int Index { get; set; }
public string Studio { get; set; }
public List<PlexMediaResponse> Media { get; set; }
public List<PlexGenreResponse> Genre { get; set; }
}
ErsatzTV.Infrastructure/Plex/PlexServerApiClient.cs
+14 -2
View File
@@ -177,9 +177,15 @@ namespace ErsatzTV.Infrastructure.Plex
DateAdded = dateAdded,
DateUpdated = lastWriteTime,
Genres = Optional(response.Genre).Flatten().Map(g => new Genre { Name = g.Tag }).ToList(),
Tags = new List<Tag>()
Tags = new List<Tag>(),
Studios = new List<Studio>()
};
if (!string.IsNullOrWhiteSpace(response.Studio))
{
metadata.Studios.Add(new Studio { Name = response.Studio });
}
if (DateTime.TryParse(response.OriginallyAvailableAt, out DateTime releaseDate))
{
metadata.ReleaseDate = releaseDate;
@@ -279,9 +285,15 @@ namespace ErsatzTV.Infrastructure.Plex
DateAdded = dateAdded,
DateUpdated = lastWriteTime,
Genres = Optional(response.Genre).Flatten().Map(g => new Genre { Name = g.Tag }).ToList(),
Tags = new List<Tag>()
Tags = new List<Tag>(),
Studios = new List<Studio>()
};
if (!string.IsNullOrWhiteSpace(response.Studio))
{
metadata.Studios.Add(new Studio { Name = response.Studio });
}
if (DateTime.TryParse(response.OriginallyAvailableAt, out DateTime releaseDate))
{
metadata.ReleaseDate = releaseDate;
ErsatzTV.Infrastructure/Search/SearchIndex.cs
+14 -1
View File
@@ -38,6 +38,7 @@ namespace ErsatzTV.Infrastructure.Search
private const string TitleAndYearField = "title_and_year";
private const string JumpLetterField = "jump_letter";
private const string ReleaseDateField = "release_date";
private const string StudioField = "studio";
private const string MovieType = "movie";
private const string ShowType = "show";
@@ -48,7 +49,7 @@ namespace ErsatzTV.Infrastructure.Search
private readonly ILogger<SearchIndex> _logger;
private readonly ISearchRepository _searchRepository;
public SearchIndex(
ILocalFileSystem localFileSystem,
ISearchRepository searchRepository,
@@ -263,10 +264,16 @@ namespace ErsatzTV.Infrastructure.Search
doc.Add(new TextField(TagField, tag.Name, Field.Store.NO));
}
foreach (Studio studio in metadata.Studios)
{
doc.Add(new TextField(StudioField, studio.Name, Field.Store.NO));
}
writer.UpdateDocument(new Term(IdField, movie.Id.ToString()), doc);
}
catch (Exception ex)
{
metadata.Movie = null;
_logger.LogWarning(ex, "Error indexing movie with metadata {@Metadata}", metadata);
}
}
@@ -316,10 +323,16 @@ namespace ErsatzTV.Infrastructure.Search
doc.Add(new TextField(TagField, tag.Name, Field.Store.NO));
}
foreach (Studio studio in metadata.Studios)
{
doc.Add(new TextField(StudioField, studio.Name, Field.Store.NO));
}
writer.UpdateDocument(new Term(IdField, show.Id.ToString()), doc);
}
catch (Exception ex)
{
metadata.Show = null;
_logger.LogWarning(ex, "Error indexing show with metadata {@Metadata}", metadata);
}
}