From 307940d732ecc8bded05df90d2ff6a197969d8ca Mon Sep 17 00:00:00 2001 From: Jason Dove <1695733+jasongdove@users.noreply.github.com> Date: Sat, 1 Apr 2023 09:54:19 -0500 Subject: [PATCH] add duplicate file logging (#1223) --- CHANGELOG.md | 2 + .../Scheduling/ScheduleIntegrationTests.cs | 4 +- .../Errors/MediaFileAlreadyExists.cs | 8 +++ .../Data/Repositories/EmbyMovieRepository.cs | 14 ++++- .../Repositories/EmbyTelevisionRepository.cs | 15 +++++- .../Repositories/JellyfinMovieRepository.cs | 15 +++++- .../JellyfinTelevisionRepository.cs | 15 +++++- .../Data/Repositories/MediaItemRepository.cs | 51 +++++++++++++++++++ .../Data/Repositories/MovieRepository.cs | 17 ++++++- .../Data/Repositories/PlexMovieRepository.cs | 14 ++++- .../Repositories/PlexTelevisionRepository.cs | 7 +-- .../Data/Repositories/TelevisionRepository.cs | 16 ++++-- 12 files changed, 162 insertions(+), 16 deletions(-) create mode 100644 ErsatzTV.Core/Errors/MediaFileAlreadyExists.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 03912eafa..0943d5072 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added - Use `plot` field from Other Video NFO metadata as XMLTV description +- Add detailed warning log when a file is added to ErsatzTV more than once ### Fixed - Fix updating (re-adding) Trakt lists to properly use new metadata ids that were not present when originally added - Fix local show library scanning with non-english season folder names, e.g. `Staffel 02` +- Fix bug where local libraries would merge with media server libraries when the same file was added to both libraries ### Changed - Use Poster artwork for XMLTV if available diff --git a/ErsatzTV.Core.Tests/Scheduling/ScheduleIntegrationTests.cs b/ErsatzTV.Core.Tests/Scheduling/ScheduleIntegrationTests.cs index cf2798b73..183272a1a 100644 --- a/ErsatzTV.Core.Tests/Scheduling/ScheduleIntegrationTests.cs +++ b/ErsatzTV.Core.Tests/Scheduling/ScheduleIntegrationTests.cs @@ -115,7 +115,7 @@ public class ScheduleIntegrationTests var builder = new PlayoutBuilder( new ConfigElementRepository(factory), new MediaCollectionRepository(new Mock().Object, searchIndex, factory), - new TelevisionRepository(factory), + new TelevisionRepository(factory, provider.GetRequiredService>()), new ArtistRepository(factory), new Mock().Object, new Mock().Object, @@ -275,7 +275,7 @@ public class ScheduleIntegrationTests var builder = new PlayoutBuilder( new ConfigElementRepository(factory), new MediaCollectionRepository(new Mock().Object, new Mock().Object, factory), - new TelevisionRepository(factory), + new TelevisionRepository(factory, provider.GetRequiredService>()), new ArtistRepository(factory), new Mock().Object, new Mock().Object, diff --git a/ErsatzTV.Core/Errors/MediaFileAlreadyExists.cs b/ErsatzTV.Core/Errors/MediaFileAlreadyExists.cs new file mode 100644 index 000000000..3553cc814 --- /dev/null +++ b/ErsatzTV.Core/Errors/MediaFileAlreadyExists.cs @@ -0,0 +1,8 @@ +namespace ErsatzTV.Core.Errors; + +public class MediaFileAlreadyExists : BaseError +{ + public MediaFileAlreadyExists() : base("Media file already exists") + { + } +} diff --git a/ErsatzTV.Infrastructure/Data/Repositories/EmbyMovieRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/EmbyMovieRepository.cs index e7f1a9810..33764b47d 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/EmbyMovieRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/EmbyMovieRepository.cs @@ -2,18 +2,25 @@ using ErsatzTV.Core; using ErsatzTV.Core.Domain; using ErsatzTV.Core.Emby; +using ErsatzTV.Core.Errors; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Metadata; using ErsatzTV.Infrastructure.Extensions; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; namespace ErsatzTV.Infrastructure.Data.Repositories; public class EmbyMovieRepository : IEmbyMovieRepository { private readonly IDbContextFactory _dbContextFactory; + private readonly ILogger _logger; - public EmbyMovieRepository(IDbContextFactory dbContextFactory) => _dbContextFactory = dbContextFactory; + public EmbyMovieRepository(IDbContextFactory dbContextFactory, ILogger logger) + { + _dbContextFactory = dbContextFactory; + _logger = logger; + } public async Task> GetExistingMovies(EmbyLibrary library) { @@ -178,6 +185,11 @@ public class EmbyMovieRepository : IEmbyMovieRepository { try { + if (await MediaItemRepository.MediaFileAlreadyExists(movie, dbContext, _logger)) + { + return new MediaFileAlreadyExists(); + } + // blank out etag for initial save in case other updates fail string etag = movie.Etag; movie.Etag = string.Empty; diff --git a/ErsatzTV.Infrastructure/Data/Repositories/EmbyTelevisionRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/EmbyTelevisionRepository.cs index 2bdb7ef4f..74132032f 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/EmbyTelevisionRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/EmbyTelevisionRepository.cs @@ -2,19 +2,27 @@ using ErsatzTV.Core; using ErsatzTV.Core.Domain; using ErsatzTV.Core.Emby; +using ErsatzTV.Core.Errors; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Metadata; using ErsatzTV.Infrastructure.Extensions; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; namespace ErsatzTV.Infrastructure.Data.Repositories; public class EmbyTelevisionRepository : IEmbyTelevisionRepository { private readonly IDbContextFactory _dbContextFactory; + private readonly ILogger _logger; - public EmbyTelevisionRepository(IDbContextFactory dbContextFactory) => + public EmbyTelevisionRepository( + IDbContextFactory dbContextFactory, + ILogger logger) + { _dbContextFactory = dbContextFactory; + _logger = logger; + } public async Task> GetExistingShows(EmbyLibrary library) { @@ -801,6 +809,11 @@ public class EmbyTelevisionRepository : IEmbyTelevisionRepository { try { + if (await MediaItemRepository.MediaFileAlreadyExists(episode, dbContext, _logger)) + { + return new MediaFileAlreadyExists(); + } + // blank out etag for initial save in case other updates fail string etag = episode.Etag; episode.Etag = string.Empty; diff --git a/ErsatzTV.Infrastructure/Data/Repositories/JellyfinMovieRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/JellyfinMovieRepository.cs index 019ae64d9..c397f5c33 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/JellyfinMovieRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/JellyfinMovieRepository.cs @@ -1,20 +1,28 @@ using Dapper; using ErsatzTV.Core; using ErsatzTV.Core.Domain; +using ErsatzTV.Core.Errors; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Jellyfin; using ErsatzTV.Core.Metadata; using ErsatzTV.Infrastructure.Extensions; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; namespace ErsatzTV.Infrastructure.Data.Repositories; public class JellyfinMovieRepository : IJellyfinMovieRepository { private readonly IDbContextFactory _dbContextFactory; + private readonly ILogger _logger; - public JellyfinMovieRepository(IDbContextFactory dbContextFactory) => + public JellyfinMovieRepository( + IDbContextFactory dbContextFactory, + ILogger logger) + { _dbContextFactory = dbContextFactory; + _logger = logger; + } public async Task> GetExistingMovies(JellyfinLibrary library) { @@ -359,6 +367,11 @@ public class JellyfinMovieRepository : IJellyfinMovieRepository { try { + if (await MediaItemRepository.MediaFileAlreadyExists(movie, dbContext, _logger)) + { + return new MediaFileAlreadyExists(); + } + // blank out etag for initial save in case other updates fail string etag = movie.Etag; movie.Etag = string.Empty; diff --git a/ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs index 3fdc00381..97a10e67b 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs @@ -1,20 +1,28 @@ using Dapper; using ErsatzTV.Core; using ErsatzTV.Core.Domain; +using ErsatzTV.Core.Errors; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Jellyfin; using ErsatzTV.Core.Metadata; using ErsatzTV.Infrastructure.Extensions; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; namespace ErsatzTV.Infrastructure.Data.Repositories; public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository { private readonly IDbContextFactory _dbContextFactory; + private readonly ILogger _logger; - public JellyfinTelevisionRepository(IDbContextFactory dbContextFactory) => + public JellyfinTelevisionRepository( + IDbContextFactory dbContextFactory, + ILogger logger) + { _dbContextFactory = dbContextFactory; + _logger = logger; + } public async Task> GetExistingShows(JellyfinLibrary library) { @@ -804,6 +812,11 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository { try { + if (await MediaItemRepository.MediaFileAlreadyExists(episode, dbContext, _logger)) + { + return new MediaFileAlreadyExists(); + } + // blank out etag for initial save in case other updates fail string etag = episode.Etag; episode.Etag = string.Empty; diff --git a/ErsatzTV.Infrastructure/Data/Repositories/MediaItemRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/MediaItemRepository.cs index 7348dac76..e95e262e3 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/MediaItemRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/MediaItemRepository.cs @@ -3,9 +3,11 @@ using System.Globalization; using Dapper; using ErsatzTV.Core; using ErsatzTV.Core.Domain; +using ErsatzTV.Core.Extensions; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Infrastructure.Extensions; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; namespace ErsatzTV.Infrastructure.Data.Repositories; @@ -103,6 +105,7 @@ public class MediaItemRepository : IMediaItemRepository mediaItem.State = MediaItemState.Normal; return await dbContext.Connection.ExecuteAsync( + @"UPDATE MediaItem SET State = 0 WHERE Id = @Id", new { mediaItem.Id }).ToUnit(); } @@ -121,6 +124,54 @@ public class MediaItemRepository : IMediaItemRepository return Unit.Default; } + public static async Task MediaFileAlreadyExists(MediaItem incoming, TvContext dbContext, ILogger logger) + { + string path = incoming.GetHeadVersion().MediaFiles.Head().Path; + return await MediaFileAlreadyExists(path, dbContext, logger); + } + + public static async Task MediaFileAlreadyExists(string path, TvContext dbContext, ILogger logger) + { + Option maybeMediaItemId = await dbContext.Connection + .QuerySingleOrDefaultAsync( + @"select coalesce(EpisodeId, MovieId, MusicVideoId, OtherVideoId, SongId) as MediaItemId + from MediaVersion MV + inner join MediaFile MF on MV.Id = MF.MediaVersionId + where MF.Path = @Path", + new { Path = path }) + .Map(Optional); + + foreach (int mediaItemId in maybeMediaItemId) + { + Option maybeMediaItem = await dbContext.MediaItems + .AsNoTracking() + .Include(mi => mi.LibraryPath) + .ThenInclude(lp => lp.Library) + .SelectOneAsync(mi => mi.Id, mi => mi.Id == mediaItemId); + + foreach (MediaItem mediaItem in maybeMediaItem) + { + string libraryType = mediaItem.LibraryPath.Library switch + { + PlexLibrary => "Plex Library", + EmbyLibrary => "Emby Library", + JellyfinLibrary => "Jellyfin Library", + _ => "Local Library" + }; + + string libraryName = mediaItem.LibraryPath.Library.Name; + logger.LogWarning( + "Unable to add media item; {LibraryType} '{LibraryName}' already contains path {Path}", + libraryType, + libraryName, + path); + return true; + } + } + + return false; + } + private async Task> GetAllLanguageCodes() { await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(); diff --git a/ErsatzTV.Infrastructure/Data/Repositories/MovieRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/MovieRepository.cs index 8990d1966..c12b0cfd1 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/MovieRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/MovieRepository.cs @@ -1,17 +1,24 @@ using Dapper; using ErsatzTV.Core; using ErsatzTV.Core.Domain; +using ErsatzTV.Core.Errors; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Metadata; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; namespace ErsatzTV.Infrastructure.Data.Repositories; public class MovieRepository : IMovieRepository { private readonly IDbContextFactory _dbContextFactory; + private readonly ILogger _logger; - public MovieRepository(IDbContextFactory dbContextFactory) => _dbContextFactory = dbContextFactory; + public MovieRepository(IDbContextFactory dbContextFactory, ILogger logger) + { + _dbContextFactory = dbContextFactory; + _logger = logger; + } public async Task AllMoviesExist(List movieIds) { @@ -54,6 +61,7 @@ public class MovieRepository : IMovieRepository { await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(); Option maybeExisting = await dbContext.Movies + .Filter(m => !(m is PlexMovie) && !(m is JellyfinMovie) && !(m is EmbyMovie)) .Include(i => i.MovieMetadata) .ThenInclude(mm => mm.Artwork) .Include(i => i.MovieMetadata) @@ -219,13 +227,18 @@ public class MovieRepository : IMovieRepository new { writer.Name, MetadataId = metadata.Id }).Map(result => result > 0); } - private static async Task>> AddMovie( + private async Task>> AddMovie( TvContext dbContext, int libraryPathId, string path) { try { + if (await MediaItemRepository.MediaFileAlreadyExists(path, dbContext, _logger)) + { + return new MediaFileAlreadyExists(); + } + var movie = new Movie { LibraryPathId = libraryPathId, diff --git a/ErsatzTV.Infrastructure/Data/Repositories/PlexMovieRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/PlexMovieRepository.cs index 3e43cd9cc..b06aaef04 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/PlexMovieRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/PlexMovieRepository.cs @@ -1,19 +1,26 @@ using Dapper; using ErsatzTV.Core; using ErsatzTV.Core.Domain; +using ErsatzTV.Core.Errors; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Metadata; using ErsatzTV.Core.Plex; using ErsatzTV.Infrastructure.Extensions; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; namespace ErsatzTV.Infrastructure.Data.Repositories; public class PlexMovieRepository : IPlexMovieRepository { private readonly IDbContextFactory _dbContextFactory; + private readonly ILogger _logger; - public PlexMovieRepository(IDbContextFactory dbContextFactory) => _dbContextFactory = dbContextFactory; + public PlexMovieRepository(IDbContextFactory dbContextFactory, ILogger logger) + { + _dbContextFactory = dbContextFactory; + _logger = logger; + } public async Task> GetExistingMovies(PlexLibrary library) { @@ -178,6 +185,11 @@ public class PlexMovieRepository : IPlexMovieRepository { try { + if (await MediaItemRepository.MediaFileAlreadyExists(item, dbContext, _logger)) + { + return new MediaFileAlreadyExists(); + } + // blank out etag for initial save in case stats/metadata/etc updates fail string etag = item.Etag; item.Etag = string.Empty; diff --git a/ErsatzTV.Infrastructure/Data/Repositories/PlexTelevisionRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/PlexTelevisionRepository.cs index 7d6615375..b1c7809e7 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/PlexTelevisionRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/PlexTelevisionRepository.cs @@ -1,6 +1,7 @@ using Dapper; using ErsatzTV.Core; using ErsatzTV.Core.Domain; +using ErsatzTV.Core.Errors; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Metadata; using ErsatzTV.Core.Plex; @@ -425,16 +426,16 @@ public class PlexTelevisionRepository : IPlexTelevisionRepository } } - private static async Task>> AddEpisode( + private async Task>> AddEpisode( TvContext dbContext, PlexLibrary library, PlexEpisode item) { try { - if (dbContext.MediaFiles.Any(mf => mf.Path == item.MediaVersions.Head().MediaFiles.Head().Path)) + if (await MediaItemRepository.MediaFileAlreadyExists(item, dbContext, _logger)) { - return BaseError.New("Multi-episode files are not yet supported"); + return new MediaFileAlreadyExists(); } // blank out etag for initial save in case stats/metadata/etc updates fail diff --git a/ErsatzTV.Infrastructure/Data/Repositories/TelevisionRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/TelevisionRepository.cs index 5bc7d886b..5e7fc17f6 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/TelevisionRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/TelevisionRepository.cs @@ -1,17 +1,24 @@ using Dapper; using ErsatzTV.Core; using ErsatzTV.Core.Domain; +using ErsatzTV.Core.Errors; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Metadata; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; namespace ErsatzTV.Infrastructure.Data.Repositories; public class TelevisionRepository : ITelevisionRepository { private readonly IDbContextFactory _dbContextFactory; + private readonly ILogger _logger; - public TelevisionRepository(IDbContextFactory dbContextFactory) => _dbContextFactory = dbContextFactory; + public TelevisionRepository(IDbContextFactory dbContextFactory, ILogger logger) + { + _dbContextFactory = dbContextFactory; + _logger = logger; + } public async Task AllShowsExist(List showIds) { @@ -325,6 +332,7 @@ public class TelevisionRepository : ITelevisionRepository { await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(); Option maybeExisting = await dbContext.Episodes + .Filter(e => !(e is PlexEpisode) && !(e is JellyfinEpisode) && !(e is EmbyEpisode)) .Include(i => i.EpisodeMetadata) .ThenInclude(em => em.Artwork) .Include(i => i.EpisodeMetadata) @@ -670,7 +678,7 @@ public class TelevisionRepository : ITelevisionRepository } } - private static async Task> AddEpisode( + private async Task> AddEpisode( TvContext dbContext, Season season, int libraryPathId, @@ -678,9 +686,9 @@ public class TelevisionRepository : ITelevisionRepository { try { - if (dbContext.MediaFiles.Any(mf => mf.Path == path)) + if (await MediaItemRepository.MediaFileAlreadyExists(path, dbContext, _logger)) { - return BaseError.New("Multi-episode files are not yet supported"); + return new MediaFileAlreadyExists(); } var episode = new Episode