diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e6aaf92b..bf9505e7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Enable QSV hardware acceleration for vaapi docker images ### Changed -- Use paging to synchronize movies from Plex, Jellyfin and Emby +- Use paging to synchronize all media from Plex, Jellyfin and Emby - This will reduce memory use and improve reliability of synchronizing large libraries - Disable low power mode for `h264_qsv` and `hevc_qsv` encoders diff --git a/ErsatzTV.Application/Channels/Queries/GetChannelGuideHandler.cs b/ErsatzTV.Application/Channels/Queries/GetChannelGuideHandler.cs index 78e390f4b..bd1397135 100644 --- a/ErsatzTV.Application/Channels/Queries/GetChannelGuideHandler.cs +++ b/ErsatzTV.Application/Channels/Queries/GetChannelGuideHandler.cs @@ -9,7 +9,9 @@ public class GetChannelGuideHandler : IRequestHandler> ScanCollections(string address, string apiKey) { - // get all collections from db (item id, etag) - List existingCollections = await _embyCollectionRepository.GetCollections(); - - // get all collections from emby - Either> maybeIncomingCollections = - await _embyApiClient.GetCollectionLibraryItems(address, apiKey); - - foreach (BaseError error in maybeIncomingCollections.LeftToSeq()) + try { - _logger.LogWarning("Failed to get collections from Emby: {Error}", error.ToString()); - return error; - } + var incomingItemIds = new List(); - foreach (List incomingCollections in maybeIncomingCollections.RightToSeq()) - { - // loop over collections - foreach (EmbyCollection collection in incomingCollections) + // get all collections from db (item id, etag) + List existingCollections = await _embyCollectionRepository.GetCollections(); + + await foreach (EmbyCollection collection in _embyApiClient.GetCollectionLibraryItems(address, apiKey)) { + incomingItemIds.Add(collection.ItemId); + Option maybeExisting = existingCollections.Find(c => c.ItemId == collection.ItemId); // skip if unchanged (etag) - if (await maybeExisting.Map(e => e.Etag ?? string.Empty).IfNoneAsync(string.Empty) == collection.Etag) + if (await maybeExisting.Map(e => e.Etag ?? string.Empty).IfNoneAsync(string.Empty) == + collection.Etag) { _logger.LogDebug("Emby collection {Name} is unchanged", collection.Name); continue; @@ -75,12 +69,16 @@ public class EmbyCollectionScanner : IEmbyCollectionScanner } // remove missing collections (and remove any lingering tags from those collections) - foreach (EmbyCollection collection in existingCollections - .Filter(e => incomingCollections.All(i => i.ItemId != e.ItemId))) + foreach (EmbyCollection collection in existingCollections.Filter(e => !incomingItemIds.Contains(e.ItemId))) { await _embyCollectionRepository.RemoveCollection(collection); } } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to get collections from Emby"); + return BaseError.New(ex.Message); + } return Unit.Default; } @@ -90,32 +88,31 @@ public class EmbyCollectionScanner : IEmbyCollectionScanner string apiKey, EmbyCollection collection) { - // get collection items from JF - Either> maybeItems = - await _embyApiClient.GetCollectionItems(address, apiKey, collection.ItemId); - - foreach (BaseError error in maybeItems.LeftToSeq()) + try { - _logger.LogWarning("Failed to get collection items from Emby: {Error}", error.ToString()); - return; + // get collection items from Emby + IAsyncEnumerable items = _embyApiClient.GetCollectionItems(address, apiKey, collection.ItemId); + + List removedIds = await _embyCollectionRepository.RemoveAllTags(collection); + + // sync tags on items + var addedIds = new List(); + await foreach (MediaItem item in items) + { + addedIds.Add(await _embyCollectionRepository.AddTag(item, collection)); + } + + _logger.LogDebug("Emby collection {Name} contains {Count} items", collection.Name, addedIds.Count); + + var changedIds = removedIds.Except(addedIds).ToList(); + changedIds.AddRange(addedIds.Except(removedIds)); + + await _searchIndex.RebuildItems(_searchRepository, changedIds); + _searchIndex.Commit(); } - - List removedIds = await _embyCollectionRepository.RemoveAllTags(collection); - - var embyItems = maybeItems.RightToSeq().Flatten().ToList(); - _logger.LogDebug("Emby collection {Name} contains {Count} items", collection.Name, embyItems.Count); - - // sync tags on items - var addedIds = new List(); - foreach (MediaItem item in embyItems) + catch (Exception ex) { - addedIds.Add(await _embyCollectionRepository.AddTag(item, collection)); + _logger.LogWarning(ex, "Failed to synchronize Emby collection {Name}", collection.Name); } - - var changedIds = removedIds.Except(addedIds).ToList(); - changedIds.AddRange(addedIds.Except(removedIds)); - - await _searchIndex.RebuildItems(_searchRepository, changedIds); - _searchIndex.Commit(); } } diff --git a/ErsatzTV.Core/Emby/EmbyItemType.cs b/ErsatzTV.Core/Emby/EmbyItemType.cs index 585726f8f..a1ea5d56d 100644 --- a/ErsatzTV.Core/Emby/EmbyItemType.cs +++ b/ErsatzTV.Core/Emby/EmbyItemType.cs @@ -3,4 +3,9 @@ public static class EmbyItemType { public static readonly string Movie = "Movie"; + public static readonly string Show = "Series"; + public static readonly string Season = "Season"; + public static readonly string Episode = "Episode"; + public static readonly string Collection = "BoxSet"; + public static readonly string CollectionItems = "Movie,Series,Season,Episode"; } diff --git a/ErsatzTV.Core/Emby/EmbyMovieLibraryScanner.cs b/ErsatzTV.Core/Emby/EmbyMovieLibraryScanner.cs index 22101e444..4a95ea67f 100644 --- a/ErsatzTV.Core/Emby/EmbyMovieLibraryScanner.cs +++ b/ErsatzTV.Core/Emby/EmbyMovieLibraryScanner.cs @@ -85,7 +85,7 @@ public class EmbyMovieLibraryScanner : _embyApiClient.GetLibraryItemCount( connectionParameters.Address, connectionParameters.ApiKey, - library, + library.ItemId, EmbyItemType.Movie); protected override IAsyncEnumerable GetMovieLibraryItems( diff --git a/ErsatzTV.Core/Emby/EmbyTelevisionLibraryScanner.cs b/ErsatzTV.Core/Emby/EmbyTelevisionLibraryScanner.cs index dd7fb8cc0..e2744ab0b 100644 --- a/ErsatzTV.Core/Emby/EmbyTelevisionLibraryScanner.cs +++ b/ErsatzTV.Core/Emby/EmbyTelevisionLibraryScanner.cs @@ -76,13 +76,19 @@ public class EmbyTelevisionLibraryScanner : MediaServerTelevisionLibraryScanner< cancellationToken); } - protected override Task>> GetShowLibraryItems( + protected override Task> CountShowLibraryItems( EmbyConnectionParameters connectionParameters, - EmbyLibrary library) => - _embyApiClient.GetShowLibraryItems( + EmbyLibrary library) + => _embyApiClient.GetLibraryItemCount( connectionParameters.Address, connectionParameters.ApiKey, - library.ItemId); + library.ItemId, + EmbyItemType.Show); + + protected override IAsyncEnumerable GetShowLibraryItems( + EmbyConnectionParameters connectionParameters, + EmbyLibrary library) => + _embyApiClient.GetShowLibraryItems(connectionParameters.Address, connectionParameters.ApiKey, library); protected override string MediaServerItemId(EmbyShow show) => show.ItemId; protected override string MediaServerItemId(EmbySeason season) => season.ItemId; @@ -92,23 +98,46 @@ public class EmbyTelevisionLibraryScanner : MediaServerTelevisionLibraryScanner< protected override string MediaServerEtag(EmbySeason season) => season.Etag; protected override string MediaServerEtag(EmbyEpisode episode) => episode.Etag; - protected override Task>> GetSeasonLibraryItems( + protected override Task> CountSeasonLibraryItems( + EmbyConnectionParameters connectionParameters, + EmbyLibrary library, + EmbyShow show) => + _embyApiClient.GetLibraryItemCount( + connectionParameters.Address, + connectionParameters.ApiKey, + show.ItemId, + EmbyItemType.Season); + + protected override IAsyncEnumerable GetSeasonLibraryItems( EmbyLibrary library, EmbyConnectionParameters connectionParameters, EmbyShow show) => _embyApiClient.GetSeasonLibraryItems( connectionParameters.Address, connectionParameters.ApiKey, + library, show.ItemId); - protected override Task>> GetEpisodeLibraryItems( + protected override Task> CountEpisodeLibraryItems( + EmbyConnectionParameters connectionParameters, + EmbyLibrary library, + EmbySeason season) => + _embyApiClient.GetLibraryItemCount( + connectionParameters.Address, + connectionParameters.ApiKey, + season.ItemId, + EmbyItemType.Episode); + + protected override IAsyncEnumerable GetEpisodeLibraryItems( EmbyLibrary library, EmbyConnectionParameters connectionParameters, + EmbyShow show, EmbySeason season) => _embyApiClient.GetEpisodeLibraryItems( connectionParameters.Address, connectionParameters.ApiKey, library, + show.ItemId, season.ItemId); protected override Task> GetFullMetadata( diff --git a/ErsatzTV.Core/Interfaces/Emby/IEmbyApiClient.cs b/ErsatzTV.Core/Interfaces/Emby/IEmbyApiClient.cs index cee715b6d..a5320f956 100644 --- a/ErsatzTV.Core/Interfaces/Emby/IEmbyApiClient.cs +++ b/ErsatzTV.Core/Interfaces/Emby/IEmbyApiClient.cs @@ -8,39 +8,30 @@ public interface IEmbyApiClient Task> GetServerInformation(string address, string apiKey); Task>> GetLibraries(string address, string apiKey); - IAsyncEnumerable GetMovieLibraryItems( - string address, - string apiKey, - EmbyLibrary library); + IAsyncEnumerable GetMovieLibraryItems(string address, string apiKey, EmbyLibrary library); - Task>> GetShowLibraryItems( - string address, - string apiKey, - string libraryId); + IAsyncEnumerable GetShowLibraryItems(string address, string apiKey, EmbyLibrary library); - Task>> GetSeasonLibraryItems( - string address, - string apiKey, - string showId); - - Task>> GetEpisodeLibraryItems( + IAsyncEnumerable GetSeasonLibraryItems( string address, string apiKey, EmbyLibrary library, - string seasonId); + string showId); - Task>> GetCollectionLibraryItems( - string address, - string apiKey); - - Task>> GetCollectionItems( + IAsyncEnumerable GetEpisodeLibraryItems( string address, string apiKey, - string collectionId); + EmbyLibrary library, + string showId, + string seasonId); + + IAsyncEnumerable GetCollectionLibraryItems(string address, string apiKey); + + IAsyncEnumerable GetCollectionItems(string address, string apiKey, string collectionId); Task> GetLibraryItemCount( string address, string apiKey, - EmbyLibrary library, + string parentId, string includeItemTypes); } diff --git a/ErsatzTV.Core/Interfaces/Jellyfin/IJellyfinApiClient.cs b/ErsatzTV.Core/Interfaces/Jellyfin/IJellyfinApiClient.cs index 66e8002e5..6406dcca8 100644 --- a/ErsatzTV.Core/Interfaces/Jellyfin/IJellyfinApiClient.cs +++ b/ErsatzTV.Core/Interfaces/Jellyfin/IJellyfinApiClient.cs @@ -9,35 +9,25 @@ public interface IJellyfinApiClient Task>> GetLibraries(string address, string apiKey); Task> GetAdminUserId(string address, string apiKey); - IAsyncEnumerable GetMovieLibraryItems( - string address, - string apiKey, - JellyfinLibrary library); + IAsyncEnumerable GetMovieLibraryItems(string address, string apiKey, JellyfinLibrary library); - Task>> GetShowLibraryItems( - string address, - string apiKey, - int mediaSourceId, - string libraryId); + IAsyncEnumerable GetShowLibraryItems(string address, string apiKey, JellyfinLibrary library); - Task>> GetSeasonLibraryItems( + IAsyncEnumerable GetSeasonLibraryItems( string address, string apiKey, - int mediaSourceId, + JellyfinLibrary library, string showId); - Task>> GetEpisodeLibraryItems( + IAsyncEnumerable GetEpisodeLibraryItems( string address, string apiKey, JellyfinLibrary library, string seasonId); - Task>> GetCollectionLibraryItems( - string address, - string apiKey, - int mediaSourceId); + IAsyncEnumerable GetCollectionLibraryItems(string address, string apiKey, int mediaSourceId); - Task>> GetCollectionItems( + IAsyncEnumerable GetCollectionItems( string address, string apiKey, int mediaSourceId, @@ -47,5 +37,7 @@ public interface IJellyfinApiClient string address, string apiKey, JellyfinLibrary library, - string includeItemTypes); + string parentId, + string includeItemTypes, + bool excludeFolders); } diff --git a/ErsatzTV.Core/Interfaces/Plex/IPlexServerApiClient.cs b/ErsatzTV.Core/Interfaces/Plex/IPlexServerApiClient.cs index a96327d7e..3a4b939cc 100644 --- a/ErsatzTV.Core/Interfaces/Plex/IPlexServerApiClient.cs +++ b/ErsatzTV.Core/Interfaces/Plex/IPlexServerApiClient.cs @@ -18,18 +18,28 @@ public interface IPlexServerApiClient PlexConnection connection, PlexServerAuthToken token); - Task>> GetShowLibraryContents( + IAsyncEnumerable GetShowLibraryContents( PlexLibrary library, PlexConnection connection, PlexServerAuthToken token); - Task>> GetShowSeasons( + Task> CountShowSeasons( + PlexShow show, + PlexConnection connection, + PlexServerAuthToken token); + + IAsyncEnumerable GetShowSeasons( PlexLibrary library, PlexShow show, PlexConnection connection, PlexServerAuthToken token); - Task>> GetSeasonEpisodes( + Task> CountSeasonEpisodes( + PlexSeason season, + PlexConnection connection, + PlexServerAuthToken token); + + IAsyncEnumerable GetSeasonEpisodes( PlexLibrary library, PlexSeason season, PlexConnection connection, diff --git a/ErsatzTV.Core/Jellyfin/JellyfinCollectionScanner.cs b/ErsatzTV.Core/Jellyfin/JellyfinCollectionScanner.cs index 5abda2eca..28e5e99dd 100644 --- a/ErsatzTV.Core/Jellyfin/JellyfinCollectionScanner.cs +++ b/ErsatzTV.Core/Jellyfin/JellyfinCollectionScanner.cs @@ -30,24 +30,21 @@ public class JellyfinCollectionScanner : IJellyfinCollectionScanner public async Task> ScanCollections(string address, string apiKey, int mediaSourceId) { - // get all collections from db (item id, etag) - List existingCollections = await _jellyfinCollectionRepository.GetCollections(); - - // get all collections from jellyfin - Either> maybeIncomingCollections = - await _jellyfinApiClient.GetCollectionLibraryItems(address, apiKey, mediaSourceId); - - foreach (BaseError error in maybeIncomingCollections.LeftToSeq()) + try { - _logger.LogWarning("Failed to get collections from Jellyfin: {Error}", error.ToString()); - return error; - } + var incomingItemIds = new List(); + + // get all collections from db (item id, etag) + List existingCollections = await _jellyfinCollectionRepository.GetCollections(); - foreach (List incomingCollections in maybeIncomingCollections.RightToSeq()) - { // loop over collections - foreach (JellyfinCollection collection in incomingCollections) + await foreach (JellyfinCollection collection in _jellyfinApiClient.GetCollectionLibraryItems( + address, + apiKey, + mediaSourceId)) { + incomingItemIds.Add(collection.ItemId); + Option maybeExisting = existingCollections.Find(c => c.ItemId == collection.ItemId); // skip if unchanged (etag) @@ -75,12 +72,17 @@ public class JellyfinCollectionScanner : IJellyfinCollectionScanner } // remove missing collections (and remove any lingering tags from those collections) - foreach (JellyfinCollection collection in existingCollections - .Filter(e => incomingCollections.All(i => i.ItemId != e.ItemId))) + foreach (JellyfinCollection collection in existingCollections.Filter( + e => !incomingItemIds.Contains(e.ItemId))) { await _jellyfinCollectionRepository.RemoveCollection(collection); } } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to get collections from Jellyfin"); + return BaseError.New(ex.Message); + } return Unit.Default; } @@ -91,32 +93,35 @@ public class JellyfinCollectionScanner : IJellyfinCollectionScanner int mediaSourceId, JellyfinCollection collection) { - // get collection items from JF - Either> maybeItems = - await _jellyfinApiClient.GetCollectionItems(address, apiKey, mediaSourceId, collection.ItemId); - - foreach (BaseError error in maybeItems.LeftToSeq()) + try { - _logger.LogWarning("Failed to get collection items from Jellyfin: {Error}", error.ToString()); - return; + // get collection items from JF + IAsyncEnumerable items = _jellyfinApiClient.GetCollectionItems( + address, + apiKey, + mediaSourceId, + collection.ItemId); + + List removedIds = await _jellyfinCollectionRepository.RemoveAllTags(collection); + + // sync tags on items + var addedIds = new List(); + await foreach (MediaItem item in items) + { + addedIds.Add(await _jellyfinCollectionRepository.AddTag(item, collection)); + } + + _logger.LogDebug("Jellyfin collection {Name} contains {Count} items", collection.Name, addedIds.Count); + + var changedIds = removedIds.Except(addedIds).ToList(); + changedIds.AddRange(addedIds.Except(removedIds)); + + await _searchIndex.RebuildItems(_searchRepository, changedIds); + _searchIndex.Commit(); } - - List removedIds = await _jellyfinCollectionRepository.RemoveAllTags(collection); - - var jellyfinItems = maybeItems.RightToSeq().Flatten().ToList(); - _logger.LogDebug("Jellyfin collection {Name} contains {Count} items", collection.Name, jellyfinItems.Count); - - // sync tags on items - var addedIds = new List(); - foreach (MediaItem item in jellyfinItems) + catch (Exception ex) { - addedIds.Add(await _jellyfinCollectionRepository.AddTag(item, collection)); + _logger.LogWarning(ex, "Failed to synchronize Jellyfin collection {Name}", collection.Name); } - - var changedIds = removedIds.Except(addedIds).ToList(); - changedIds.AddRange(addedIds.Except(removedIds)); - - await _searchIndex.RebuildItems(_searchRepository, changedIds); - _searchIndex.Commit(); } } diff --git a/ErsatzTV.Core/Jellyfin/JellyfinItemType.cs b/ErsatzTV.Core/Jellyfin/JellyfinItemType.cs index 443a56e39..fb197f8b7 100644 --- a/ErsatzTV.Core/Jellyfin/JellyfinItemType.cs +++ b/ErsatzTV.Core/Jellyfin/JellyfinItemType.cs @@ -3,4 +3,9 @@ public static class JellyfinItemType { public static readonly string Movie = "Movie"; + public static readonly string Show = "Series"; + public static readonly string Season = "Season"; + public static readonly string Episode = "Episode"; + public static readonly string Collection = "BoxSet"; + public static readonly string CollectionItems = "Movie,Series,Season,Episode"; } diff --git a/ErsatzTV.Core/Jellyfin/JellyfinMovieLibraryScanner.cs b/ErsatzTV.Core/Jellyfin/JellyfinMovieLibraryScanner.cs index 09d412bc9..b4aec737a 100644 --- a/ErsatzTV.Core/Jellyfin/JellyfinMovieLibraryScanner.cs +++ b/ErsatzTV.Core/Jellyfin/JellyfinMovieLibraryScanner.cs @@ -87,7 +87,9 @@ public class JellyfinMovieLibraryScanner : connectionParameters.Address, connectionParameters.ApiKey, library, - JellyfinItemType.Movie); + library.ItemId, + JellyfinItemType.Movie, + true); protected override IAsyncEnumerable GetMovieLibraryItems( JellyfinConnectionParameters connectionParameters, diff --git a/ErsatzTV.Core/Jellyfin/JellyfinTelevisionLibraryScanner.cs b/ErsatzTV.Core/Jellyfin/JellyfinTelevisionLibraryScanner.cs index 77e3b7015..6b0a048ed 100644 --- a/ErsatzTV.Core/Jellyfin/JellyfinTelevisionLibraryScanner.cs +++ b/ErsatzTV.Core/Jellyfin/JellyfinTelevisionLibraryScanner.cs @@ -77,14 +77,21 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan cancellationToken); } - protected override Task>> GetShowLibraryItems( + protected override Task> CountShowLibraryItems( JellyfinConnectionParameters connectionParameters, - JellyfinLibrary library) => - _jellyfinApiClient.GetShowLibraryItems( + JellyfinLibrary library) + => _jellyfinApiClient.GetLibraryItemCount( connectionParameters.Address, connectionParameters.ApiKey, - library.MediaSourceId, - library.ItemId); + library, + library.ItemId, + JellyfinItemType.Show, + false); + + protected override IAsyncEnumerable GetShowLibraryItems( + JellyfinConnectionParameters connectionParameters, + JellyfinLibrary library) => + _jellyfinApiClient.GetShowLibraryItems(connectionParameters.Address, connectionParameters.ApiKey, library); protected override string MediaServerItemId(JellyfinShow show) => show.ItemId; protected override string MediaServerItemId(JellyfinSeason season) => season.ItemId; @@ -94,19 +101,44 @@ public class JellyfinTelevisionLibraryScanner : MediaServerTelevisionLibraryScan protected override string MediaServerEtag(JellyfinSeason season) => season.Etag; protected override string MediaServerEtag(JellyfinEpisode episode) => episode.Etag; - protected override Task>> GetSeasonLibraryItems( + protected override Task> CountSeasonLibraryItems( + JellyfinConnectionParameters connectionParameters, + JellyfinLibrary library, + JellyfinShow show) => + _jellyfinApiClient.GetLibraryItemCount( + connectionParameters.Address, + connectionParameters.ApiKey, + library, + show.ItemId, + JellyfinItemType.Season, + false); + + protected override IAsyncEnumerable GetSeasonLibraryItems( JellyfinLibrary library, JellyfinConnectionParameters connectionParameters, JellyfinShow show) => _jellyfinApiClient.GetSeasonLibraryItems( connectionParameters.Address, connectionParameters.ApiKey, - library.MediaSourceId, + library, show.ItemId); - protected override Task>> GetEpisodeLibraryItems( + protected override Task> CountEpisodeLibraryItems( + JellyfinConnectionParameters connectionParameters, + JellyfinLibrary library, + JellyfinSeason season) => + _jellyfinApiClient.GetLibraryItemCount( + connectionParameters.Address, + connectionParameters.ApiKey, + library, + season.ItemId, + JellyfinItemType.Episode, + true); + + protected override IAsyncEnumerable GetEpisodeLibraryItems( JellyfinLibrary library, JellyfinConnectionParameters connectionParameters, + JellyfinShow _, JellyfinSeason season) => _jellyfinApiClient.GetEpisodeLibraryItems( connectionParameters.Address, diff --git a/ErsatzTV.Core/Metadata/MediaServerMovieLibraryScanner.cs b/ErsatzTV.Core/Metadata/MediaServerMovieLibraryScanner.cs index 9bb2f605b..bad847636 100644 --- a/ErsatzTV.Core/Metadata/MediaServerMovieLibraryScanner.cs +++ b/ErsatzTV.Core/Metadata/MediaServerMovieLibraryScanner.cs @@ -59,19 +59,23 @@ public abstract class MediaServerMovieLibraryScanner> entries = await GetShowLibraryItems(connectionParameters, library); - - foreach (BaseError error in entries.LeftToSeq()) + Either maybeCount = await CountShowLibraryItems(connectionParameters, library); + foreach (BaseError error in maybeCount.LeftToSeq()) { return error; } - return await ScanLibrary( - televisionRepository, - connectionParameters, - library, - getLocalPath, - ffmpegPath, - ffprobePath, - entries.RightToSeq().Flatten().ToList(), - deepScan, - cancellationToken); + foreach (int count in maybeCount.RightToSeq()) + { + _logger.LogDebug("Library {Library} contains {Count} shows", library.Name, count); + + return await ScanLibrary( + televisionRepository, + connectionParameters, + library, + getLocalPath, + ffmpegPath, + ffprobePath, + GetShowLibraryItems(connectionParameters, library), + count, + deepScan, + cancellationToken); + } + + // this won't happen + return Unit.Default; } catch (Exception ex) when (ex is TaskCanceledException or OperationCanceledException) { @@ -84,7 +92,11 @@ public abstract class MediaServerTelevisionLibraryScanner>> GetShowLibraryItems( + protected abstract Task> CountShowLibraryItems( + TConnectionParameters connectionParameters, + TLibrary library); + + protected abstract IAsyncEnumerable GetShowLibraryItems( TConnectionParameters connectionParameters, TLibrary library); @@ -102,21 +114,24 @@ public abstract class MediaServerTelevisionLibraryScanner getLocalPath, string ffmpegPath, string ffprobePath, - List showEntries, + IAsyncEnumerable showEntries, + int totalShowCount, bool deepScan, CancellationToken cancellationToken) { + var incomingItemIds = new List(); List existingShows = await televisionRepository.GetExistingShows(library); - var sortedShows = showEntries.OrderBy(s => s.ShowMetadata.Head().SortTitle).ToList(); - foreach (TShow incoming in showEntries) + await foreach (TShow incoming in showEntries.WithCancellation(cancellationToken)) { if (cancellationToken.IsCancellationRequested) { return new ScanCanceled(); } - decimal percentCompletion = (decimal)sortedShows.IndexOf(incoming) / sortedShows.Count; + incomingItemIds.Add(MediaServerItemId(incoming)); + + decimal percentCompletion = Math.Clamp((decimal)incomingItemIds.Count / totalShowCount, 0, 1); await _mediator.Publish(new LibraryScanProgress(library.Id, percentCompletion), cancellationToken); Either> maybeShow = await televisionRepository @@ -138,16 +153,23 @@ public abstract class MediaServerTelevisionLibraryScanner result in maybeShow.RightToSeq()) { - Either> entries = await GetSeasonLibraryItems( - library, + Either maybeCount = await CountSeasonLibraryItems( connectionParameters, + library, result.Item); - - foreach (BaseError error in entries.LeftToSeq()) + foreach (BaseError error in maybeCount.LeftToSeq()) { return error; } + foreach (int count in maybeCount.RightToSeq()) + { + _logger.LogDebug( + "Show {Title} contains {Count} seasons", + result.Item.ShowMetadata.Head().Title, + count); + } + Either scanResult = await ScanSeasons( televisionRepository, library, @@ -156,7 +178,7 @@ public abstract class MediaServerTelevisionLibraryScanner s.MediaServerItemId) - .Except(showEntries.Map(MediaServerItemId)).ToList(); + var fileNotFoundItemIds = existingShows.Map(s => s.MediaServerItemId).Except(incomingItemIds).ToList(); List ids = await televisionRepository.FlagFileNotFoundShows(library, fileNotFoundItemIds); await _searchIndex.RebuildItems(_searchRepository, ids); @@ -185,14 +206,25 @@ public abstract class MediaServerTelevisionLibraryScanner>> GetSeasonLibraryItems( + protected abstract Task> CountSeasonLibraryItems( + TConnectionParameters connectionParameters, + TLibrary library, + TShow show); + + protected abstract IAsyncEnumerable GetSeasonLibraryItems( TLibrary library, TConnectionParameters connectionParameters, TShow show); - protected abstract Task>> GetEpisodeLibraryItems( + protected abstract Task> CountEpisodeLibraryItems( + TConnectionParameters connectionParameters, + TLibrary library, + TSeason season); + + protected abstract IAsyncEnumerable GetEpisodeLibraryItems( TLibrary library, TConnectionParameters connectionParameters, + TShow show, TSeason season); protected abstract Task> GetFullMetadata( @@ -236,14 +268,14 @@ public abstract class MediaServerTelevisionLibraryScanner seasonEntries, + IAsyncEnumerable seasonEntries, bool deepScan, CancellationToken cancellationToken) { + var incomingItemIds = new List(); List existingSeasons = await televisionRepository.GetExistingSeasons(library, show); - var sortedSeasons = seasonEntries.OrderBy(s => s.SeasonNumber).ToList(); - foreach (TSeason incoming in sortedSeasons) + await foreach (TSeason incoming in seasonEntries.WithCancellation(cancellationToken)) { incoming.ShowId = show.Id; @@ -252,6 +284,8 @@ public abstract class MediaServerTelevisionLibraryScanner> maybeSeason = await televisionRepository .GetOrAdd(library, incoming) .BindT(existing => UpdateMetadata(connectionParameters, library, existing, incoming, deepScan)); @@ -272,16 +306,24 @@ public abstract class MediaServerTelevisionLibraryScanner result in maybeSeason.RightToSeq()) { - Either> entries = await GetEpisodeLibraryItems( - library, + Either maybeCount = await CountEpisodeLibraryItems( connectionParameters, + library, result.Item); - - foreach (BaseError error in entries.LeftToSeq()) + foreach (BaseError error in maybeCount.LeftToSeq()) { return error; } + foreach (int count in maybeCount.RightToSeq()) + { + _logger.LogDebug( + "Show {Title} season {Season} contains {Count} episodes", + show.ShowMetadata.Head().Title, + result.Item.SeasonNumber, + count); + } + Either scanResult = await ScanEpisodes( televisionRepository, library, @@ -291,7 +333,7 @@ public abstract class MediaServerTelevisionLibraryScanner s.MediaServerItemId) - .Except(seasonEntries.Map(MediaServerItemId)).ToList(); + var fileNotFoundItemIds = existingSeasons.Map(s => s.MediaServerItemId).Except(incomingItemIds).ToList(); List ids = await televisionRepository.FlagFileNotFoundSeasons(library, fileNotFoundItemIds); await _searchIndex.RebuildItems(_searchRepository, ids); @@ -329,20 +370,22 @@ public abstract class MediaServerTelevisionLibraryScanner episodeEntries, + IAsyncEnumerable episodeEntries, bool deepScan, CancellationToken cancellationToken) { + var incomingItemIds = new List(); List existingEpisodes = await televisionRepository.GetExistingEpisodes(library, season); - var sortedEpisodes = episodeEntries.OrderBy(s => s.EpisodeMetadata.Head().EpisodeNumber).ToList(); - foreach (TEpisode incoming in sortedEpisodes) + await foreach (TEpisode incoming in episodeEntries.WithCancellation(cancellationToken)) { if (cancellationToken.IsCancellationRequested) { return new ScanCanceled(); } + incomingItemIds.Add(MediaServerItemId(incoming)); + string localPath = getLocalPath(incoming); if (await ShouldScanItem( televisionRepository, @@ -414,8 +457,7 @@ public abstract class MediaServerTelevisionLibraryScanner m.MediaServerItemId) - .Except(episodeEntries.Map(MediaServerItemId)).ToList(); + var fileNotFoundItemIds = existingEpisodes.Map(m => m.MediaServerItemId).Except(incomingItemIds).ToList(); List ids = await televisionRepository.FlagFileNotFoundEpisodes(library, fileNotFoundItemIds); await _searchIndex.RebuildItems(_searchRepository, ids); diff --git a/ErsatzTV.Core/Plex/PlexTelevisionLibraryScanner.cs b/ErsatzTV.Core/Plex/PlexTelevisionLibraryScanner.cs index 66f0f8cbd..5b8afbaca 100644 --- a/ErsatzTV.Core/Plex/PlexTelevisionLibraryScanner.cs +++ b/ErsatzTV.Core/Plex/PlexTelevisionLibraryScanner.cs @@ -137,7 +137,15 @@ public class PlexTelevisionLibraryScanner : // } // } - protected override Task>> GetShowLibraryItems( + protected override Task> CountShowLibraryItems( + PlexConnectionParameters connectionParameters, + PlexLibrary library) => + _plexServerApiClient.GetLibraryItemCount( + library, + connectionParameters.Connection, + connectionParameters.Token); + + protected override IAsyncEnumerable GetShowLibraryItems( PlexConnectionParameters connectionParameters, PlexLibrary library) => _plexServerApiClient.GetShowLibraryContents( @@ -145,7 +153,16 @@ public class PlexTelevisionLibraryScanner : connectionParameters.Connection, connectionParameters.Token); - protected override Task>> GetSeasonLibraryItems( + protected override Task> CountSeasonLibraryItems( + PlexConnectionParameters connectionParameters, + PlexLibrary library, + PlexShow show) => + _plexServerApiClient.CountShowSeasons( + show, + connectionParameters.Connection, + connectionParameters.Token); + + protected override IAsyncEnumerable GetSeasonLibraryItems( PlexLibrary library, PlexConnectionParameters connectionParameters, PlexShow show) => @@ -155,9 +172,19 @@ public class PlexTelevisionLibraryScanner : connectionParameters.Connection, connectionParameters.Token); - protected override Task>> GetEpisodeLibraryItems( + protected override Task> CountEpisodeLibraryItems( + PlexConnectionParameters connectionParameters, + PlexLibrary library, + PlexSeason season) => + _plexServerApiClient.CountSeasonEpisodes( + season, + connectionParameters.Connection, + connectionParameters.Token); + + protected override IAsyncEnumerable GetEpisodeLibraryItems( PlexLibrary library, PlexConnectionParameters connectionParameters, + PlexShow _, PlexSeason season) => _plexServerApiClient.GetSeasonEpisodes( library, diff --git a/ErsatzTV.FFmpeg.Tests/PipelineBuilderTests.cs b/ErsatzTV.FFmpeg.Tests/PipelineBuilderTests.cs index 7a39d55d1..a4659cc68 100644 --- a/ErsatzTV.FFmpeg.Tests/PipelineBuilderTests.cs +++ b/ErsatzTV.FFmpeg.Tests/PipelineBuilderTests.cs @@ -75,7 +75,8 @@ public class PipelineGeneratorTests result.PipelineSteps.Should().Contain(ps => ps is EncoderLibx265); string command = PrintCommand(videoInputFile, audioInputFile, None, None, result); - command.Should().Be("-threads 1 -nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -ss 00:00:01 -c:v h264 -re -i /tmp/whatever.mkv -map 0:1 -map 0:0 -muxdelay 0 -muxpreload 0 -movflags +faststart -flags cgop -sc_threshold 0 -video_track_timescale 90000 -b:v 2000k -maxrate:v 2000k -bufsize:v 4000k -c:a aac -ac 2 -b:a 320k -maxrate:a 320k -bufsize:a 640k -ar 48k -c:v libx265 -tag:v hvc1 -x265-params log-level=error -f mpegts -mpegts_flags +initial_discontinuity pipe:1"); + command.Should().Be( + "-threads 1 -nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -ss 00:00:01 -c:v h264 -re -i /tmp/whatever.mkv -map 0:1 -map 0:0 -muxdelay 0 -muxpreload 0 -movflags +faststart -flags cgop -sc_threshold 0 -video_track_timescale 90000 -b:v 2000k -maxrate:v 2000k -bufsize:v 4000k -c:a aac -ac 2 -b:a 320k -maxrate:a 320k -bufsize:a 640k -ar 48k -c:v libx265 -tag:v hvc1 -x265-params log-level=error -f mpegts -mpegts_flags +initial_discontinuity pipe:1"); } [Test] diff --git a/ErsatzTV.Infrastructure/AsyncEnumerable.cs b/ErsatzTV.Infrastructure/AsyncEnumerable.cs new file mode 100644 index 000000000..b1696bb14 --- /dev/null +++ b/ErsatzTV.Infrastructure/AsyncEnumerable.cs @@ -0,0 +1,26 @@ +namespace ErsatzTV.Infrastructure; + +public static class AsyncEnumerable +{ + /// + /// Creates an which yields no results, similar to + /// . + /// + public static IAsyncEnumerable Empty() => EmptyAsyncEnumerator.Instance; + + private class EmptyAsyncEnumerator : IAsyncEnumerator, IAsyncEnumerable + { + public static readonly EmptyAsyncEnumerator Instance = new(); + + public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return this; + } + + public T Current => default; + public ValueTask DisposeAsync() => default; + + public ValueTask MoveNextAsync() => new(false); + } +} diff --git a/ErsatzTV.Infrastructure/Emby/EmbyApiClient.cs b/ErsatzTV.Infrastructure/Emby/EmbyApiClient.cs index 03e0f3406..964802599 100644 --- a/ErsatzTV.Infrastructure/Emby/EmbyApiClient.cs +++ b/ErsatzTV.Infrastructure/Emby/EmbyApiClient.cs @@ -71,11 +71,141 @@ public class EmbyApiClient : IEmbyApiClient } } - public async IAsyncEnumerable GetMovieLibraryItems(string address, string apiKey, EmbyLibrary library) + public IAsyncEnumerable GetMovieLibraryItems(string address, string apiKey, EmbyLibrary library) + => GetPagedLibraryContents( + address, + apiKey, + library, + library.ItemId, + EmbyItemType.Movie, + (service, itemId, skip, pageSize) => service.GetMovieLibraryItems( + apiKey, + itemId, + startIndex: skip, + limit: pageSize), + (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToMovie(lib, item)).Flatten()); + + public IAsyncEnumerable GetShowLibraryItems(string address, string apiKey, EmbyLibrary library) + => GetPagedLibraryContents( + address, + apiKey, + library, + library.ItemId, + EmbyItemType.Show, + (service, itemId, skip, pageSize) => service.GetShowLibraryItems( + apiKey, + itemId, + startIndex: skip, + limit: pageSize), + (_, item) => ProjectToShow(item)); + + public IAsyncEnumerable GetSeasonLibraryItems( + string address, + string apiKey, + EmbyLibrary library, + string showId) => GetPagedLibraryContents( + address, + apiKey, + library, + showId, + EmbyItemType.Season, + (service, itemId, skip, pageSize) => service.GetSeasonLibraryItems( + apiKey, + itemId, + startIndex: skip, + limit: pageSize), + (_, item) => ProjectToSeason(item)); + + public IAsyncEnumerable GetEpisodeLibraryItems( + string address, + string apiKey, + EmbyLibrary library, + string showId, + string seasonId) => GetPagedLibraryContents( + address, + apiKey, + library, + seasonId, + EmbyItemType.Episode, + (service, _, skip, pageSize) => service.GetEpisodeLibraryItems( + apiKey, + showId, + seasonId, + startIndex: skip, + limit: pageSize), + (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToEpisode(lib, item)).Flatten()); + + public IAsyncEnumerable GetCollectionLibraryItems(string address, string apiKey) + { + // TODO: should we enumerate collection libraries here? + + if (_memoryCache.TryGetValue("emby_collections_library_item_id", out string itemId)) + { + return GetPagedLibraryContents( + address, + apiKey, + None, + itemId, + EmbyItemType.Collection, + (service, _, skip, pageSize) => service.GetCollectionLibraryItems( + apiKey, + itemId, + startIndex: skip, + limit: pageSize), + (_, item) => ProjectToCollection(item)); + } + + return AsyncEnumerable.Empty(); + } + + public IAsyncEnumerable GetCollectionItems( + string address, + string apiKey, + string collectionId) => + GetPagedLibraryContents( + address, + apiKey, + None, + collectionId, + EmbyItemType.CollectionItems, + (service, _, skip, pageSize) => service.GetCollectionItems( + apiKey, + collectionId, + startIndex: skip, + limit: pageSize), + (_, item) => ProjectToCollectionMediaItem(item)); + + public async Task> GetLibraryItemCount( + string address, + string apiKey, + string parentId, + string includeItemTypes) + { + try + { + IEmbyApi service = RestService.For(address); + EmbyLibraryItemsResponse items = await service.GetLibraryStats(apiKey, parentId, includeItemTypes); + return items.TotalRecordCount; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error getting Emby library item count"); + return BaseError.New(ex.Message); + } + } + + private static async IAsyncEnumerable GetPagedLibraryContents( + string address, + string apiKey, + Option maybeLibrary, + string parentId, + string itemType, + Func> getItems, + Func, EmbyLibraryItemResponse, Option> mapper) { IEmbyApi service = RestService.For(address); int size = await service - .GetLibraryStats(apiKey, library.ItemId, EmbyItemType.Movie) + .GetLibraryStats(apiKey, parentId, itemType) .Map(r => r.TotalRecordCount); const int PAGE_SIZE = 10; @@ -86,146 +216,18 @@ public class EmbyApiClient : IEmbyApiClient { int skip = i * PAGE_SIZE; - Task> result = service - .GetMovieLibraryItems(apiKey, library.ItemId, startIndex: skip, limit: PAGE_SIZE) - .Map(items => items.Items.Map(item => ProjectToMovie(library, item)).Somes()); + Task> result = getItems(service, parentId, skip, PAGE_SIZE) + .Map(items => items.Items.Map(item => mapper(maybeLibrary, item)).Somes()); - foreach (EmbyMovie movie in await result) +#pragma warning disable VSTHRD003 + foreach (TItem item in await result) +#pragma warning restore VSTHRD003 { - yield return movie; + yield return item; } } } - public async Task>> GetShowLibraryItems( - string address, - string apiKey, - string libraryId) - { - try - { - IEmbyApi service = RestService.For(address); - EmbyLibraryItemsResponse items = await service.GetShowLibraryItems(apiKey, libraryId); - return items.Items - .Map(ProjectToShow) - .Somes() - .ToList(); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error getting emby show library items"); - return BaseError.New(ex.Message); - } - } - - public async Task>> GetSeasonLibraryItems( - string address, - string apiKey, - string showId) - { - try - { - IEmbyApi service = RestService.For(address); - EmbyLibraryItemsResponse items = await service.GetSeasonLibraryItems(apiKey, showId); - return items.Items - .Map(ProjectToSeason) - .Somes() - .ToList(); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error getting emby show library items"); - return BaseError.New(ex.Message); - } - } - - public async Task>> GetEpisodeLibraryItems( - string address, - string apiKey, - EmbyLibrary library, - string seasonId) - { - try - { - IEmbyApi service = RestService.For(address); - EmbyLibraryItemsResponse items = await service.GetEpisodeLibraryItems(apiKey, seasonId); - return items.Items - .Map(i => ProjectToEpisode(library, i)) - .Somes() - .ToList(); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error getting emby episode library items"); - return BaseError.New(ex.Message); - } - } - - public async Task>> GetCollectionLibraryItems(string address, string apiKey) - { - try - { - // TODO: should we enumerate collection libraries here? - - if (_memoryCache.TryGetValue("emby_collections_library_item_id", out string itemId)) - { - IEmbyApi service = RestService.For(address); - EmbyLibraryItemsResponse items = await service.GetCollectionLibraryItems(apiKey, itemId); - return items.Items - .Map(ProjectToCollection) - .Somes() - .ToList(); - } - - return BaseError.New("Emby collection item id is not available"); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error getting Emby collection library items"); - return BaseError.New(ex.Message); - } - } - - public async Task>> GetCollectionItems( - string address, - string apiKey, - string collectionId) - { - try - { - IEmbyApi service = RestService.For(address); - EmbyLibraryItemsResponse items = await service.GetCollectionItems(apiKey, collectionId); - return items.Items - .Map(ProjectToCollectionMediaItem) - .Somes() - .ToList(); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error getting Emby collection items"); - return BaseError.New(ex.Message); - } - } - - public async Task> GetLibraryItemCount( - string address, - string apiKey, - EmbyLibrary library, - string includeItemTypes) - { - try - { - IEmbyApi service = RestService.For(address); - EmbyLibraryItemsResponse items = await service.GetLibraryStats(apiKey, library.ItemId, includeItemTypes); - return items.TotalRecordCount; - } - catch (Exception ex) - { - _logger.LogError(ex, "Error getting Emby library item count"); - return BaseError.New(ex.Message); - } - } - private Option ProjectToCollection(EmbyLibraryItemResponse item) { try diff --git a/ErsatzTV.Infrastructure/Emby/IEmbyApi.cs b/ErsatzTV.Infrastructure/Emby/IEmbyApi.cs index 740109a01..cb88f10a0 100644 --- a/ErsatzTV.Infrastructure/Emby/IEmbyApi.cs +++ b/ErsatzTV.Infrastructure/Emby/IEmbyApi.cs @@ -32,7 +32,7 @@ public interface IEmbyApi [Query] int limit = 0); - [Get("/Items")] + [Get("/Items?sortOrder=Ascending&sortBy=SortName")] public Task GetMovieLibraryItems( [Header("X-Emby-Token")] string apiKey, @@ -50,7 +50,7 @@ public interface IEmbyApi [Query] int limit = 0); - [Get("/Items")] + [Get("/Items?sortOrder=Ascending&sortBy=SortName")] public Task GetShowLibraryItems( [Header("X-Emby-Token")] string apiKey, @@ -62,40 +62,40 @@ public interface IEmbyApi [Query] string includeItemTypes = "Series", [Query] - bool recursive = true); + bool recursive = true, + [Query] + int startIndex = 0, + [Query] + int limit = 0); - [Get("/Items")] + [Get("/Shows/{parentId}/Seasons?sortOrder=Ascending&sortBy=SortName")] public Task GetSeasonLibraryItems( [Header("X-Emby-Token")] string apiKey, - [Query] string parentId, [Query] string fields = "Path,DateCreated,Etag,Taglines,ProviderIds", [Query] - string includeItemTypes = "Season", + int startIndex = 0, [Query] - string excludeLocationTypes = "Virtual", - [Query] - bool recursive = true); + int limit = 0); - [Get("/Items")] + [Get("/Shows/{showId}/Episodes?sortOrder=Ascending&sortBy=SortName")] public Task GetEpisodeLibraryItems( [Header("X-Emby-Token")] string apiKey, + string showId, [Query] - string parentId, + string seasonId, [Query] string fields = "Path,DateCreated,Etag,Overview,ProductionYear,PremiereDate,MediaSources,LocationType,ProviderIds,People", [Query] - string includeItemTypes = "Episode", + int startIndex = 0, [Query] - string excludeLocationTypes = "Virtual", - [Query] - bool recursive = true); + int limit = 0); - [Get("/Items")] + [Get("/Items?sortOrder=Ascending&sortBy=SortName")] public Task GetCollectionLibraryItems( [Header("X-Emby-Token")] string apiKey, @@ -106,9 +106,13 @@ public interface IEmbyApi [Query] string includeItemTypes = "BoxSet", [Query] - bool recursive = true); + bool recursive = true, + [Query] + int startIndex = 0, + [Query] + int limit = 0); - [Get("/Items")] + [Get("/Items?sortOrder=Ascending&sortBy=SortName")] public Task GetCollectionItems( [Header("X-Emby-Token")] string apiKey, @@ -121,5 +125,9 @@ public interface IEmbyApi [Query] string excludeLocationTypes = "Virtual", [Query] - bool recursive = true); + bool recursive = true, + [Query] + int startIndex = 0, + [Query] + int limit = 0); } diff --git a/ErsatzTV.Infrastructure/Jellyfin/IJellyfinApi.cs b/ErsatzTV.Infrastructure/Jellyfin/IJellyfinApi.cs index 7ecf6e943..a6cd8a39a 100644 --- a/ErsatzTV.Infrastructure/Jellyfin/IJellyfinApi.cs +++ b/ErsatzTV.Infrastructure/Jellyfin/IJellyfinApi.cs @@ -63,7 +63,7 @@ public interface IJellyfinApi [Query] int limit = 0); - [Get("/Items")] + [Get("/Items?sortOrder=Ascending&sortBy=SortName")] public Task GetShowLibraryItems( [Header("X-Emby-Token")] string apiKey, @@ -77,9 +77,13 @@ public interface IJellyfinApi [Query] string includeItemTypes = "Series", [Query] - bool recursive = true); + bool recursive = true, + [Query] + int startIndex = 0, + [Query] + int limit = 0); - [Get("/Items")] + [Get("/Items?sortOrder=Ascending&sortBy=SortName")] public Task GetSeasonLibraryItems( [Header("X-Emby-Token")] string apiKey, @@ -92,9 +96,13 @@ public interface IJellyfinApi [Query] string includeItemTypes = "Season", [Query] - bool recursive = true); + bool recursive = true, + [Query] + int startIndex = 0, + [Query] + int limit = 0); - [Get("/Items")] + [Get("/Items?sortOrder=Ascending&sortBy=SortName")] public Task GetEpisodeLibraryItems( [Header("X-Emby-Token")] string apiKey, @@ -107,9 +115,13 @@ public interface IJellyfinApi [Query] string includeItemTypes = "Episode", [Query] - bool recursive = true); + bool recursive = true, + [Query] + int startIndex = 0, + [Query] + int limit = 0); - [Get("/Items")] + [Get("/Items?sortOrder=Ascending&sortBy=SortName")] public Task GetCollectionLibraryItems( [Header("X-Emby-Token")] string apiKey, @@ -122,9 +134,13 @@ public interface IJellyfinApi [Query] string includeItemTypes = "BoxSet", [Query] - bool recursive = true); + bool recursive = true, + [Query] + int startIndex = 0, + [Query] + int limit = 0); - [Get("/Items")] + [Get("/Items?sortOrder=Ascending&sortBy=SortName")] public Task GetCollectionItems( [Header("X-Emby-Token")] string apiKey, @@ -137,5 +153,9 @@ public interface IJellyfinApi [Query] string includeItemTypes = "Movie,Series,Season,Episode", [Query] - bool recursive = true); + bool recursive = true, + [Query] + int startIndex = 0, + [Query] + int limit = 0); } diff --git a/ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs b/ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs index 52e6562d9..c901f461a 100644 --- a/ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs +++ b/ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs @@ -91,16 +91,178 @@ public class JellyfinApiClient : IJellyfinApiClient } } - public async IAsyncEnumerable GetMovieLibraryItems( + public IAsyncEnumerable GetMovieLibraryItems( string address, string apiKey, - JellyfinLibrary library) + JellyfinLibrary library) => + GetPagedLibraryItems( + address, + apiKey, + library, + library.MediaSourceId, + library.ItemId, + JellyfinItemType.Movie, + (service, userId, itemId, skip, pageSize) => service.GetMovieLibraryItems( + apiKey, + userId, + itemId, + startIndex: skip, + limit: pageSize), + (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToMovie(lib, item)).Flatten()); + + public IAsyncEnumerable GetShowLibraryItems( + string address, + string apiKey, + JellyfinLibrary library) => + GetPagedLibraryItems( + address, + apiKey, + library, + library.MediaSourceId, + library.ItemId, + JellyfinItemType.Show, + (service, userId, itemId, skip, pageSize) => service.GetShowLibraryItems( + apiKey, + userId, + itemId, + startIndex: skip, + limit: pageSize), + (_, item) => ProjectToShow(item)); + + public IAsyncEnumerable GetSeasonLibraryItems( + string address, + string apiKey, + JellyfinLibrary library, + string showId) => + GetPagedLibraryItems( + address, + apiKey, + library, + library.MediaSourceId, + showId, + JellyfinItemType.Season, + (service, userId, _, skip, pageSize) => service.GetSeasonLibraryItems( + apiKey, + userId, + showId, + startIndex: skip, + limit: pageSize), + (_, item) => ProjectToSeason(item)); + + public IAsyncEnumerable GetEpisodeLibraryItems( + string address, + string apiKey, + JellyfinLibrary library, + string seasonId) => + GetPagedLibraryItems( + address, + apiKey, + library, + library.MediaSourceId, + seasonId, + JellyfinItemType.Episode, + (service, userId, _, skip, pageSize) => service.GetEpisodeLibraryItems( + apiKey, + userId, + seasonId, + startIndex: skip, + limit: pageSize), + (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToEpisode(lib, item)).Flatten()); + + public IAsyncEnumerable GetCollectionLibraryItems( + string address, + string apiKey, + int mediaSourceId) { - if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{library.MediaSourceId}", out string userId)) + // TODO: should we enumerate collection libraries here? + + if (_memoryCache.TryGetValue("jellyfin_collections_library_item_id", out string itemId)) + { + return GetPagedLibraryItems( + address, + apiKey, + None, + mediaSourceId, + itemId, + JellyfinItemType.Collection, + (service, userId, _, skip, pageSize) => service.GetCollectionLibraryItems( + apiKey, + userId, + itemId, + startIndex: skip, + limit: pageSize), + (_, item) => ProjectToCollection(item)); + } + + return AsyncEnumerable.Empty(); + } + + public IAsyncEnumerable GetCollectionItems( + string address, + string apiKey, + int mediaSourceId, + string collectionId) => + GetPagedLibraryItems( + address, + apiKey, + None, + mediaSourceId, + collectionId, + JellyfinItemType.CollectionItems, + (service, userId, _, skip, pageSize) => service.GetCollectionItems( + apiKey, + userId, + collectionId, + startIndex: skip, + limit: pageSize), + (_, item) => ProjectToCollectionMediaItem(item)); + + public async Task> GetLibraryItemCount( + string address, + string apiKey, + JellyfinLibrary library, + string parentId, + string includeItemTypes, + bool excludeFolders) + { + try + { + if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{library.MediaSourceId}", out string userId)) + { + IJellyfinApi service = RestService.For(address); + JellyfinLibraryItemsResponse items = await service.GetLibraryStats( + apiKey, + userId, + parentId, + includeItemTypes, + filters: excludeFolders ? "IsNotFolder" : null); + return items.TotalRecordCount; + } + + return BaseError.New("Jellyfin admin user id is not available"); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error getting jellyfin library item count"); + return BaseError.New(ex.Message); + } + } + + private async IAsyncEnumerable GetPagedLibraryItems( + string address, + string apiKey, + Option maybeLibrary, + int mediaSourceId, + string parentId, + string itemType, + Func> getItems, + Func, JellyfinLibraryItemResponse, Option> mapper) + { + if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{mediaSourceId}", out string userId)) { IJellyfinApi service = RestService.For(address); int size = await service - .GetLibraryStats(apiKey, userId, library.ItemId, JellyfinItemType.Movie) + .GetLibraryStats(apiKey, userId, parentId, itemType) .Map(r => r.TotalRecordCount); const int PAGE_SIZE = 10; @@ -111,191 +273,17 @@ public class JellyfinApiClient : IJellyfinApiClient { int skip = i * PAGE_SIZE; - Task> result = service - .GetMovieLibraryItems(apiKey, userId, library.ItemId, startIndex: skip, limit: PAGE_SIZE) - .Map(items => items.Items.Map(item => ProjectToMovie(library, item)).Somes()); + Task> result = getItems(service, userId, parentId, skip, PAGE_SIZE) + .Map(items => items.Items.Map(item => mapper(maybeLibrary, item)).Somes()); - foreach (JellyfinMovie movie in await result) + foreach (TItem item in await result) { - yield return movie; + yield return item; } } } } - public async Task>> GetShowLibraryItems( - string address, - string apiKey, - int mediaSourceId, - string libraryId) - { - try - { - if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{mediaSourceId}", out string userId)) - { - IJellyfinApi service = RestService.For(address); - JellyfinLibraryItemsResponse items = await service.GetShowLibraryItems(apiKey, userId, libraryId); - return items.Items - .Map(ProjectToShow) - .Somes() - .ToList(); - } - - return BaseError.New("Jellyfin admin user id is not available"); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error getting jellyfin show library items"); - return BaseError.New(ex.Message); - } - } - - public async Task>> GetSeasonLibraryItems( - string address, - string apiKey, - int mediaSourceId, - string showId) - { - try - { - if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{mediaSourceId}", out string userId)) - { - IJellyfinApi service = RestService.For(address); - JellyfinLibraryItemsResponse items = await service.GetSeasonLibraryItems(apiKey, userId, showId); - return items.Items - .Map(ProjectToSeason) - .Somes() - .ToList(); - } - - return BaseError.New("Jellyfin admin user id is not available"); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error getting jellyfin show library items"); - return BaseError.New(ex.Message); - } - } - - public async Task>> GetEpisodeLibraryItems( - string address, - string apiKey, - JellyfinLibrary library, - string seasonId) - { - try - { - if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{library.MediaSourceId}", out string userId)) - { - IJellyfinApi service = RestService.For(address); - JellyfinLibraryItemsResponse items = await service.GetEpisodeLibraryItems(apiKey, userId, seasonId); - return items.Items - .Map(i => ProjectToEpisode(library, i)) - .Somes() - .ToList(); - } - - return BaseError.New("Jellyfin admin user id is not available"); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error getting jellyfin episode library items"); - return BaseError.New(ex.Message); - } - } - - public async Task>> GetCollectionLibraryItems( - string address, - string apiKey, - int mediaSourceId) - { - try - { - if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{mediaSourceId}", out string userId)) - { - // TODO: should we enumerate collection libraries here? - - if (_memoryCache.TryGetValue("jellyfin_collections_library_item_id", out string itemId)) - { - IJellyfinApi service = RestService.For(address); - JellyfinLibraryItemsResponse items = - await service.GetCollectionLibraryItems(apiKey, userId, itemId); - return items.Items - .Map(ProjectToCollection) - .Somes() - .ToList(); - } - - return BaseError.New("Jellyfin collection item id is not available"); - } - - return BaseError.New("Jellyfin admin user id is not available"); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error getting jellyfin collection library items"); - return BaseError.New(ex.Message); - } - } - - public async Task>> GetCollectionItems( - string address, - string apiKey, - int mediaSourceId, - string collectionId) - { - try - { - if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{mediaSourceId}", out string userId)) - { - IJellyfinApi service = RestService.For(address); - JellyfinLibraryItemsResponse items = await service.GetCollectionItems( - apiKey, - userId, - collectionId); - return items.Items - .Map(ProjectToCollectionMediaItem) - .Somes() - .ToList(); - } - - return BaseError.New("Jellyfin admin user id is not available"); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error getting jellyfin collection items"); - return BaseError.New(ex.Message); - } - } - - public async Task> GetLibraryItemCount( - string address, - string apiKey, - JellyfinLibrary library, - string includeItemTypes) - { - try - { - if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{library.MediaSourceId}", out string userId)) - { - IJellyfinApi service = RestService.For(address); - JellyfinLibraryItemsResponse items = await service.GetLibraryStats( - apiKey, - userId, - library.ItemId, - includeItemTypes); - return items.TotalRecordCount; - } - - return BaseError.New("Jellyfin admin user id is not available"); - } - catch (Exception ex) - { - _logger.LogError(ex, "Error getting jellyfin library item count"); - return BaseError.New(ex.Message); - } - } - private Option ProjectToCollectionMediaItem(JellyfinLibraryItemResponse item) { try diff --git a/ErsatzTV.Infrastructure/Plex/IPlexServerApi.cs b/ErsatzTV.Infrastructure/Plex/IPlexServerApi.cs index 154c22c6d..491e25cb0 100644 --- a/ErsatzTV.Infrastructure/Plex/IPlexServerApi.cs +++ b/ErsatzTV.Infrastructure/Plex/IPlexServerApi.cs @@ -24,14 +24,6 @@ public interface IPlexServerApi [Query] [AliasAs("X-Plex-Token")] string token); - [Get("/library/sections/{key}/all")] - [Headers("Accept: application/json")] - public Task>> - GetLibrarySectionContents( - string key, - [Query] [AliasAs("X-Plex-Token")] - string token); - [Get("/library/sections/{key}/all")] [Headers("Accept: application/json")] public Task>> @@ -60,19 +52,41 @@ public interface IPlexServerApi [Query] [AliasAs("X-Plex-Token")] string token); + [Get("/library/metadata/{key}/children?X-Plex-Container-Start=0&X-Plex-Container-Size=0")] + [Headers("Accept: text/xml")] + public Task CountShowChildren( + string key, + [Query] [AliasAs("X-Plex-Token")] + string token); + [Get("/library/metadata/{key}/children")] [Headers("Accept: text/xml")] public Task GetShowChildren( string key, + [Query] [AliasAs("X-Plex-Container-Start")] + int skip, + [Query] [AliasAs("X-Plex-Container-Size")] + int take, [Query] [AliasAs("X-Plex-Token")] string token); + [Get("/library/metadata/{key}/children?X-Plex-Container-Start=0&X-Plex-Container-Size=0")] + [Headers("Accept: text/xml")] + public Task CountSeasonChildren( + string key, + [Query] [AliasAs("X-Plex-Token")] + string token); + [Get("/library/metadata/{key}/children")] [Headers("Accept: text/xml")] public Task GetSeasonChildren( string key, + [Query] [AliasAs("X-Plex-Container-Start")] + int skip, + [Query] [AliasAs("X-Plex-Container-Size")] + int take, [Query] [AliasAs("X-Plex-Token")] string token); } diff --git a/ErsatzTV.Infrastructure/Plex/PlexEtag.cs b/ErsatzTV.Infrastructure/Plex/PlexEtag.cs index df009f97c..88f8e6000 100644 --- a/ErsatzTV.Infrastructure/Plex/PlexEtag.cs +++ b/ErsatzTV.Infrastructure/Plex/PlexEtag.cs @@ -8,10 +8,8 @@ public class PlexEtag { private readonly RecyclableMemoryStreamManager _recyclableMemoryStreamManager; - public PlexEtag(RecyclableMemoryStreamManager recyclableMemoryStreamManager) - { + public PlexEtag(RecyclableMemoryStreamManager recyclableMemoryStreamManager) => _recyclableMemoryStreamManager = recyclableMemoryStreamManager; - } public string ForMovie(PlexMetadataResponse response) { diff --git a/ErsatzTV.Infrastructure/Plex/PlexServerApiClient.cs b/ErsatzTV.Infrastructure/Plex/PlexServerApiClient.cs index 3da850e44..43aae1ad3 100644 --- a/ErsatzTV.Infrastructure/Plex/PlexServerApiClient.cs +++ b/ErsatzTV.Infrastructure/Plex/PlexServerApiClient.cs @@ -74,67 +74,58 @@ public class PlexServerApiClient : IPlexServerApiClient } } - public async IAsyncEnumerable GetMovieLibraryContents( + public IAsyncEnumerable GetMovieLibraryContents( PlexLibrary library, PlexConnection connection, PlexServerAuthToken token) { - IPlexServerApi xmlService = XmlServiceFor(connection.Uri); - int size = await xmlService.GetLibrarySection(library.Key, token.AuthToken).Map(r => r.TotalSize); - - const int PAGE_SIZE = 10; - - IPlexServerApi service = RestService.For(connection.Uri); - int pages = (size - 1) / PAGE_SIZE + 1; - - for (var i = 0; i < pages; i++) + Task CountItems(IPlexServerApi service) { - int skip = i * PAGE_SIZE; + return service.GetLibrarySection(library.Key, token.AuthToken); + } - Task> result = service - .GetLibrarySectionContents(library.Key, skip, PAGE_SIZE, token.AuthToken) + Task> GetItems(IPlexServerApi _, IPlexServerApi jsonService, int skip, int pageSize) + { + return jsonService + .GetLibrarySectionContents(library.Key, skip, pageSize, token.AuthToken) .Map(r => r.MediaContainer.Metadata.Filter(m => m.Media.Count > 0 && m.Media[0].Part.Count > 0)) .Map(list => list.Map(metadata => ProjectToMovie(metadata, library.MediaSourceId))); - - foreach (PlexMovie movie in await result) - { - yield return movie; - } } + + return GetPagedLibraryContents(connection, CountItems, GetItems); } - public async Task>> GetShowLibraryContents( + public IAsyncEnumerable GetShowLibraryContents( PlexLibrary library, PlexConnection connection, PlexServerAuthToken token) { - try + Task CountItems(IPlexServerApi service) { - IPlexServerApi service = RestService.For(connection.Uri); - return await service.GetLibrarySectionContents(library.Key, token.AuthToken) + return service.GetLibrarySection(library.Key, token.AuthToken); + } + + Task> GetItems(IPlexServerApi _, IPlexServerApi jsonService, int skip, int pageSize) + { + return jsonService + .GetLibrarySectionContents(library.Key, skip, pageSize, token.AuthToken) .Map(r => r.MediaContainer.Metadata) - .Map( - list => (list ?? new List()) - .Map(metadata => ProjectToShow(metadata, library.MediaSourceId)).ToList()); - } - catch (Exception ex) - { - return BaseError.New(ex.ToString()); + .Map(list => list.Map(metadata => ProjectToShow(metadata, library.MediaSourceId))); } + + return GetPagedLibraryContents(connection, CountItems, GetItems); } - public async Task>> GetShowSeasons( - PlexLibrary library, + public async Task> CountShowSeasons( PlexShow show, PlexConnection connection, PlexServerAuthToken token) { try { + string showMetadataKey = show.Key.Split("/").Reverse().Skip(1).Head(); IPlexServerApi service = XmlServiceFor(connection.Uri); - return await service.GetShowChildren(show.Key.Split("/").Reverse().Skip(1).Head(), token.AuthToken) - .Map(r => r.Metadata.Filter(m => !m.Key.Contains("allLeaves"))) - .Map(list => list.Map(metadata => ProjectToSeason(metadata, library.MediaSourceId)).ToList()); + return await service.CountShowChildren(showMetadataKey, token.AuthToken).Map(r => r.TotalSize); } catch (Exception ex) { @@ -142,19 +133,39 @@ public class PlexServerApiClient : IPlexServerApiClient } } - public async Task>> GetSeasonEpisodes( + public IAsyncEnumerable GetShowSeasons( PlexLibrary library, + PlexShow show, + PlexConnection connection, + PlexServerAuthToken token) + { + string showMetadataKey = show.Key.Split("/").Reverse().Skip(1).Head(); + + Task CountItems(IPlexServerApi service) + { + return service.CountShowChildren(showMetadataKey, token.AuthToken); + } + + Task> GetItems(IPlexServerApi xmlService, IPlexServerApi _, int skip, int pageSize) + { + return xmlService.GetShowChildren(showMetadataKey, skip, pageSize, token.AuthToken) + .Map(r => r.Metadata.Filter(m => !m.Key.Contains("allLeaves"))) + .Map(list => list.Map(metadata => ProjectToSeason(metadata, library.MediaSourceId))); + } + + return GetPagedLibraryContents(connection, CountItems, GetItems); + } + + public async Task> CountSeasonEpisodes( PlexSeason season, PlexConnection connection, PlexServerAuthToken token) { try { + string seasonMetadataKey = season.Key.Split("/").Reverse().Skip(1).Head(); IPlexServerApi service = XmlServiceFor(connection.Uri); - return await service.GetSeasonChildren(season.Key.Split("/").Reverse().Skip(1).Head(), token.AuthToken) - .Map(r => r.Metadata.Filter(m => m.Media.Count > 0 && m.Media[0].Part.Count > 0)) - .Map(list => list.Map(metadata => ProjectToEpisode(metadata, library.MediaSourceId))) - .Map(ProcessMultiEpisodeFiles); + return await service.CountSeasonChildren(seasonMetadataKey, token.AuthToken).Map(r => r.TotalSize); } catch (Exception ex) { @@ -162,6 +173,29 @@ public class PlexServerApiClient : IPlexServerApiClient } } + public IAsyncEnumerable GetSeasonEpisodes( + PlexLibrary library, + PlexSeason season, + PlexConnection connection, + PlexServerAuthToken token) + { + string seasonMetadataKey = season.Key.Split("/").Reverse().Skip(1).Head(); + + Task CountItems(IPlexServerApi service) + { + return service.CountSeasonChildren(seasonMetadataKey, token.AuthToken); + } + + Task> GetItems(IPlexServerApi xmlService, IPlexServerApi _, int skip, int pageSize) + { + return xmlService.GetSeasonChildren(seasonMetadataKey, skip, pageSize, token.AuthToken) + .Map(r => r.Metadata.Filter(m => m.Media.Count > 0 && m.Media[0].Part.Count > 0)) + .Map(list => list.Map(metadata => ProjectToEpisode(metadata, library.MediaSourceId))); + } + + return GetPagedLibraryContents(connection, CountItems, GetItems); + } + public async Task> GetMovieMetadata( PlexLibrary library, string key, @@ -279,7 +313,34 @@ public class PlexServerApiClient : IPlexServerApiClient } } + private async IAsyncEnumerable GetPagedLibraryContents( + PlexConnection connection, + Func> countItems, + Func>> getItems) + { + IPlexServerApi xmlService = XmlServiceFor(connection.Uri); + int size = await countItems(xmlService).Map(r => r.TotalSize); + + const int PAGE_SIZE = 10; + + IPlexServerApi jsonService = RestService.For(connection.Uri); + int pages = (size - 1) / PAGE_SIZE + 1; + + for (var i = 0; i < pages; i++) + { + int skip = i * PAGE_SIZE; + + Task> result = getItems(xmlService, jsonService, skip, PAGE_SIZE); + + foreach (TItem item in await result) + { + yield return item; + } + } + } + + // TODO: fix this with the addition of paging private List ProcessMultiEpisodeFiles(IEnumerable episodes) { // add all metadata from duplicate paths to first entry with given path