update media server scanning and paging (#1770)

* update media server scanning and paging

* remove unused types

ErsatzTV.Infrastructure/Emby/EmbyApiClient.cs

@@ -69,13 +69,11 @@ public class EmbyApiClient : IEmbyApiClient
         }
     }
 
-    public IAsyncEnumerable<EmbyMovie> GetMovieLibraryItems(string address, string apiKey, EmbyLibrary library)
+    public IAsyncEnumerable<Tuple<EmbyMovie, int>> GetMovieLibraryItems(string address, string apiKey, EmbyLibrary library)
         => GetPagedLibraryContents(
             address,
-            apiKey,
             library,
             library.ItemId,
-            EmbyItemType.Movie,
             (service, itemId, skip, pageSize) => service.GetMovieLibraryItems(
                 apiKey,
                 itemId,
@@ -83,13 +81,11 @@ public class EmbyApiClient : IEmbyApiClient
                 limit: pageSize),
             (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToMovie(lib, item)).Flatten());
 
-    public IAsyncEnumerable<EmbyShow> GetShowLibraryItems(string address, string apiKey, EmbyLibrary library)
+    public IAsyncEnumerable<Tuple<EmbyShow, int>> GetShowLibraryItems(string address, string apiKey, EmbyLibrary library)
         => GetPagedLibraryContents(
             address,
-            apiKey,
             library,
             library.ItemId,
-            EmbyItemType.Show,
             (service, itemId, skip, pageSize) => service.GetShowLibraryItems(
                 apiKey,
                 itemId,
@@ -97,16 +93,14 @@ public class EmbyApiClient : IEmbyApiClient
                 limit: pageSize),
             (_, item) => ProjectToShow(item));
 
-    public IAsyncEnumerable<EmbySeason> GetSeasonLibraryItems(
+    public IAsyncEnumerable<Tuple<EmbySeason, int>> GetSeasonLibraryItems(
         string address,
         string apiKey,
         EmbyLibrary library,
         string showId) => GetPagedLibraryContents(
         address,
-        apiKey,
         library,
         showId,
-        EmbyItemType.Season,
         (service, itemId, skip, pageSize) => service.GetSeasonLibraryItems(
             apiKey,
             itemId,
@@ -114,17 +108,15 @@ public class EmbyApiClient : IEmbyApiClient
             limit: pageSize),
         (_, item) => ProjectToSeason(item));
 
-    public IAsyncEnumerable<EmbyEpisode> GetEpisodeLibraryItems(
+    public IAsyncEnumerable<Tuple<EmbyEpisode, int>> GetEpisodeLibraryItems(
         string address,
         string apiKey,
         EmbyLibrary library,
         string showId,
         string seasonId) => GetPagedLibraryContents(
         address,
-        apiKey,
         library,
         seasonId,
-        EmbyItemType.Episode,
         (service, _, skip, pageSize) => service.GetEpisodeLibraryItems(
             apiKey,
             showId,
@@ -133,7 +125,7 @@ public class EmbyApiClient : IEmbyApiClient
             limit: pageSize),
         (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToEpisode(lib, item)).Flatten());
 
-    public IAsyncEnumerable<EmbyCollection> GetCollectionLibraryItems(string address, string apiKey)
+    public IAsyncEnumerable<Tuple<EmbyCollection, int>> GetCollectionLibraryItems(string address, string apiKey)
     {
         // TODO: should we enumerate collection libraries here?
 
@@ -141,10 +133,8 @@ public class EmbyApiClient : IEmbyApiClient
         {
             return GetPagedLibraryContents(
                 address,
-                apiKey,
                 None,
                 itemId,
-                EmbyItemType.Collection,
                 (service, _, skip, pageSize) => service.GetCollectionLibraryItems(
                     apiKey,
                     itemId,
@@ -153,19 +143,17 @@ public class EmbyApiClient : IEmbyApiClient
                 (_, item) => ProjectToCollection(item));
         }
 
-        return AsyncEnumerable.Empty<EmbyCollection>();
+        return AsyncEnumerable.Empty<Tuple<EmbyCollection, int>>();
     }
 
-    public IAsyncEnumerable<MediaItem> GetCollectionItems(
+    public IAsyncEnumerable<Tuple<MediaItem, int>> GetCollectionItems(
         string address,
         string apiKey,
         string collectionId) =>
         GetPagedLibraryContents(
             address,
-            apiKey,
             None,
             collectionId,
-            EmbyItemType.CollectionItems,
             (service, _, skip, pageSize) => service.GetCollectionItems(
                 apiKey,
                 collectionId,
@@ -173,25 +161,6 @@ public class EmbyApiClient : IEmbyApiClient
                 limit: pageSize),
             (_, item) => ProjectToCollectionMediaItem(item));
 
-    public async Task<Either<BaseError, int>> GetLibraryItemCount(
-        string address,
-        string apiKey,
-        string parentId,
-        string includeItemTypes)
-    {
-        try
-        {
-            IEmbyApi service = RestService.For<IEmbyApi>(address);
-            EmbyLibraryItemsResponse items = await service.GetLibraryStats(apiKey, parentId, includeItemTypes);
-            return items.TotalRecordCount;
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError(ex, "Error getting Emby library item count");
-            return BaseError.New(ex.Message);
-        }
-    }
-
     public async Task<Either<BaseError, MediaVersion>> GetPlaybackInfo(
         string address,
         string apiKey,
@@ -212,36 +181,31 @@ public class EmbyApiClient : IEmbyApiClient
         }
     }
 
-    private static async IAsyncEnumerable<TItem> GetPagedLibraryContents<TItem>(
+    private static async IAsyncEnumerable<Tuple<TItem, int>> GetPagedLibraryContents<TItem>(
         string address,
-        string apiKey,
         Option<EmbyLibrary> maybeLibrary,
         string parentId,
-        string itemType,
         Func<IEmbyApi, string, int, int, Task<EmbyLibraryItemsResponse>> getItems,
         Func<Option<EmbyLibrary>, EmbyLibraryItemResponse, Option<TItem>> mapper)
     {
         IEmbyApi service = RestService.For<IEmbyApi>(address);
-        int size = await service
-            .GetLibraryStats(apiKey, parentId, itemType)
-            .Map(r => r.TotalRecordCount);
-
         const int PAGE_SIZE = 10;
 
-        int pages = (size - 1) / PAGE_SIZE + 1;
-
+        int pages = int.MaxValue;
         for (var i = 0; i < pages; i++)
         {
             int skip = i * PAGE_SIZE;
 
-            Task<IEnumerable<TItem>> result = getItems(service, parentId, skip, PAGE_SIZE)
-                .Map(items => items.Items.Map(item => mapper(maybeLibrary, item)).Somes());
+            EmbyLibraryItemsResponse result = await getItems(service, parentId, skip, PAGE_SIZE);
+
+            // update page count
+            pages = Math.Min(pages, (result.TotalRecordCount - 1) / PAGE_SIZE + 1);
 
 #pragma warning disable VSTHRD003
-            foreach (TItem item in await result)
+            foreach (TItem item in result.Items.Map(item => mapper(maybeLibrary, item)).Somes())
 #pragma warning restore VSTHRD003
             {
-                yield return item;
+                yield return new Tuple<TItem, int>(item, result.TotalRecordCount);
             }
         }
     }

ErsatzTV.Infrastructure/Emby/IEmbyApi.cs

@@ -17,21 +17,6 @@ public interface IEmbyApi
         [Header("X-Emby-Token")]
         string apiKey);
 
-    [Get("/Items")]
-    public Task<EmbyLibraryItemsResponse> GetLibraryStats(
-        [Header("X-Emby-Token")]
-        string apiKey,
-        [Query]
-        string parentId,
-        [Query]
-        string includeItemTypes,
-        [Query]
-        bool recursive = true,
-        [Query]
-        int startIndex = 0,
-        [Query]
-        int limit = 0);
-
     [Get("/Items?sortOrder=Ascending&sortBy=SortName")]
     public Task<EmbyLibraryItemsResponse> GetMovieLibraryItems(
         [Header("X-Emby-Token")]

ErsatzTV.Infrastructure/Emby/Models/EmbyItemsCountsResponse.cs

@@ -0,0 +1,8 @@
+namespace ErsatzTV.Infrastructure.Emby.Models;
+
+public class EmbyItemsCountsResponse
+{
+    public int MovieCount { get; set; }
+    public int SeriesCount { get; set; }
+    public int EpisodeCount { get; set; }
+}

ErsatzTV.Infrastructure/Jellyfin/IJellyfinApi.cs

@@ -22,25 +22,6 @@ public interface IJellyfinApi
         [Header("X-Emby-Token")]
         string apiKey);
 
-    [Get("/Items")]
-    public Task<JellyfinLibraryItemsResponse> GetLibraryStats(
-        [Header("X-Emby-Token")]
-        string apiKey,
-        [Query]
-        string userId,
-        [Query]
-        string parentId,
-        [Query]
-        string includeItemTypes,
-        [Query]
-        bool recursive = true,
-        [Query]
-        string filters = "IsNotFolder",
-        [Query]
-        int startIndex = 0,
-        [Query]
-        int limit = 0);
-
     [Get("/Items?sortOrder=Ascending&sortBy=SortName")]
     public Task<JellyfinLibraryItemsResponse> GetMovieLibraryItems(
         [Header("X-Emby-Token")]

ErsatzTV.Infrastructure/Jellyfin/JellyfinApiClient.cs

@@ -93,17 +93,15 @@ public class JellyfinApiClient : IJellyfinApiClient
         }
     }
 
-    public IAsyncEnumerable<JellyfinMovie> GetMovieLibraryItems(
+    public IAsyncEnumerable<Tuple<JellyfinMovie, int>> GetMovieLibraryItems(
         string address,
         string apiKey,
         JellyfinLibrary library) =>
         GetPagedLibraryItems(
             address,
-            apiKey,
             library,
             library.MediaSourceId,
             library.ItemId,
-            JellyfinItemType.Movie,
             (service, userId, itemId, skip, pageSize) => service.GetMovieLibraryItems(
                 apiKey,
                 userId,
@@ -112,17 +110,15 @@ public class JellyfinApiClient : IJellyfinApiClient
                 limit: pageSize),
             (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToMovie(lib, item)).Flatten());
 
-    public IAsyncEnumerable<JellyfinShow> GetShowLibraryItems(
+    public IAsyncEnumerable<Tuple<JellyfinShow, int>> GetShowLibraryItems(
         string address,
         string apiKey,
         JellyfinLibrary library) =>
         GetPagedLibraryItems(
             address,
-            apiKey,
             library,
             library.MediaSourceId,
             library.ItemId,
-            JellyfinItemType.Show,
             (service, userId, itemId, skip, pageSize) => service.GetShowLibraryItems(
                 apiKey,
                 userId,
@@ -131,18 +127,16 @@ public class JellyfinApiClient : IJellyfinApiClient
                 limit: pageSize),
             (_, item) => ProjectToShow(item));
 
-    public IAsyncEnumerable<JellyfinSeason> GetSeasonLibraryItems(
+    public IAsyncEnumerable<Tuple<JellyfinSeason, int>> GetSeasonLibraryItems(
         string address,
         string apiKey,
         JellyfinLibrary library,
         string showId) =>
         GetPagedLibraryItems(
             address,
-            apiKey,
             library,
             library.MediaSourceId,
             showId,
-            JellyfinItemType.Season,
             (service, userId, _, skip, pageSize) => service.GetSeasonLibraryItems(
                 apiKey,
                 userId,
@@ -151,18 +145,16 @@ public class JellyfinApiClient : IJellyfinApiClient
                 limit: pageSize),
             (_, item) => ProjectToSeason(item));
 
-    public IAsyncEnumerable<JellyfinEpisode> GetEpisodeLibraryItems(
+    public IAsyncEnumerable<Tuple<JellyfinEpisode, int>> GetEpisodeLibraryItems(
         string address,
         string apiKey,
         JellyfinLibrary library,
         string seasonId) =>
         GetPagedLibraryItems(
             address,
-            apiKey,
             library,
             library.MediaSourceId,
             seasonId,
-            JellyfinItemType.Episode,
             (service, userId, _, skip, pageSize) => service.GetEpisodeLibraryItems(
                 apiKey,
                 userId,
@@ -171,7 +163,7 @@ public class JellyfinApiClient : IJellyfinApiClient
                 limit: pageSize),
             (maybeLibrary, item) => maybeLibrary.Map(lib => ProjectToEpisode(lib, item)).Flatten());
 
-    public IAsyncEnumerable<JellyfinCollection> GetCollectionLibraryItems(
+    public IAsyncEnumerable<Tuple<JellyfinCollection, int>> GetCollectionLibraryItems(
         string address,
         string apiKey,
         int mediaSourceId)
@@ -182,11 +174,9 @@ public class JellyfinApiClient : IJellyfinApiClient
         {
             return GetPagedLibraryItems(
                 address,
-                apiKey,
                 None,
                 mediaSourceId,
                 itemId,
-                JellyfinItemType.Collection,
                 (service, userId, _, skip, pageSize) => service.GetCollectionLibraryItems(
                     apiKey,
                     userId,
@@ -196,21 +186,19 @@ public class JellyfinApiClient : IJellyfinApiClient
                 (_, item) => ProjectToCollection(item));
         }
 
-        return AsyncEnumerable.Empty<JellyfinCollection>();
+        return AsyncEnumerable.Empty<Tuple<JellyfinCollection, int>>();
     }
 
-    public IAsyncEnumerable<MediaItem> GetCollectionItems(
+    public IAsyncEnumerable<Tuple<MediaItem, int>> GetCollectionItems(
         string address,
         string apiKey,
         int mediaSourceId,
         string collectionId) =>
         GetPagedLibraryItems(
             address,
-            apiKey,
             None,
             mediaSourceId,
             collectionId,
-            JellyfinItemType.CollectionItems,
             (service, userId, _, skip, pageSize) => service.GetCollectionItems(
                 apiKey,
                 userId,
@@ -219,37 +207,6 @@ public class JellyfinApiClient : IJellyfinApiClient
                 limit: pageSize),
             (_, item) => ProjectToCollectionMediaItem(item));
 
-    public async Task<Either<BaseError, int>> GetLibraryItemCount(
-        string address,
-        string apiKey,
-        JellyfinLibrary library,
-        string parentId,
-        string includeItemTypes,
-        bool excludeFolders)
-    {
-        try
-        {
-            if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{library.MediaSourceId}", out string userId))
-            {
-                IJellyfinApi service = RestService.For<IJellyfinApi>(address);
-                JellyfinLibraryItemsResponse items = await service.GetLibraryStats(
-                    apiKey,
-                    userId,
-                    parentId,
-                    includeItemTypes,
-                    filters: excludeFolders ? "IsNotFolder" : null);
-                return items.TotalRecordCount;
-            }
-
-            return BaseError.New("Jellyfin admin user id is not available");
-        }
-        catch (Exception ex)
-        {
-            _logger.LogError(ex, "Error getting jellyfin library item count");
-            return BaseError.New(ex.Message);
-        }
-    }
-
     public async Task<Either<BaseError, MediaVersion>> GetPlaybackInfo(
         string address,
         string apiKey,
@@ -275,40 +232,33 @@ public class JellyfinApiClient : IJellyfinApiClient
         }
     }
 
-    private async IAsyncEnumerable<TItem> GetPagedLibraryItems<TItem>(
+    private async IAsyncEnumerable<Tuple<TItem, int>> GetPagedLibraryItems<TItem>(
         string address,
-        string apiKey,
         Option<JellyfinLibrary> maybeLibrary,
         int mediaSourceId,
         string parentId,
-        string itemType,
         Func<IJellyfinApi, string, string, int, int, Task<JellyfinLibraryItemsResponse>> getItems,
         Func<Option<JellyfinLibrary>, JellyfinLibraryItemResponse, Option<TItem>> mapper)
     {
         if (_memoryCache.TryGetValue($"jellyfin_admin_user_id.{mediaSourceId}", out string userId))
         {
             IJellyfinApi service = RestService.For<IJellyfinApi>(address);
-            string filters = itemType == JellyfinItemType.Movie || itemType == JellyfinItemType.Episode
-                ? "IsNotFolder"
-                : null;
-            int size = await service
-                .GetLibraryStats(apiKey, userId, parentId, itemType, filters: filters)
-                .Map(r => r.TotalRecordCount);
 
             const int PAGE_SIZE = 10;
 
-            int pages = (size - 1) / PAGE_SIZE + 1;
-
+            int pages = int.MaxValue;
             for (var i = 0; i < pages; i++)
             {
                 int skip = i * PAGE_SIZE;
 
-                Task<IEnumerable<TItem>> result = getItems(service, userId, parentId, skip, PAGE_SIZE)
-                    .Map(items => items.Items.Map(item => mapper(maybeLibrary, item)).Somes());
+                JellyfinLibraryItemsResponse result = await getItems(service, userId, parentId, skip, PAGE_SIZE);
 
-                foreach (TItem item in await result)
+                // update page count
+                pages = Math.Min(pages, (result.TotalRecordCount - 1) / PAGE_SIZE + 1);
+
+                foreach (TItem item in result.Items.Map(item => mapper(maybeLibrary, item)).Somes())
                 {
-                    yield return item;
+                    yield return new Tuple<TItem, int>(item, result.TotalRecordCount);
                 }
             }
         }

ErsatzTV.Infrastructure/Plex/PlexServerApiClient.cs

@@ -66,7 +66,7 @@ public class PlexServerApiClient : IPlexServerApiClient
         }
     }
 
-    public IAsyncEnumerable<PlexMovie> GetMovieLibraryContents(
+    public IAsyncEnumerable<Tuple<PlexMovie, int>> GetMovieLibraryContents(
         PlexLibrary library,
         PlexConnection connection,
         PlexServerAuthToken token)
@@ -89,7 +89,7 @@ public class PlexServerApiClient : IPlexServerApiClient
         return GetPagedLibraryContents(connection, CountItems, GetItems);
     }
 
-    public IAsyncEnumerable<PlexShow> GetShowLibraryContents(
+    public IAsyncEnumerable<Tuple<PlexShow, int>> GetShowLibraryContents(
         PlexLibrary library,
         PlexConnection connection,
         PlexServerAuthToken token)
@@ -110,24 +110,7 @@ public class PlexServerApiClient : IPlexServerApiClient
         return GetPagedLibraryContents(connection, CountItems, GetItems);
     }
 
-    public async Task<Either<BaseError, int>> CountShowSeasons(
-        PlexShow show,
-        PlexConnection connection,
-        PlexServerAuthToken token)
-    {
-        try
-        {
-            string showMetadataKey = show.Key.Split("/").Reverse().Skip(1).Head();
-            IPlexServerApi service = XmlServiceFor(connection.Uri);
-            return await service.CountShowChildren(showMetadataKey, token.AuthToken).Map(r => r.TotalSize);
-        }
-        catch (Exception ex)
-        {
-            return BaseError.New(ex.ToString());
-        }
-    }
-
-    public IAsyncEnumerable<PlexSeason> GetShowSeasons(
+    public IAsyncEnumerable<Tuple<PlexSeason, int>> GetShowSeasons(
         PlexLibrary library,
         PlexShow show,
         PlexConnection connection,
@@ -150,24 +133,7 @@ public class PlexServerApiClient : IPlexServerApiClient
         return GetPagedLibraryContents(connection, CountItems, GetItems);
     }
 
-    public async Task<Either<BaseError, int>> CountSeasonEpisodes(
-        PlexSeason season,
-        PlexConnection connection,
-        PlexServerAuthToken token)
-    {
-        try
-        {
-            string seasonMetadataKey = season.Key.Split("/").Reverse().Skip(1).Head();
-            IPlexServerApi service = XmlServiceFor(connection.Uri);
-            return await service.CountSeasonChildren(seasonMetadataKey, token.AuthToken).Map(r => r.TotalSize);
-        }
-        catch (Exception ex)
-        {
-            return BaseError.New(ex.ToString());
-        }
-    }
-
-    public IAsyncEnumerable<PlexEpisode> GetSeasonEpisodes(
+    public IAsyncEnumerable<Tuple<PlexEpisode, int>> GetSeasonEpisodes(
         PlexLibrary library,
         PlexSeason season,
         PlexConnection connection,
@@ -276,23 +242,7 @@ public class PlexServerApiClient : IPlexServerApiClient
         }
     }
 
-    public async Task<Either<BaseError, int>> GetLibraryItemCount(
-        PlexLibrary library,
-        PlexConnection connection,
-        PlexServerAuthToken token)
-    {
-        try
-        {
-            IPlexServerApi service = XmlServiceFor(connection.Uri);
-            return await service.GetLibrarySection(library.Key, token.AuthToken).Map(r => r.TotalSize);
-        }
-        catch (Exception ex)
-        {
-            return BaseError.New(ex.ToString());
-        }
-    }
-
-    public IAsyncEnumerable<PlexCollection> GetAllCollections(
+    public IAsyncEnumerable<Tuple<PlexCollection, int>> GetAllCollections(
         PlexConnection connection,
         PlexServerAuthToken token,
         CancellationToken cancellationToken)
@@ -313,7 +263,7 @@ public class PlexServerApiClient : IPlexServerApiClient
         }
     }
 
-    public IAsyncEnumerable<MediaItem> GetCollectionItems(
+    public IAsyncEnumerable<Tuple<MediaItem, int>> GetCollectionItems(
         PlexConnection connection,
         PlexServerAuthToken token,
         string key,
@@ -335,7 +285,7 @@ public class PlexServerApiClient : IPlexServerApiClient
         }
     }
 
-    private static async IAsyncEnumerable<TItem> GetPagedLibraryContents<TItem>(
+    private static async IAsyncEnumerable<Tuple<TItem, int>> GetPagedLibraryContents<TItem>(
         PlexConnection connection,
         Func<IPlexServerApi, Task<PlexXmlMediaContainerStatsResponse>> countItems,
         Func<IPlexServerApi, IPlexServerApi, int, int, Task<IEnumerable<TItem>>> getItems)
@@ -357,7 +307,7 @@ public class PlexServerApiClient : IPlexServerApiClient
 
             foreach (TItem item in await result)
             {
-                yield return item;
+                yield return new Tuple<TItem, int>(item, size);
             }
         }
     }