add library scan progress detail (#133)

* add library scan progress detail

* scan plex libraries on plex thread

ErsatzTV.Core/Domain/Library/LibraryPath.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 
 namespace ErsatzTV.Core.Domain
 {
@@ -6,6 +7,7 @@ namespace ErsatzTV.Core.Domain
     {
         public int Id { get; set; }
         public string Path { get; set; }
+        public DateTime? LastScan { get; set; }
 
         public int LibraryId { get; set; }
         public Library Library { get; set; }

ErsatzTV.Core/ErsatzTV.Core.csproj

@@ -11,6 +11,7 @@
         <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       </PackageReference>
       <PackageReference Include="LanguageExt.Core" Version="3.4.15" />
+      <PackageReference Include="MediatR" Version="9.0.0" />
       <PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
       <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" />
       <PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="16.9.60">

ErsatzTV.Core/FFmpeg/FFmpegProcessBuilder.cs

@@ -199,6 +199,7 @@ namespace ErsatzTV.Core.FFmpeg
                 "-c", "copy",
                 "-muxdelay", "0",
                 "-muxpreload", "0"
+                // "-avoid_negative_ts", "make_zero"
             };
             _arguments.AddRange(arguments);
             return this;

ErsatzTV.Core/Interfaces/Metadata/IMovieFolderScanner.cs

@@ -7,6 +7,11 @@ namespace ErsatzTV.Core.Interfaces.Metadata
 {
     public interface IMovieFolderScanner
     {
-        Task<Either<BaseError, Unit>> ScanFolder(LibraryPath libraryPath, string ffprobePath, DateTimeOffset lastScan);
+        Task<Either<BaseError, Unit>> ScanFolder(
+            LibraryPath libraryPath,
+            string ffprobePath,
+            DateTimeOffset lastScan,
+            decimal progressMin,
+            decimal progressMax);
     }
 }

ErsatzTV.Core/Interfaces/Metadata/IMusicVideoFolderScanner.cs

@@ -7,6 +7,11 @@ namespace ErsatzTV.Core.Interfaces.Metadata
 {
     public interface IMusicVideoFolderScanner
     {
-        Task<Either<BaseError, Unit>> ScanFolder(LibraryPath libraryPath, string ffprobePath, DateTimeOffset lastScan);
+        Task<Either<BaseError, Unit>> ScanFolder(
+            LibraryPath libraryPath,
+            string ffprobePath,
+            DateTimeOffset lastScan,
+            decimal progressMin,
+            decimal progressMax);
     }
 }

ErsatzTV.Core/Interfaces/Metadata/ITelevisionFolderScanner.cs

@@ -7,6 +7,11 @@ namespace ErsatzTV.Core.Interfaces.Metadata
 {
     public interface ITelevisionFolderScanner
     {
-        Task<Either<BaseError, Unit>> ScanFolder(LibraryPath libraryPath, string ffprobePath, DateTimeOffset lastScan);
+        Task<Either<BaseError, Unit>> ScanFolder(
+            LibraryPath libraryPath,
+            string ffprobePath,
+            DateTimeOffset lastScan,
+            decimal progressMin,
+            decimal progressMax);
     }
 }

ErsatzTV.Core/Interfaces/Repositories/ILibraryRepository.cs

@@ -12,6 +12,7 @@ namespace ErsatzTV.Core.Interfaces.Repositories
         Task<Option<LocalLibrary>> GetLocal(int libraryId);
         Task<List<Library>> GetAll();
         Task<Unit> UpdateLastScan(Library library);
+        Task<Unit> UpdateLastScan(LibraryPath libraryPath);
         Task<List<LibraryPath>> GetLocalPaths(int libraryId);
         Task<Option<LibraryPath>> GetPath(int libraryPathId);
         Task<int> CountMediaItemsByPath(int libraryPathId);

ErsatzTV.Core/Metadata/LibraryScanProgress.cs

@@ -0,0 +1,6 @@
+using MediatR;
+
+namespace ErsatzTV.Core.Metadata
+{
+    public record LibraryScanProgress(int LibraryId, decimal Progress) : INotification;
+}

ErsatzTV.Core/Metadata/MovieFolderScanner.cs

@@ -10,9 +10,11 @@ using ErsatzTV.Core.Interfaces.Metadata;
 using ErsatzTV.Core.Interfaces.Repositories;
 using ErsatzTV.Core.Interfaces.Search;
 using LanguageExt;
+using MediatR;
 using Microsoft.Extensions.Logging;
 using static LanguageExt.Prelude;
 using Seq = LanguageExt.Seq;
+using Unit = LanguageExt.Unit;
 
 namespace ErsatzTV.Core.Metadata
 {
@@ -21,6 +23,7 @@ namespace ErsatzTV.Core.Metadata
         private readonly ILocalFileSystem _localFileSystem;
         private readonly ILocalMetadataProvider _localMetadataProvider;
         private readonly ILogger<MovieFolderScanner> _logger;
+        private readonly IMediator _mediator;
         private readonly IMovieRepository _movieRepository;
         private readonly ISearchIndex _searchIndex;
 
@@ -32,6 +35,7 @@ namespace ErsatzTV.Core.Metadata
             IMetadataRepository metadataRepository,
             IImageCache imageCache,
             ISearchIndex searchIndex,
+            IMediator mediator,
             ILogger<MovieFolderScanner> logger)
             : base(localFileSystem, localStatisticsProvider, metadataRepository, imageCache, logger)
         {
@@ -39,19 +43,26 @@ namespace ErsatzTV.Core.Metadata
             _movieRepository = movieRepository;
             _localMetadataProvider = localMetadataProvider;
             _searchIndex = searchIndex;
+            _mediator = mediator;
             _logger = logger;
         }
 
         public async Task<Either<BaseError, Unit>> ScanFolder(
             LibraryPath libraryPath,
             string ffprobePath,
-            DateTimeOffset lastScan)
+            DateTimeOffset lastScan,
+            decimal progressMin,
+            decimal progressMax)
         {
+            decimal progressSpread = progressMax - progressMin;
+
             if (!_localFileSystem.IsLibraryPathAccessible(libraryPath))
             {
                 return new MediaSourceInaccessible();
             }
 
+            var foldersCompleted = 0;
+
             var folderQueue = new Queue<string>();
             foreach (string folder in _localFileSystem.ListSubdirectories(libraryPath.Path).OrderBy(identity))
             {
@@ -60,7 +71,12 @@ namespace ErsatzTV.Core.Metadata
 
             while (folderQueue.Count > 0)
             {
+                decimal percentCompletion = (decimal) foldersCompleted / (foldersCompleted + folderQueue.Count);
+                await _mediator.Publish(
+                    new LibraryScanProgress(libraryPath.LibraryId, progressMin + percentCompletion * progressSpread));
+
                 string movieFolder = folderQueue.Dequeue();
+                foldersCompleted++;
 
                 var allFiles = _localFileSystem.ListFiles(movieFolder)
                     .Filter(f => VideoFileExtensions.Contains(Path.GetExtension(f)))

ErsatzTV.Core/Metadata/MusicVideoFolderScanner.cs

@@ -10,8 +10,10 @@ using ErsatzTV.Core.Interfaces.Metadata;
 using ErsatzTV.Core.Interfaces.Repositories;
 using ErsatzTV.Core.Interfaces.Search;
 using LanguageExt;
+using MediatR;
 using Microsoft.Extensions.Logging;
 using static LanguageExt.Prelude;
+using Unit = LanguageExt.Unit;
 
 namespace ErsatzTV.Core.Metadata
 {
@@ -20,6 +22,7 @@ namespace ErsatzTV.Core.Metadata
         private readonly ILocalFileSystem _localFileSystem;
         private readonly ILocalMetadataProvider _localMetadataProvider;
         private readonly ILogger<MusicVideoFolderScanner> _logger;
+        private readonly IMediator _mediator;
         private readonly IMusicVideoRepository _musicVideoRepository;
         private readonly ISearchIndex _searchIndex;
 
@@ -31,6 +34,7 @@ namespace ErsatzTV.Core.Metadata
             IImageCache imageCache,
             ISearchIndex searchIndex,
             IMusicVideoRepository musicVideoRepository,
+            IMediator mediator,
             ILogger<MusicVideoFolderScanner> logger) : base(
             localFileSystem,
             localStatisticsProvider,
@@ -42,19 +46,26 @@ namespace ErsatzTV.Core.Metadata
             _localMetadataProvider = localMetadataProvider;
             _searchIndex = searchIndex;
             _musicVideoRepository = musicVideoRepository;
+            _mediator = mediator;
             _logger = logger;
         }
 
         public async Task<Either<BaseError, Unit>> ScanFolder(
             LibraryPath libraryPath,
             string ffprobePath,
-            DateTimeOffset lastScan)
+            DateTimeOffset lastScan,
+            decimal progressMin,
+            decimal progressMax)
         {
+            decimal progressSpread = progressMax - progressMin;
+
             if (!_localFileSystem.IsLibraryPathAccessible(libraryPath))
             {
                 return new MediaSourceInaccessible();
             }
 
+            var foldersCompleted = 0;
+
             var folderQueue = new Queue<string>();
             foreach (string folder in _localFileSystem.ListSubdirectories(libraryPath.Path).OrderBy(identity))
             {
@@ -63,9 +74,14 @@ namespace ErsatzTV.Core.Metadata
 
             while (folderQueue.Count > 0)
             {
-                string movieFolder = folderQueue.Dequeue();
+                decimal percentCompletion = (decimal) foldersCompleted / (foldersCompleted + folderQueue.Count);
+                await _mediator.Publish(
+                    new LibraryScanProgress(libraryPath.LibraryId, progressMin + percentCompletion * progressSpread));
 
-                var allFiles = _localFileSystem.ListFiles(movieFolder)
+                string musicVideoFolder = folderQueue.Dequeue();
+                foldersCompleted++;
+
+                var allFiles = _localFileSystem.ListFiles(musicVideoFolder)
                     .Filter(f => VideoFileExtensions.Contains(Path.GetExtension(f)))
                     .Filter(
                         f => !ExtraFiles.Any(
@@ -74,7 +90,8 @@ namespace ErsatzTV.Core.Metadata
 
                 if (allFiles.Count == 0)
                 {
-                    foreach (string subdirectory in _localFileSystem.ListSubdirectories(movieFolder).OrderBy(identity))
+                    foreach (string subdirectory in _localFileSystem.ListSubdirectories(musicVideoFolder)
+                        .OrderBy(identity))
                     {
                         folderQueue.Enqueue(subdirectory);
                     }
@@ -82,7 +99,7 @@ namespace ErsatzTV.Core.Metadata
                     continue;
                 }
 
-                if (_localFileSystem.GetLastWriteTime(movieFolder) < lastScan)
+                if (_localFileSystem.GetLastWriteTime(musicVideoFolder) < lastScan)
                 {
                     continue;
                 }

ErsatzTV.Core/Metadata/TelevisionFolderScanner.cs

@@ -10,8 +10,10 @@ using ErsatzTV.Core.Interfaces.Metadata;
 using ErsatzTV.Core.Interfaces.Repositories;
 using ErsatzTV.Core.Interfaces.Search;
 using LanguageExt;
+using MediatR;
 using Microsoft.Extensions.Logging;
 using static LanguageExt.Prelude;
+using Unit = LanguageExt.Unit;
 
 namespace ErsatzTV.Core.Metadata
 {
@@ -20,6 +22,7 @@ namespace ErsatzTV.Core.Metadata
         private readonly ILocalFileSystem _localFileSystem;
         private readonly ILocalMetadataProvider _localMetadataProvider;
         private readonly ILogger<TelevisionFolderScanner> _logger;
+        private readonly IMediator _mediator;
         private readonly ISearchIndex _searchIndex;
         private readonly ITelevisionRepository _televisionRepository;
 
@@ -31,6 +34,7 @@ namespace ErsatzTV.Core.Metadata
             IMetadataRepository metadataRepository,
             IImageCache imageCache,
             ISearchIndex searchIndex,
+            IMediator mediator,
             ILogger<TelevisionFolderScanner> logger) : base(
             localFileSystem,
             localStatisticsProvider,
@@ -42,14 +46,19 @@ namespace ErsatzTV.Core.Metadata
             _televisionRepository = televisionRepository;
             _localMetadataProvider = localMetadataProvider;
             _searchIndex = searchIndex;
+            _mediator = mediator;
             _logger = logger;
         }
 
         public async Task<Either<BaseError, Unit>> ScanFolder(
             LibraryPath libraryPath,
             string ffprobePath,
-            DateTimeOffset lastScan)
+            DateTimeOffset lastScan,
+            decimal progressMin,
+            decimal progressMax)
         {
+            decimal progressSpread = progressMax - progressMin;
+
             if (!_localFileSystem.IsLibraryPathAccessible(libraryPath))
             {
                 return new MediaSourceInaccessible();
@@ -62,6 +71,10 @@ namespace ErsatzTV.Core.Metadata
 
             foreach (string showFolder in allShowFolders)
             {
+                decimal percentCompletion = (decimal) allShowFolders.IndexOf(showFolder) / allShowFolders.Count;
+                await _mediator.Publish(
+                    new LibraryScanProgress(libraryPath.LibraryId, progressMin + percentCompletion * progressSpread));
+
                 Either<BaseError, MediaItemScanResult<Show>> maybeShow =
                     await FindOrCreateShow(libraryPath.Id, showFolder)
                         .BindT(show => UpdateMetadataForShow(show, showFolder))

ErsatzTV.Core/Plex/PlexLibraryScanner.cs

@@ -3,16 +3,21 @@ using System.Threading.Tasks;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Interfaces.Repositories;
 using LanguageExt;
+using Microsoft.Extensions.Logging;
 using static LanguageExt.Prelude;
 
 namespace ErsatzTV.Core.Plex
 {
     public abstract class PlexLibraryScanner
     {
+        private readonly ILogger<PlexLibraryScanner> _logger;
         private readonly IMetadataRepository _metadataRepository;
 
-        protected PlexLibraryScanner(IMetadataRepository metadataRepository) =>
+        protected PlexLibraryScanner(IMetadataRepository metadataRepository, ILogger<PlexLibraryScanner> logger)
+        {
             _metadataRepository = metadataRepository;
+            _logger = logger;
+        }
 
         protected async Task<Unit> UpdateArtworkIfNeeded(
             Domain.Metadata existingMetadata,
@@ -27,6 +32,8 @@ namespace ErsatzTV.Core.Plex
                 await maybeIncomingArtwork.Match(
                     async incomingArtwork =>
                     {
+                        _logger.LogDebug("Refreshing Plex {Attribute} from {Path}", artworkKind, incomingArtwork.Path);
+
                         Option<Artwork> maybeExistingArtwork = Optional(existingMetadata.Artwork).Flatten()
                             .Find(a => a.ArtworkKind == artworkKind);
 

ErsatzTV.Core/Plex/PlexMovieLibraryScanner.cs

@@ -7,13 +7,16 @@ using ErsatzTV.Core.Interfaces.Repositories;
 using ErsatzTV.Core.Interfaces.Search;
 using ErsatzTV.Core.Metadata;
 using LanguageExt;
+using MediatR;
 using Microsoft.Extensions.Logging;
+using Unit = LanguageExt.Unit;
 
 namespace ErsatzTV.Core.Plex
 {
     public class PlexMovieLibraryScanner : PlexLibraryScanner, IPlexMovieLibraryScanner
     {
         private readonly ILogger<PlexMovieLibraryScanner> _logger;
+        private readonly IMediator _mediator;
         private readonly IMetadataRepository _metadataRepository;
         private readonly IMovieRepository _movieRepository;
         private readonly IPlexServerApiClient _plexServerApiClient;
@@ -24,13 +27,15 @@ namespace ErsatzTV.Core.Plex
             IMovieRepository movieRepository,
             IMetadataRepository metadataRepository,
             ISearchIndex searchIndex,
+            IMediator mediator,
             ILogger<PlexMovieLibraryScanner> logger)
-            : base(metadataRepository)
+            : base(metadataRepository, logger)
         {
             _plexServerApiClient = plexServerApiClient;
             _movieRepository = movieRepository;
             _metadataRepository = metadataRepository;
             _searchIndex = searchIndex;
+            _mediator = mediator;
             _logger = logger;
         }
 
@@ -49,6 +54,9 @@ namespace ErsatzTV.Core.Plex
                 {
                     foreach (PlexMovie incoming in movieEntries)
                     {
+                        decimal percentCompletion = (decimal) movieEntries.IndexOf(incoming) / movieEntries.Count;
+                        await _mediator.Publish(new LibraryScanProgress(plexMediaSourceLibrary.Id, percentCompletion));
+
                         // TODO: figure out how to rebuild playlists
                         Either<BaseError, MediaItemScanResult<PlexMovie>> maybeMovie = await _movieRepository
                             .GetOrAdd(plexMediaSourceLibrary, incoming)
@@ -81,6 +89,8 @@ namespace ErsatzTV.Core.Plex
                     var movieKeys = movieEntries.Map(s => s.Key).ToList();
                     List<int> ids = await _movieRepository.RemoveMissingPlexMovies(plexMediaSourceLibrary, movieKeys);
                     await _searchIndex.RemoveItems(ids);
+
+                    await _mediator.Publish(new LibraryScanProgress(plexMediaSourceLibrary.Id, 0));
                 },
                 error =>
                 {
@@ -113,6 +123,11 @@ namespace ErsatzTV.Core.Plex
                 await maybeStatistics.Match(
                     async mediaVersion =>
                     {
+                        _logger.LogDebug(
+                            "Refreshing {Attribute} from {Path}",
+                            "Plex Statistics",
+                            existingVersion.MediaFiles.Head().Path);
+
                         existingVersion.SampleAspectRatio = mediaVersion.SampleAspectRatio;
                         existingVersion.VideoScanKind = mediaVersion.VideoScanKind;
                         existingVersion.DateUpdated = mediaVersion.DateUpdated;
@@ -135,6 +150,11 @@ namespace ErsatzTV.Core.Plex
 
             if (incomingMetadata.DateUpdated > existingMetadata.DateUpdated)
             {
+                _logger.LogDebug(
+                    "Refreshing {Attribute} from {Path}",
+                    "Plex Metadata",
+                    existing.MediaVersions.Head().MediaFiles.Head().Path);
+
                 foreach (Genre genre in existingMetadata.Genres
                     .Filter(g => incomingMetadata.Genres.All(g2 => g2.Name != g.Name))
                     .ToList())

ErsatzTV.Core/Plex/PlexTelevisionLibraryScanner.cs

@@ -7,14 +7,17 @@ using ErsatzTV.Core.Interfaces.Repositories;
 using ErsatzTV.Core.Interfaces.Search;
 using ErsatzTV.Core.Metadata;
 using LanguageExt;
+using MediatR;
 using Microsoft.Extensions.Logging;
 using static LanguageExt.Prelude;
+using Unit = LanguageExt.Unit;
 
 namespace ErsatzTV.Core.Plex
 {
     public class PlexTelevisionLibraryScanner : PlexLibraryScanner, IPlexTelevisionLibraryScanner
     {
         private readonly ILogger<PlexTelevisionLibraryScanner> _logger;
+        private readonly IMediator _mediator;
         private readonly IMetadataRepository _metadataRepository;
         private readonly IPlexServerApiClient _plexServerApiClient;
         private readonly ISearchIndex _searchIndex;
@@ -25,13 +28,15 @@ namespace ErsatzTV.Core.Plex
             ITelevisionRepository televisionRepository,
             IMetadataRepository metadataRepository,
             ISearchIndex searchIndex,
+            IMediator mediator,
             ILogger<PlexTelevisionLibraryScanner> logger)
-            : base(metadataRepository)
+            : base(metadataRepository, logger)
         {
             _plexServerApiClient = plexServerApiClient;
             _televisionRepository = televisionRepository;
             _metadataRepository = metadataRepository;
             _searchIndex = searchIndex;
+            _mediator = mediator;
             _logger = logger;
         }
 
@@ -50,6 +55,9 @@ namespace ErsatzTV.Core.Plex
                 {
                     foreach (PlexShow incoming in showEntries)
                     {
+                        decimal percentCompletion = (decimal) showEntries.IndexOf(incoming) / showEntries.Count;
+                        await _mediator.Publish(new LibraryScanProgress(plexMediaSourceLibrary.Id, percentCompletion));
+
                         // TODO: figure out how to rebuild playlists
                         Either<BaseError, MediaItemScanResult<PlexShow>> maybeShow = await _televisionRepository
                             .GetOrAddPlexShow(plexMediaSourceLibrary, incoming)
@@ -85,6 +93,8 @@ namespace ErsatzTV.Core.Plex
                         await _televisionRepository.RemoveMissingPlexShows(plexMediaSourceLibrary, showKeys);
                     await _searchIndex.RemoveItems(ids);
 
+                    await _mediator.Publish(new LibraryScanProgress(plexMediaSourceLibrary.Id, 0));
+
                     return Unit.Default;
                 },
                 error =>