timothy/ersatztv
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

optimize local library scanning (#112)

Browse Source
This commit is contained in:
Jason Dove
2021-03-29 05:34:33 -05:00
committed by GitHub
parent f06a04ed0e
commit 076a88230e
6 changed files with 90 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
ErsatzTV.Application/MediaSources/Commands/ScanLocalLibraryHandler.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
@@ -61,21 +62,30 @@ namespace ErsatzTV.Application.MediaSources.Commands
var lastScan = new DateTimeOffset(localLibrary.LastScan ?? DateTime.MinValue, TimeSpan.Zero);
if (forceScan || lastScan < DateTimeOffset.Now - TimeSpan.FromHours(6))
{
var sw = new Stopwatch();
sw.Start();
foreach (LibraryPath libraryPath in localLibrary.Paths)
{
switch (localLibrary.MediaKind)
{
case LibraryMediaKind.Movies:
await _movieFolderScanner.ScanFolder(libraryPath, ffprobePath);
await _movieFolderScanner.ScanFolder(libraryPath, ffprobePath, lastScan);
break;
case LibraryMediaKind.Shows:
await _televisionFolderScanner.ScanFolder(libraryPath, ffprobePath);
await _televisionFolderScanner.ScanFolder(libraryPath, ffprobePath, lastScan);
break;
}
}
localLibrary.LastScan = DateTime.UtcNow;
await _libraryRepository.UpdateLastScan(localLibrary);
sw.Stop();
_logger.LogDebug(
"Scan of library {Name} completed in {Duration}",
localLibrary.Name,
TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds));
}
else
{

60
ErsatzTV.Core.Tests/Metadata/MovieFolderScannerTests.cs
View File

@@ -81,7 +81,10 @@ namespace ErsatzTV.Core.Tests.Metadata
);
var libraryPath = new LibraryPath { Path = BadFakeRoot };
Either<BaseError, Unit> result = await service.ScanFolder(libraryPath, FFprobePath);
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue);
result.IsLeft.Should().BeTrue();
result.IfLeft(error => error.Should().BeOfType<MediaSourceInaccessible>());
@@ -101,7 +104,10 @@ namespace ErsatzTV.Core.Tests.Metadata
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
Either<BaseError, Unit> result = await service.ScanFolder(libraryPath, FFprobePath);
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue);
result.IsRight.Should().BeTrue();
@@ -137,7 +143,10 @@ namespace ErsatzTV.Core.Tests.Metadata
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
Either<BaseError, Unit> result = await service.ScanFolder(libraryPath, FFprobePath);
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue);
result.IsRight.Should().BeTrue();
@@ -174,7 +183,10 @@ namespace ErsatzTV.Core.Tests.Metadata
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
Either<BaseError, Unit> result = await service.ScanFolder(libraryPath, FFprobePath);
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue);
result.IsRight.Should().BeTrue();
@@ -215,7 +227,10 @@ namespace ErsatzTV.Core.Tests.Metadata
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
Either<BaseError, Unit> result = await service.ScanFolder(libraryPath, FFprobePath);
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue);
result.IsRight.Should().BeTrue();
@@ -259,7 +274,10 @@ namespace ErsatzTV.Core.Tests.Metadata
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
Either<BaseError, Unit> result = await service.ScanFolder(libraryPath, FFprobePath);
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue);
result.IsRight.Should().BeTrue();
@@ -303,7 +321,10 @@ namespace ErsatzTV.Core.Tests.Metadata
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
Either<BaseError, Unit> result = await service.ScanFolder(libraryPath, FFprobePath);
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue);
result.IsRight.Should().BeTrue();
@@ -346,7 +367,10 @@ namespace ErsatzTV.Core.Tests.Metadata
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
Either<BaseError, Unit> result = await service.ScanFolder(libraryPath, FFprobePath);
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue);
result.IsRight.Should().BeTrue();
@@ -385,7 +409,10 @@ namespace ErsatzTV.Core.Tests.Metadata
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
Either<BaseError, Unit> result = await service.ScanFolder(libraryPath, FFprobePath);
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue);
result.IsRight.Should().BeTrue();
@@ -418,7 +445,10 @@ namespace ErsatzTV.Core.Tests.Metadata
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
Either<BaseError, Unit> result = await service.ScanFolder(libraryPath, FFprobePath);
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue);
result.IsRight.Should().BeTrue();
@@ -453,7 +483,10 @@ namespace ErsatzTV.Core.Tests.Metadata
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
Either<BaseError, Unit> result = await service.ScanFolder(libraryPath, FFprobePath);
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue);
result.IsRight.Should().BeTrue();
@@ -477,7 +510,10 @@ namespace ErsatzTV.Core.Tests.Metadata
);
var libraryPath = new LibraryPath { Id = 1, Path = FakeRoot };
Either<BaseError, Unit> result = await service.ScanFolder(libraryPath, FFprobePath);
Either<BaseError, Unit> result = await service.ScanFolder(
libraryPath,
FFprobePath,
DateTimeOffset.MinValue);
result.IsRight.Should().BeTrue();

5
ErsatzTV.Core/Interfaces/Metadata/IMovieFolderScanner.cs
View File

@@ -1,4 +1,5 @@
using System.Threading.Tasks;
using System;
using System.Threading.Tasks;
using ErsatzTV.Core.Domain;
using LanguageExt;
@@ -6,6 +7,6 @@ namespace ErsatzTV.Core.Interfaces.Metadata
{
public interface IMovieFolderScanner
{
Task<Either<BaseError, Unit>> ScanFolder(LibraryPath libraryPath, string ffprobePath);
Task<Either<BaseError, Unit>> ScanFolder(LibraryPath libraryPath, string ffprobePath, DateTimeOffset lastScan);
}
}

5
ErsatzTV.Core/Interfaces/Metadata/ITelevisionFolderScanner.cs
View File

@@ -1,4 +1,5 @@
using System.Threading.Tasks;
using System;
using System.Threading.Tasks;
using ErsatzTV.Core.Domain;
using LanguageExt;
@@ -6,6 +7,6 @@ namespace ErsatzTV.Core.Interfaces.Metadata
{
public interface ITelevisionFolderScanner
{
Task<Either<BaseError, Unit>> ScanFolder(LibraryPath libraryPath, string ffprobePath);
Task<Either<BaseError, Unit>> ScanFolder(LibraryPath libraryPath, string ffprobePath, DateTimeOffset lastScan);
}
}

10
ErsatzTV.Core/Metadata/MovieFolderScanner.cs
View File

@@ -42,7 +42,10 @@ namespace ErsatzTV.Core.Metadata
_logger = logger;
}
public async Task<Either<BaseError, Unit>> ScanFolder(LibraryPath libraryPath, string ffprobePath)
public async Task<Either<BaseError, Unit>> ScanFolder(
LibraryPath libraryPath,
string ffprobePath,
DateTimeOffset lastScan)
{
if (!_localFileSystem.IsLibraryPathAccessible(libraryPath))
{
@@ -76,6 +79,11 @@ namespace ErsatzTV.Core.Metadata
continue;
}
if (_localFileSystem.GetLastWriteTime(movieFolder) < lastScan)
{
continue;
}
foreach (string file in allFiles.OrderBy(identity))
{
// TODO: figure out how to rebuild playlists

20
ErsatzTV.Core/Metadata/TelevisionFolderScanner.cs
View File

@@ -45,7 +45,10 @@ namespace ErsatzTV.Core.Metadata
_logger = logger;
}
public async Task<Either<BaseError, Unit>> ScanFolder(LibraryPath libraryPath, string ffprobePath)
public async Task<Either<BaseError, Unit>> ScanFolder(
LibraryPath libraryPath,
string ffprobePath,
DateTimeOffset lastScan)
{
if (!_localFileSystem.IsLibraryPathAccessible(libraryPath))
{
@@ -77,7 +80,7 @@ namespace ErsatzTV.Core.Metadata
await _searchIndex.UpdateItems(new List<MediaItem> { result.Item });
}
await ScanSeasons(libraryPath, ffprobePath, result.Item, showFolder);
await ScanSeasons(libraryPath, ffprobePath, result.Item, showFolder, lastScan);
},
_ => Task.FromResult(Unit.Default));
}
@@ -113,7 +116,8 @@ namespace ErsatzTV.Core.Metadata
LibraryPath libraryPath,
string ffprobePath,
Show show,
string showFolder)
string showFolder,
DateTimeOffset lastScan)
{
foreach (string seasonFolder in _localFileSystem.ListSubdirectories(showFolder).Filter(ShouldIncludeFolder)
.OrderBy(identity))
@@ -127,7 +131,7 @@ namespace ErsatzTV.Core.Metadata
.BindT(season => UpdatePoster(season, seasonFolder));
await maybeSeason.Match(
season => ScanEpisodes(libraryPath, ffprobePath, season, seasonFolder),
season => ScanEpisodes(libraryPath, ffprobePath, season, seasonFolder, lastScan),
_ => Task.FromResult(Unit.Default));
});
}
@@ -139,8 +143,14 @@ namespace ErsatzTV.Core.Metadata
LibraryPath libraryPath,
string ffprobePath,
Season season,
string seasonPath)
string seasonPath,
DateTimeOffset lastScan)
{
if (_localFileSystem.GetLastWriteTime(seasonPath) < lastScan)
{
return Unit.Default;
}
foreach (string file in _localFileSystem.ListFiles(seasonPath)
.Filter(f => VideoFileExtensions.Contains(Path.GetExtension(f))).OrderBy(identity))
{