diff --git a/CHANGELOG.md b/CHANGELOG.md index c47380f4d..647be092c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - This mode is used when a schedule is updated, or when collection modifications trigger a playout rebuild - `Reset` - this mode will rebuild the entire playout and will NOT maintain progress - This mode is only used when the `Reset Playout` button is clicked on the Playouts page +- Use ffmpeg to resize images; this should help reduce ErsatzTV's memory use +- Use ffprobe to check for animated logos and watermarks; this should help reduce ErsatzTV's memory use ## [0.4.5-alpha] - 2022-03-29 ### Fixed diff --git a/ErsatzTV.Application/ErsatzTV.Application.csproj b/ErsatzTV.Application/ErsatzTV.Application.csproj index 3ccfc42c8..73813cb3e 100644 --- a/ErsatzTV.Application/ErsatzTV.Application.csproj +++ b/ErsatzTV.Application/ErsatzTV.Application.csproj @@ -7,7 +7,7 @@ - + diff --git a/ErsatzTV.Application/Images/Queries/GetCachedImagePathHandler.cs b/ErsatzTV.Application/Images/Queries/GetCachedImagePathHandler.cs index 7e845828d..e5aef1920 100644 --- a/ErsatzTV.Application/Images/Queries/GetCachedImagePathHandler.cs +++ b/ErsatzTV.Application/Images/Queries/GetCachedImagePathHandler.cs @@ -1,5 +1,10 @@ -using ErsatzTV.Core; +using System.Diagnostics; +using CliWrap; +using ErsatzTV.Core; +using ErsatzTV.Core.Domain; +using ErsatzTV.Core.Interfaces.FFmpeg; using ErsatzTV.Core.Interfaces.Images; +using ErsatzTV.Core.Interfaces.Repositories; using Winista.Mime; namespace ErsatzTV.Application.Images; @@ -9,12 +14,32 @@ public class { private static readonly MimeTypes MimeTypes = new(); private readonly IImageCache _imageCache; + private readonly IFFmpegProcessService _ffmpegProcessService; + private readonly IConfigElementRepository _configElementRepository; - public GetCachedImagePathHandler(IImageCache imageCache) => _imageCache = imageCache; + public GetCachedImagePathHandler( + IImageCache imageCache, + IFFmpegProcessService ffmpegProcessService, + IConfigElementRepository configElementRepository) + { + _imageCache = imageCache; + _ffmpegProcessService = ffmpegProcessService; + _configElementRepository = configElementRepository; + } public async Task> Handle( GetCachedImagePath request, CancellationToken cancellationToken) + { + Validation validation = await Validate(); + return await validation.Match( + ffmpegPath => Handle(ffmpegPath, request), + error => Task.FromResult>(error.Join())); + } + + private async Task> Handle( + string ffmpegPath, + GetCachedImagePath request) { try { @@ -24,23 +49,44 @@ public class request.FileName, request.ArtworkKind, Optional(request.MaxHeight)); + + if (cachePath == null) + { + return BaseError.New("Failed to generate cache path for image"); + } + if (!File.Exists(cachePath)) { if (request.MaxHeight.HasValue) { - string originalPath = _imageCache.GetPathForImage(request.FileName, request.ArtworkKind, None); - byte[] contents = await File.ReadAllBytesAsync(originalPath, cancellationToken); - Either resizeResult = - await _imageCache.ResizeImage(contents, request.MaxHeight.Value); - resizeResult.IfRight(result => contents = result); - string baseFolder = Path.GetDirectoryName(cachePath); if (baseFolder != null && !Directory.Exists(baseFolder)) { Directory.CreateDirectory(baseFolder); } - await File.WriteAllBytesAsync(cachePath, contents, cancellationToken); + // ffmpeg needs the extension to determine the output codec + string withExtension = cachePath + ".jpg"; + + string originalPath = _imageCache.GetPathForImage(request.FileName, request.ArtworkKind, None); + + Process process = _ffmpegProcessService.ResizeImage( + ffmpegPath, + originalPath, + withExtension, + request.MaxHeight.Value); + + CommandResult resize = await Cli.Wrap(process.StartInfo.FileName) + .WithArguments(process.StartInfo.ArgumentList) + .WithValidation(CommandResultValidation.None) + .ExecuteAsync(); + + if (resize.ExitCode != 0) + { + return BaseError.New($"Failed to resize image; exit code {resize.ExitCode}"); + } + + File.Move(withExtension, cachePath); mimeType = new MimeType("image/jpeg"); } @@ -61,4 +107,12 @@ public class return BaseError.New(ex.Message); } } + + private async Task> Validate() => + await ValidateFFmpegPath(); + + private Task> ValidateFFmpegPath() => + _configElementRepository.GetValue(ConfigElementKey.FFmpegPath) + .FilterT(File.Exists) + .Map(ffmpegPath => ffmpegPath.ToValidation("FFmpeg path does not exist on the file system")); } \ No newline at end of file diff --git a/ErsatzTV.Application/Streaming/Queries/FFmpegProcessHandler.cs b/ErsatzTV.Application/Streaming/Queries/FFmpegProcessHandler.cs index c8aa980f4..db986616b 100644 --- a/ErsatzTV.Application/Streaming/Queries/FFmpegProcessHandler.cs +++ b/ErsatzTV.Application/Streaming/Queries/FFmpegProcessHandler.cs @@ -17,9 +17,9 @@ public abstract class FFmpegProcessHandler : IRequestHandler> Handle(T request, CancellationToken cancellationToken) { await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); - Validation> validation = await Validate(dbContext, request); + Validation> validation = await Validate(dbContext, request); return await validation.Match( - tuple => GetProcess(dbContext, request, tuple.Item1, tuple.Item2, cancellationToken), + tuple => GetProcess(dbContext, request, tuple.Item1, tuple.Item2, tuple.Item3, cancellationToken), error => Task.FromResult>(error.Join())); } @@ -28,13 +28,15 @@ public abstract class FFmpegProcessHandler : IRequestHandler>> Validate( + private static async Task>> Validate( TvContext dbContext, T request) => - (await ChannelMustExist(dbContext, request), await FFmpegPathMustExist(dbContext)) - .Apply((channel, ffmpegPath) => Tuple(channel, ffmpegPath)); + (await ChannelMustExist(dbContext, request), await FFmpegPathMustExist(dbContext), + await FFprobePathMustExist(dbContext)) + .Apply((channel, ffmpegPath, ffprobePath) => Tuple(channel, ffmpegPath, ffprobePath)); private static Task> ChannelMustExist(TvContext dbContext, T request) => dbContext.Channels @@ -63,4 +65,9 @@ public abstract class FFmpegProcessHandler : IRequestHandler(ConfigElementKey.FFmpegPath) .FilterT(File.Exists) .Map(maybePath => maybePath.ToValidation("FFmpeg path does not exist on filesystem")); + + private static Task> FFprobePathMustExist(TvContext dbContext) => + dbContext.ConfigElements.GetValue(ConfigElementKey.FFprobePath) + .FilterT(File.Exists) + .Map(maybePath => maybePath.ToValidation("FFprobe path does not exist on filesystem")); } \ No newline at end of file diff --git a/ErsatzTV.Application/Streaming/Queries/GetConcatProcessByChannelNumberHandler.cs b/ErsatzTV.Application/Streaming/Queries/GetConcatProcessByChannelNumberHandler.cs index 67d5fb9a8..fff0900e6 100644 --- a/ErsatzTV.Application/Streaming/Queries/GetConcatProcessByChannelNumberHandler.cs +++ b/ErsatzTV.Application/Streaming/Queries/GetConcatProcessByChannelNumberHandler.cs @@ -25,6 +25,7 @@ public class GetConcatProcessByChannelNumberHandler : FFmpegProcessHandler GetErrorProcess request, Channel channel, string ffmpegPath, + string ffprobePath, CancellationToken cancellationToken) { Process process = await _ffmpegProcessService.ForError( diff --git a/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs b/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs index 9dc0dc410..5536d79b9 100644 --- a/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs +++ b/ErsatzTV.Application/Streaming/Queries/GetPlayoutItemProcessByChannelNumberHandler.cs @@ -58,6 +58,7 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler< GetPlayoutItemProcessByChannelNumber request, Channel channel, string ffmpegPath, + string ffprobePath, CancellationToken cancellationToken) { DateTimeOffset now = request.Now; @@ -128,6 +129,7 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler< channel, maybeGlobalWatermark, ffmpegPath, + ffprobePath, cancellationToken); } @@ -137,6 +139,7 @@ public class GetPlayoutItemProcessByChannelNumberHandler : FFmpegProcessHandler< Process process = await _ffmpegProcessService.ForPlayoutItem( ffmpegPath, + ffprobePath, saveReports, channel, videoVersion, diff --git a/ErsatzTV.Application/Streaming/Queries/GetWrappedProcessByChannelNumberHandler.cs b/ErsatzTV.Application/Streaming/Queries/GetWrappedProcessByChannelNumberHandler.cs index c3f986b91..fa07ec618 100644 --- a/ErsatzTV.Application/Streaming/Queries/GetWrappedProcessByChannelNumberHandler.cs +++ b/ErsatzTV.Application/Streaming/Queries/GetWrappedProcessByChannelNumberHandler.cs @@ -25,6 +25,7 @@ public class GetWrappedProcessByChannelNumberHandler : FFmpegProcessHandler - + - + @@ -21,7 +21,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs b/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs index 9a3050c73..3fd6f1670 100644 --- a/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs +++ b/ErsatzTV.Core.Tests/FFmpeg/TranscodingTests.cs @@ -11,6 +11,7 @@ using ErsatzTV.Core.Interfaces.Images; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Metadata; using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Moq; using NUnit.Framework; @@ -220,6 +221,7 @@ public class TranscodingTests imageCache.Object, new Mock().Object, new Mock().Object, + new Mock().Object, LoggerFactory.CreateLogger()); var service = new FFmpegLibraryProcessService( @@ -318,6 +320,7 @@ public class TranscodingTests using Process process = await service.ForPlayoutItem( ExecutableName("ffmpeg"), + ExecutableName("ffprobe"), false, new Channel(Guid.NewGuid()) { diff --git a/ErsatzTV.Core/ErsatzTV.Core.csproj b/ErsatzTV.Core/ErsatzTV.Core.csproj index 1eb09ded9..b87579a62 100644 --- a/ErsatzTV.Core/ErsatzTV.Core.csproj +++ b/ErsatzTV.Core/ErsatzTV.Core.csproj @@ -7,11 +7,12 @@ - + + diff --git a/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs b/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs index 6248b174c..486efdd63 100644 --- a/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs +++ b/ErsatzTV.Core/FFmpeg/FFmpegLibraryProcessService.cs @@ -34,6 +34,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService public async Task ForPlayoutItem( string ffmpegPath, + string ffprobePath, bool saveReports, Channel channel, MediaVersion videoVersion, @@ -72,7 +73,13 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService targetFramerate); Option watermarkOptions = - await _ffmpegProcessService.GetWatermarkOptions(channel, globalWatermark, videoVersion, None, None); + await _ffmpegProcessService.GetWatermarkOptions( + ffprobePath, + channel, + globalWatermark, + videoVersion, + None, + None); Option> maybeFadePoints = watermarkOptions .Map(o => o.Watermark) @@ -316,6 +323,25 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService public Process WrapSegmenter(string ffmpegPath, bool saveReports, Channel channel, string scheme, string host) => _ffmpegProcessService.WrapSegmenter(ffmpegPath, saveReports, channel, scheme, host); + public Process ResizeImage(string ffmpegPath, string inputFile, string outputFile, int height) + { + var videoInputFile = new VideoInputFile( + inputFile, + new List { new(0, string.Empty, None, FrameSize.Unknown, None, true) }); + + var pipelineBuilder = new PipelineBuilder( + videoInputFile, + None, + None, + None, + FileSystemLayout.FFmpegReportsFolder, + _logger); + + FFmpegPipeline pipeline = pipelineBuilder.Resize(outputFile, new FrameSize(-1, height)); + + return GetProcess(ffmpegPath, videoInputFile, None, None, None, pipeline, false); + } + public Process ConvertToPng(string ffmpegPath, string inputFile, string outputFile) => _ffmpegProcessService.ConvertToPng(ffmpegPath, inputFile, outputFile); @@ -324,6 +350,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService public Task> GenerateSongImage( string ffmpegPath, + string ffprobePath, Option subtitleFile, Channel channel, Option globalWatermark, @@ -338,6 +365,7 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService CancellationToken cancellationToken) => _ffmpegProcessService.GenerateSongImage( ffmpegPath, + ffprobePath, subtitleFile, channel, globalWatermark, @@ -357,7 +385,8 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService Option audioInputFile, Option watermarkInputFile, Option concatInputFile, - FFmpegPipeline pipeline) + FFmpegPipeline pipeline, + bool log = true) { IEnumerable loggedSteps = pipeline.PipelineSteps.Map(ps => ps.GetType().Name); IEnumerable loggedVideoFilters = @@ -365,12 +394,15 @@ public class FFmpegLibraryProcessService : IFFmpegProcessService IEnumerable loggedAudioFilters = audioInputFile.Map(f => f.FilterSteps.Map(af => af.GetType().Name)).Flatten(); - _logger.LogDebug( - "FFmpeg pipeline {PipelineSteps}, {AudioFilters}, {VideoFilters}", - loggedSteps, - loggedAudioFilters, - loggedVideoFilters - ); + if (log) + { + _logger.LogDebug( + "FFmpeg pipeline {PipelineSteps}, {AudioFilters}, {VideoFilters}", + loggedSteps, + loggedAudioFilters, + loggedVideoFilters + ); + } IList environmentVariables = CommandGenerator.GenerateEnvironmentVariables(pipeline.PipelineSteps); diff --git a/ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs b/ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs index b589615d3..f90a53d75 100644 --- a/ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs +++ b/ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs @@ -1,21 +1,23 @@ using System.Diagnostics; using Bugsnag; using CliWrap; +using CliWrap.Buffered; using ErsatzTV.Core.Domain; -using ErsatzTV.Core.Domain.Filler; using ErsatzTV.Core.Interfaces.FFmpeg; using ErsatzTV.Core.Interfaces.Images; using ErsatzTV.FFmpeg.State; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; namespace ErsatzTV.Core.FFmpeg; -public class FFmpegProcessService : IFFmpegProcessService +public class FFmpegProcessService { private readonly IFFmpegStreamSelector _ffmpegStreamSelector; private readonly IImageCache _imageCache; private readonly ITempFilePool _tempFilePool; private readonly IClient _client; + private readonly IMemoryCache _memoryCache; private readonly ILogger _logger; private readonly FFmpegPlaybackSettingsCalculator _playbackSettingsCalculator; @@ -25,6 +27,7 @@ public class FFmpegProcessService : IFFmpegProcessService IImageCache imageCache, ITempFilePool tempFilePool, IClient client, + IMemoryCache memoryCache, ILogger logger) { _playbackSettingsCalculator = ffmpegPlaybackSettingsService; @@ -32,33 +35,10 @@ public class FFmpegProcessService : IFFmpegProcessService _imageCache = imageCache; _tempFilePool = tempFilePool; _client = client; + _memoryCache = memoryCache; _logger = logger; } - public Task ForPlayoutItem( - string ffmpegPath, - bool saveReports, - Channel channel, - MediaVersion videoVersion, - MediaVersion audioVersion, - string videoPath, - string audioPath, - DateTimeOffset start, - DateTimeOffset finish, - DateTimeOffset now, - Option globalWatermark, - VaapiDriver vaapiDriver, - string vaapiDevice, - bool hlsRealtime, - FillerKind fillerKind, - TimeSpan inPoint, - TimeSpan outPoint, - long ptsOffset, - Option targetFramerate) - { - throw new NotSupportedException(); - } - public async Task ForError( string ffmpegPath, Channel channel, @@ -140,11 +120,6 @@ public class FFmpegProcessService : IFFmpegProcessService } } - public Process ConcatChannel(string ffmpegPath, bool saveReports, Channel channel, string scheme, string host) - { - throw new NotSupportedException(); - } - public Process WrapSegmenter(string ffmpegPath, bool saveReports, Channel channel, string scheme, string host) { FFmpegPlaybackSettings playbackSettings = _playbackSettingsCalculator.ConcatSettings; @@ -186,6 +161,7 @@ public class FFmpegProcessService : IFFmpegProcessService public async Task> GenerateSongImage( string ffmpegPath, + string ffprobePath, Option subtitleFile, Channel channel, Option globalWatermark, @@ -220,7 +196,13 @@ public class FFmpegProcessService : IFFmpegProcessService : None; Option watermarkOptions = - await GetWatermarkOptions(channel, globalWatermark, videoVersion, watermarkOverride, watermarkPath); + await GetWatermarkOptions( + ffprobePath, + channel, + globalWatermark, + videoVersion, + watermarkOverride, + watermarkPath); FFmpegPlaybackSettings playbackSettings = _playbackSettingsCalculator.CalculateErrorSettings(channel.FFmpegProfile); @@ -289,6 +271,7 @@ public class FFmpegProcessService : IFFmpegProcessService displaySize.Width != target.Width || displaySize.Height != target.Height; internal async Task GetWatermarkOptions( + string ffprobePath, Channel channel, Option globalWatermark, MediaVersion videoVersion, @@ -320,7 +303,7 @@ public class FFmpegProcessService : IFFmpegProcessService await watermarkOverride.IfNoneAsync(channel.Watermark), customPath, None, - await _imageCache.IsAnimated(customPath)); + await IsAnimated(ffprobePath, customPath)); case ChannelWatermarkImageSource.ChannelLogo: Option maybeChannelPath = channel.Artwork .Filter(a => a.ArtworkKind == ArtworkKind.Logo) @@ -331,7 +314,7 @@ public class FFmpegProcessService : IFFmpegProcessService maybeChannelPath, None, await maybeChannelPath.Match( - p => _imageCache.IsAnimated(p), + p => IsAnimated(ffprobePath, p), () => Task.FromResult(false))); default: throw new NotSupportedException("Unsupported watermark image source"); @@ -352,7 +335,7 @@ public class FFmpegProcessService : IFFmpegProcessService await watermarkOverride.IfNoneAsync(watermark), customPath, None, - await _imageCache.IsAnimated(customPath)); + await IsAnimated(ffprobePath, customPath)); case ChannelWatermarkImageSource.ChannelLogo: Option maybeChannelPath = channel.Artwork .Filter(a => a.ArtworkKind == ArtworkKind.Logo) @@ -363,7 +346,7 @@ public class FFmpegProcessService : IFFmpegProcessService maybeChannelPath, None, await maybeChannelPath.Match( - p => _imageCache.IsAnimated(p), + p => IsAnimated(ffprobePath, p), () => Task.FromResult(false))); default: throw new NotSupportedException("Unsupported watermark image source"); @@ -373,4 +356,55 @@ public class FFmpegProcessService : IFFmpegProcessService return new WatermarkOptions(None, None, None, false); } + + private async Task IsAnimated(string ffprobePath, string path) + { + try + { + var cacheKey = $"image.animated.{Path.GetFileName(path)}"; + if (_memoryCache.TryGetValue(cacheKey, out bool animated)) + { + return animated; + } + + BufferedCommandResult result = await Cli.Wrap(ffprobePath) + .WithArguments( + new[] + { + "-loglevel", "error", + "-select_streams", "v:0", + "-count_frames", + "-show_entries", "stream=nb_read_frames", + "-print_format", "csv", + path + }) + .WithValidation(CommandResultValidation.None) + .ExecuteBufferedAsync(); + + if (result.ExitCode == 0) + { + string output = result.StandardOutput; + output = output.Replace("stream,", string.Empty); + if (int.TryParse(output, out int frameCount)) + { + bool isAnimated = frameCount > 1; + _memoryCache.Set(cacheKey, isAnimated, TimeSpan.FromDays(1)); + return isAnimated; + } + } + else + { + _logger.LogWarning( + "Error checking frame count for file {File}l exit code {ExitCode}", + path, + result.ExitCode); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error checking frame count for file {File}", path); + } + + return false; + } } \ No newline at end of file diff --git a/ErsatzTV.Core/FFmpeg/SongVideoGenerator.cs b/ErsatzTV.Core/FFmpeg/SongVideoGenerator.cs index 4c4ed4f5e..11a72dae7 100644 --- a/ErsatzTV.Core/FFmpeg/SongVideoGenerator.cs +++ b/ErsatzTV.Core/FFmpeg/SongVideoGenerator.cs @@ -31,6 +31,7 @@ public class SongVideoGenerator : ISongVideoGenerator Channel channel, Option maybeGlobalWatermark, string ffmpegPath, + string ffprobePath, CancellationToken cancellationToken) { Option subtitleFile = None; @@ -190,9 +191,7 @@ public class SongVideoGenerator : ISongVideoGenerator { string hash = hashes[NextRandom(hashes.Count)]; - backgroundPath = await _imageCache.WriteBlurHash( - hash, - channel.FFmpegProfile.Resolution); + backgroundPath = _imageCache.WriteBlurHash(hash, channel.FFmpegProfile.Resolution); videoVersion.Height = channel.FFmpegProfile.Resolution.Height; videoVersion.Width = channel.FFmpegProfile.Resolution.Width; @@ -214,6 +213,7 @@ public class SongVideoGenerator : ISongVideoGenerator Either maybeSongImage = await _ffmpegProcessService.GenerateSongImage( ffmpegPath, + ffprobePath, subtitleFile, channel, maybeGlobalWatermark, diff --git a/ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegProcessService.cs b/ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegProcessService.cs index 634800648..8f5bd4c0f 100644 --- a/ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegProcessService.cs +++ b/ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegProcessService.cs @@ -10,6 +10,7 @@ public interface IFFmpegProcessService { Task ForPlayoutItem( string ffmpegPath, + string ffprobePath, bool saveReports, Channel channel, MediaVersion videoVersion, @@ -41,12 +42,15 @@ public interface IFFmpegProcessService Process WrapSegmenter(string ffmpegPath, bool saveReports, Channel channel, string scheme, string host); + Process ResizeImage(string ffmpegPath, string inputFile, string outputFile, int height); + Process ConvertToPng(string ffmpegPath, string inputFile, string outputFile); Process ExtractAttachedPicAsPng(string ffmpegPath, string inputFile, int streamIndex, string outputFile); Task> GenerateSongImage( string ffmpegPath, + string ffprobePath, Option subtitleFile, Channel channel, Option globalWatermark, diff --git a/ErsatzTV.Core/Interfaces/FFmpeg/ISongVideoGenerator.cs b/ErsatzTV.Core/Interfaces/FFmpeg/ISongVideoGenerator.cs index 18ee2bb9a..66c6bb246 100644 --- a/ErsatzTV.Core/Interfaces/FFmpeg/ISongVideoGenerator.cs +++ b/ErsatzTV.Core/Interfaces/FFmpeg/ISongVideoGenerator.cs @@ -9,5 +9,6 @@ public interface ISongVideoGenerator Channel channel, Option maybeGlobalWatermark, string ffmpegPath, + string ffprobePath, CancellationToken cancellationToken); } \ No newline at end of file diff --git a/ErsatzTV.Core/Interfaces/Images/IImageCache.cs b/ErsatzTV.Core/Interfaces/Images/IImageCache.cs index 4f160b4ef..5bb912b50 100644 --- a/ErsatzTV.Core/Interfaces/Images/IImageCache.cs +++ b/ErsatzTV.Core/Interfaces/Images/IImageCache.cs @@ -5,11 +5,9 @@ namespace ErsatzTV.Core.Interfaces.Images; public interface IImageCache { - Task> ResizeImage(byte[] imageBuffer, int height); Task> SaveArtworkToCache(Stream stream, ArtworkKind artworkKind); Task> CopyArtworkToCache(string path, ArtworkKind artworkKind); string GetPathForImage(string fileName, ArtworkKind artworkKind, Option maybeMaxHeight); - Task IsAnimated(string fileName); - Task CalculateBlurHash(string fileName, ArtworkKind artworkKind, int x, int y); - Task WriteBlurHash(string blurHash, IDisplaySize targetSize); + string CalculateBlurHash(string fileName, ArtworkKind artworkKind, int x, int y); + string WriteBlurHash(string blurHash, IDisplaySize targetSize); } \ No newline at end of file diff --git a/ErsatzTV.Core/Metadata/LocalFolderScanner.cs b/ErsatzTV.Core/Metadata/LocalFolderScanner.cs index 18d3ee2d0..b0e671122 100644 --- a/ErsatzTV.Core/Metadata/LocalFolderScanner.cs +++ b/ErsatzTV.Core/Metadata/LocalFolderScanner.cs @@ -206,21 +206,9 @@ public abstract class LocalFolderScanner if (metadata is SongMetadata) { - artwork.BlurHash43 = await _imageCache.CalculateBlurHash( - cacheName, - artworkKind, - 4, - 3); - artwork.BlurHash54 = await _imageCache.CalculateBlurHash( - cacheName, - artworkKind, - 5, - 4); - artwork.BlurHash64 = await _imageCache.CalculateBlurHash( - cacheName, - artworkKind, - 6, - 4); + artwork.BlurHash43 = _imageCache.CalculateBlurHash(cacheName, artworkKind, 4, 3); + artwork.BlurHash54 = _imageCache.CalculateBlurHash(cacheName, artworkKind, 5, 4); + artwork.BlurHash64 = _imageCache.CalculateBlurHash(cacheName, artworkKind, 6, 4); } await _metadataRepository.UpdateArtworkPath(artwork); @@ -238,21 +226,9 @@ public abstract class LocalFolderScanner if (metadata is SongMetadata) { - artwork.BlurHash43 = await _imageCache.CalculateBlurHash( - cacheName, - artworkKind, - 4, - 3); - artwork.BlurHash54 = await _imageCache.CalculateBlurHash( - cacheName, - artworkKind, - 5, - 4); - artwork.BlurHash64 = await _imageCache.CalculateBlurHash( - cacheName, - artworkKind, - 6, - 4); + artwork.BlurHash43 = _imageCache.CalculateBlurHash(cacheName, artworkKind, 4, 3); + artwork.BlurHash54 = _imageCache.CalculateBlurHash(cacheName, artworkKind, 5, 4); + artwork.BlurHash64 = _imageCache.CalculateBlurHash(cacheName, artworkKind, 6, 4); } metadata.Artwork.Add(artwork); diff --git a/ErsatzTV.FFmpeg.Tests/ErsatzTV.FFmpeg.Tests.csproj b/ErsatzTV.FFmpeg.Tests/ErsatzTV.FFmpeg.Tests.csproj index 7a549293f..86e3e68cb 100644 --- a/ErsatzTV.FFmpeg.Tests/ErsatzTV.FFmpeg.Tests.csproj +++ b/ErsatzTV.FFmpeg.Tests/ErsatzTV.FFmpeg.Tests.csproj @@ -8,10 +8,10 @@ - + - + all diff --git a/ErsatzTV.FFmpeg.Tests/PipelineBuilderTests.cs b/ErsatzTV.FFmpeg.Tests/PipelineBuilderTests.cs index 84bb928fa..d4b66f743 100644 --- a/ErsatzTV.FFmpeg.Tests/PipelineBuilderTests.cs +++ b/ErsatzTV.FFmpeg.Tests/PipelineBuilderTests.cs @@ -155,6 +155,33 @@ public class PipelineGeneratorTests "-threads 1 -nostdin -hide_banner -nostats -loglevel error -fflags +genpts+discardcorrupt+igndts -i /tmp/whatever.mkv -map 0:1 -map 0:0 -muxdelay 0 -muxpreload 0 -movflags +faststart -flags cgop -sc_threshold 0 -c:v copy -c:a copy -f mpegts -mpegts_flags +initial_discontinuity pipe:1"); } + [Test] + public void Resize_Image_Test() + { + var height = 200; + + var videoInputFile = new VideoInputFile( + "/test/input/file.png", + new List + { + new(0, string.Empty, Option.None, FrameSize.Unknown, Option.None, true) + }); + + var pipelineBuilder = new PipelineBuilder( + videoInputFile, + Option.None, + Option.None, + Option.None, + "", + _logger); + + FFmpegPipeline result = pipelineBuilder.Resize("/test/output/file.jpg", new FrameSize(-1, height)); + + string command = PrintCommand(videoInputFile, None, None, None, result); + + command.Should().Be("-nostdin -hide_banner -nostats -loglevel error -i /test/input/file.png -vf scale=-1:200 /test/output/file.jpg"); + } + private static string PrintCommand( Option videoInputFile, Option audioInputFile, diff --git a/ErsatzTV.FFmpeg/Filter/ScaleImageFilter.cs b/ErsatzTV.FFmpeg/Filter/ScaleImageFilter.cs new file mode 100644 index 000000000..2f28a132a --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/ScaleImageFilter.cs @@ -0,0 +1,22 @@ +namespace ErsatzTV.FFmpeg.Filter; + +public class ScaleImageFilter : BaseFilter +{ + private readonly FrameSize _scaledSize; + + public ScaleImageFilter(FrameSize scaledSize) + { + _scaledSize = scaledSize; + } + + public override string Filter => $"scale={_scaledSize.Width}:{_scaledSize.Height}"; + + // public override IList OutputOptions => new List { "-q:v", "2" }; + + public override FrameState NextState(FrameState currentState) => currentState with + { + ScaledSize = _scaledSize, + PaddedSize = _scaledSize, + FrameDataLocation = FrameDataLocation.Software + }; +} diff --git a/ErsatzTV.FFmpeg/Filter/VideoFilter.cs b/ErsatzTV.FFmpeg/Filter/VideoFilter.cs new file mode 100644 index 000000000..2c93b6cc2 --- /dev/null +++ b/ErsatzTV.FFmpeg/Filter/VideoFilter.cs @@ -0,0 +1,27 @@ +using ErsatzTV.FFmpeg.Environment; + +namespace ErsatzTV.FFmpeg.Filter; + +public class VideoFilter : IPipelineStep +{ + private readonly IEnumerable _filterSteps; + + public VideoFilter(IEnumerable filterSteps) + { + _filterSteps = filterSteps; + } + + private IList Arguments() => + new List + { + "-vf", + string.Join(",", _filterSteps.Map(fs => fs.Filter)) + }; + + public IList EnvironmentVariables => Array.Empty(); + public IList GlobalOptions => Array.Empty(); + public IList InputOptions(InputFile inputFile) => Array.Empty(); + public IList FilterOptions => Arguments(); + public IList OutputOptions => Array.Empty(); + public FrameState NextState(FrameState currentState) => currentState; +} diff --git a/ErsatzTV.FFmpeg/FrameSize.cs b/ErsatzTV.FFmpeg/FrameSize.cs index 7854f1632..219e13159 100644 --- a/ErsatzTV.FFmpeg/FrameSize.cs +++ b/ErsatzTV.FFmpeg/FrameSize.cs @@ -1,3 +1,6 @@ namespace ErsatzTV.FFmpeg; -public record FrameSize(int Width, int Height); +public record FrameSize(int Width, int Height) +{ + public static FrameSize Unknown = new(-1, -1); +} diff --git a/ErsatzTV.FFmpeg/Option/FileNameOutputOption.cs b/ErsatzTV.FFmpeg/Option/FileNameOutputOption.cs new file mode 100644 index 000000000..5359c13e4 --- /dev/null +++ b/ErsatzTV.FFmpeg/Option/FileNameOutputOption.cs @@ -0,0 +1,13 @@ +namespace ErsatzTV.FFmpeg.Option; + +public class FileNameOutputOption : OutputOption +{ + private readonly string _outputFile; + + public FileNameOutputOption(string outputFile) + { + _outputFile = outputFile; + } + + public override IList OutputOptions => new List { _outputFile }; +} diff --git a/ErsatzTV.FFmpeg/PipelineBuilder.cs b/ErsatzTV.FFmpeg/PipelineBuilder.cs index 03b884348..5737069d2 100644 --- a/ErsatzTV.FFmpeg/PipelineBuilder.cs +++ b/ErsatzTV.FFmpeg/PipelineBuilder.cs @@ -1,4 +1,5 @@ -using ErsatzTV.FFmpeg.Decoder; +using System.Numerics; +using ErsatzTV.FFmpeg.Decoder; using ErsatzTV.FFmpeg.Encoder; using ErsatzTV.FFmpeg.Environment; using ErsatzTV.FFmpeg.Filter; @@ -53,6 +54,24 @@ public class PipelineBuilder _logger = logger; } + public FFmpegPipeline Resize(string outputFile, FrameSize scaledSize) + { + _pipelineSteps.Clear(); + _pipelineSteps.Add(new NoStandardInputOption()); + _pipelineSteps.Add(new HideBannerOption()); + _pipelineSteps.Add(new NoStatsOption()); + _pipelineSteps.Add(new LoglevelErrorOption()); + + IPipelineFilterStep scaleStep = new ScaleImageFilter(scaledSize); + _videoInputFile.Iter(f => f.FilterSteps.Add(scaleStep)); + + _pipelineSteps.Add(new VideoFilter(new[] { scaleStep })); + _pipelineSteps.Add(scaleStep); + _pipelineSteps.Add(new FileNameOutputOption(outputFile)); + + return new FFmpegPipeline(_pipelineSteps); + } + public FFmpegPipeline Concat(ConcatInputFile concatInputFile, FFmpegState ffmpegState) { concatInputFile.AddOption(new ConcatInputFormat()); @@ -87,7 +106,7 @@ public class PipelineBuilder return new FFmpegPipeline(_pipelineSteps); } - + public FFmpegPipeline Build(FFmpegState ffmpegState, FrameState desiredState) { var allVideoStreams = _videoInputFile.SelectMany(f => f.VideoStreams).ToList(); diff --git a/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj b/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj index 0ffcb5312..2cc0e0d3e 100644 --- a/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj +++ b/ErsatzTV.Infrastructure/ErsatzTV.Infrastructure.csproj @@ -8,7 +8,7 @@ - + @@ -27,7 +27,6 @@ - diff --git a/ErsatzTV.Infrastructure/Images/ImageCache.cs b/ErsatzTV.Infrastructure/Images/ImageCache.cs index 412d8102d..a7c312b9e 100644 --- a/ErsatzTV.Infrastructure/Images/ImageCache.cs +++ b/ErsatzTV.Infrastructure/Images/ImageCache.cs @@ -1,4 +1,5 @@ -using System.Security.Cryptography; +using System.Drawing.Imaging; +using System.Security.Cryptography; using System.Text; using ErsatzTV.Core; using ErsatzTV.Core.Domain; @@ -6,12 +7,8 @@ using ErsatzTV.Core.FFmpeg; using ErsatzTV.Core.Interfaces.FFmpeg; using ErsatzTV.Core.Interfaces.Images; using ErsatzTV.Core.Interfaces.Metadata; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging; -using SixLabors.ImageSharp; -using SixLabors.ImageSharp.Formats.Jpeg; -using SixLabors.ImageSharp.PixelFormats; -using SixLabors.ImageSharp.Processing; +using Decoder = System.Drawing.Common.Blurhash.Decoder; +using Encoder = System.Drawing.Common.Blurhash.Encoder; namespace ErsatzTV.Infrastructure.Images; @@ -19,43 +16,16 @@ public class ImageCache : IImageCache { private static readonly SHA1 Crypto; private readonly ILocalFileSystem _localFileSystem; - private readonly ILogger _logger; - private readonly IMemoryCache _memoryCache; private readonly ITempFilePool _tempFilePool; static ImageCache() => Crypto = SHA1.Create(); public ImageCache( ILocalFileSystem localFileSystem, - IMemoryCache memoryCache, - ITempFilePool tempFilePool, - ILogger logger) + ITempFilePool tempFilePool) { _localFileSystem = localFileSystem; - _memoryCache = memoryCache; _tempFilePool = tempFilePool; - _logger = logger; - } - - public async Task> ResizeImage(byte[] imageBuffer, int height) - { - await using var inStream = new MemoryStream(imageBuffer); - using Image image = await Image.LoadAsync(inStream); - - var size = new Size { Height = height }; - - image.Mutate( - i => i.Resize( - new ResizeOptions - { - Mode = ResizeMode.Max, - Size = size - })); - - await using var outStream = new MemoryStream(); - await image.SaveAsync(outStream, new JpegEncoder { Quality = 90 }); - - return outStream.ToArray(); } public async Task> SaveArtworkToCache(Stream stream, ArtworkKind artworkKind) @@ -156,39 +126,18 @@ public class ImageCache : IImageCache return Path.Combine(baseFolder, fileName); } - public async Task IsAnimated(string fileName) + public string CalculateBlurHash(string fileName, ArtworkKind artworkKind, int x, int y) { - try - { - var cacheKey = $"image.animated.{Path.GetFileName(fileName)}"; - if (_memoryCache.TryGetValue(cacheKey, out bool animated)) - { - return animated; - } - - using Image image = await Image.LoadAsync(fileName); - animated = image.Frames.Count > 1; - _memoryCache.Set(cacheKey, animated, TimeSpan.FromDays(1)); - - return animated; - } - catch (Exception ex) - { - _logger.LogError(ex, "Unable to check image for animation"); - return false; - } - } - - public async Task CalculateBlurHash(string fileName, ArtworkKind artworkKind, int x, int y) - { - var encoder = new Blurhash.ImageSharp.Encoder(); + var encoder = new Encoder(); string targetFile = GetPathForImage(fileName, artworkKind, Option.None); - await using var fs = new FileStream(targetFile, FileMode.Open, FileAccess.Read); - using var image = await Image.LoadAsync(fs); - return encoder.Encode(image, x, y); + // ReSharper disable once ConvertToUsingDeclaration + using (var image = System.Drawing.Image.FromFile(targetFile)) + { + return encoder.Encode(image, x, y); + } } - public async Task WriteBlurHash(string blurHash, IDisplaySize targetSize) + public string WriteBlurHash(string blurHash, IDisplaySize targetSize) { byte[] bytes = Encoding.UTF8.GetBytes(blurHash); string base64 = Convert.ToBase64String(bytes).Replace("+", "_").Replace("/", "-").Replace("=", ""); @@ -197,10 +146,13 @@ public class ImageCache : IImageCache { string folder = Path.GetDirectoryName(targetFile); _localFileSystem.EnsureFolderExists(folder); - - var decoder = new Blurhash.ImageSharp.Decoder(); - using Image image = decoder.Decode(blurHash, targetSize.Width, targetSize.Height); - await image.SaveAsPngAsync(targetFile); + + var decoder = new Decoder(); + // ReSharper disable once ConvertToUsingDeclaration + using (System.Drawing.Image image = decoder.Decode(blurHash, targetSize.Width, targetSize.Height)) + { + image.Save(targetFile, ImageFormat.Png); + } } return targetFile; diff --git a/ErsatzTV/ErsatzTV.csproj b/ErsatzTV/ErsatzTV.csproj index 1b459d778..6b2e1c075 100644 --- a/ErsatzTV/ErsatzTV.csproj +++ b/ErsatzTV/ErsatzTV.csproj @@ -54,12 +54,12 @@ - + - + @@ -72,7 +72,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - +