ersatztv/ErsatzTV.Infrastructure/Streaming/Graphics/Image/ImageElementBase.cs

using System.Runtime.InteropServices;
using ErsatzTV.Core.Domain;
using ErsatzTV.Core.Images;
using ErsatzTV.Core.Interfaces.Streaming;
using ErsatzTV.FFmpeg.State;
using ErsatzTV.Infrastructure.Images;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.Formats;
using SixLabors.ImageSharp.Formats.Gif;
using SixLabors.ImageSharp.Formats.Png;
using SixLabors.ImageSharp.Formats.Webp;
using SixLabors.ImageSharp.Metadata;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
using SkiaSharp;
using Image = SixLabors.ImageSharp.Image;

namespace ErsatzTV.Infrastructure.Streaming.Graphics;

public abstract class ImageElementBase(IRemoteImageFetcher remoteImageFetcher) : GraphicsElement, IDisposable
{
    /// <summary>
    ///     Ceiling on total pixels RETAINED after scaling — frames x scaled width x scaled height.
    ///     Independent of the source decode budget in <see cref="RemoteImageDecodeBudget" />: a
    ///     100x100 source is trivial to decode but, at 600
    ///     frames scaled to 1920x1080, retains ~5 GB of <see cref="SKBitmap" />. At 4 bytes per
    ///     pixel this bounds retention at ~800 MB, which still allows ~96 full-frame 1080p frames
    ///     (~3s at 30fps) or 600 frames of a 577x577 logo.
    /// </summary>
    internal const long MaxRemoteScaledPixels = 200_000_000;

    private readonly List<double> _frameDelays = [];
    private readonly List<SKBitmap> _scaledFrames = [];
    private double _animatedDurationSeconds;
    private ushort _repeatCount;

    private Image _sourceImage;

    protected SKPointI Location { get; private set; }

    public virtual void Dispose()
    {
        GC.SuppressFinalize(this);
        _sourceImage?.Dispose();
        _scaledFrames?.ForEach(f => f.Dispose());
    }

    protected async Task LoadImage(
        Resolution squarePixelFrameSize,
        Resolution frameSize,
        string image,
        WatermarkLocation location,
        bool scale,
        double? scaleWidthPercent,
        double? horizontalMarginPercent,
        double? verticalMarginPercent,
        bool placeWithinSourceContent,
        CancellationToken cancellationToken)
    {
        bool isRemoteUri = Uri.TryCreate(image, UriKind.Absolute, out Uri uriResult)
                           && (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps);

        if (isRemoteUri)
        {
            await using Stream imageStream = await remoteImageFetcher.Fetch(uriResult, cancellationToken);
            _sourceImage = await DecodeRemoteImage(imageStream, uriResult, cancellationToken);
        }
        else
        {
            _sourceImage = await Image.LoadAsync(image!, cancellationToken);
        }

        int scaledWidth = _sourceImage.Width;
        int scaledHeight = _sourceImage.Height;
        if (scale)
        {
            scaledWidth = (int)Math.Round((scaleWidthPercent ?? 100) / 100.0 * frameSize.Width);
            double aspectRatio = (double)_sourceImage.Height / _sourceImage.Width;
            scaledHeight = (int)(scaledWidth * aspectRatio);
        }

        if (isRemoteUri)
        {
            EnsureScaledFramesAffordable(_sourceImage.Frames.Count, scaledWidth, scaledHeight, uriResult);
        }

        (int horizontalMargin, int verticalMargin) = placeWithinSourceContent
            ? SourceContentMargins(
                squarePixelFrameSize,
                frameSize,
                horizontalMarginPercent ?? 0,
                verticalMarginPercent ?? 0)
            : NormalMargins(frameSize, horizontalMarginPercent ?? 0, verticalMarginPercent ?? 0);

        Location = CalculatePosition(
            location,
            frameSize.Width,
            frameSize.Height,
            scaledWidth,
            scaledHeight,
            horizontalMargin,
            verticalMargin);

        if (_sourceImage.Metadata.DecodedImageFormat == GifFormat.Instance)
        {
            _repeatCount = _sourceImage.Metadata.GetFormatMetadata(GifFormat.Instance).RepeatCount;
        }

        _animatedDurationSeconds = 0;

        for (var i = 0; i < _sourceImage.Frames.Count; i++)
        {
            Image frame = _sourceImage.Frames.CloneFrame(i);
            frame.Mutate(ctx => ctx.Resize(scaledWidth, scaledHeight));
            _scaledFrames.Add(ToSkiaBitmap(frame));

            double frameDelay = GetFrameDelaySeconds(_sourceImage, i);
            _animatedDurationSeconds += frameDelay;
            _frameDelays.Add(frameDelay);
        }
    }

    /// <summary>
    ///     Decodes a remote image only after the header says decoding it is affordable.
    /// </summary>
    /// <remarks>
    ///     The fetcher's byte cap does NOT bound this: a decompression bomb is small on the wire and
    ///     huge in memory. A 4 KB PNG can declare 30000x30000 (~3.6 GB), and a 60 KiB GIF can
    ///     declare 2500x2500 across 600 frames (~14 GiB). The budget is therefore on the PRODUCT of
    ///     dimensions and frames, read from the header before the decoder allocates.
    ///     Local images are deliberately not checked — they are files an operator put on disk, not
    ///     bytes an arbitrary host returned. (ersatztv#511)
    /// </remarks>
    internal static async Task<Image> DecodeRemoteImage(Stream stream, Uri uri, CancellationToken cancellationToken)
        => await RemoteImageValidator.DecodeAndValidate(stream, uri, cancellationToken);

    /// <summary>
    ///     Bounds what is RETAINED after scaling. Separate from the source budget because the two
    ///     are independent: a cheap-to-decode 100x100 source scaled to 1920x1080 across 600 frames
    ///     retains ~5 GB. Only applied to remote images, matching the rest of this guard.
    /// </summary>
    internal static void EnsureScaledFramesAffordable(int frameCount, int scaledWidth, int scaledHeight, Uri uri)
    {
        long retainedPixels = (long)Math.Max(frameCount, 1) * scaledWidth * scaledHeight;
        if (retainedPixels > MaxRemoteScaledPixels)
        {
            throw new InvalidOperationException(
                $"Remote image {uri} scales to {frameCount} frames of {scaledWidth}x{scaledHeight} "
                + $"({retainedPixels} pixels), over the {MaxRemoteScaledPixels} pixel limit");
        }
    }

    protected static SKBitmap ToSkiaBitmap(Image image)
    {
        using Image<Rgba32> rgbaImage = image.CloneAs<Rgba32>();

        int width = rgbaImage.Width;
        int height = rgbaImage.Height;

        var info = new SKImageInfo(width, height, SKColorType.Rgba8888, SKAlphaType.Unpremul);
        var skBitmap = new SKBitmap(info);
        if (!skBitmap.TryAllocPixels(info))
        {
            skBitmap.Dispose();
            throw new InvalidOperationException("Failed to allocate pixels for SKBitmap.");
        }

        var pixelArray = new Rgba32[width * height];
        rgbaImage.CopyPixelDataTo(pixelArray);

        var bytes = new byte[pixelArray.Length * 4];
        MemoryMarshal.AsBytes(pixelArray.AsSpan()).CopyTo(bytes);

        IntPtr dstPtr = skBitmap.GetPixels(out _);
        Marshal.Copy(bytes, 0, dstPtr, bytes.Length);

        return skBitmap;
    }

    protected static double GetFrameDelaySeconds(Image image, int frameIndex)
    {
        IImageFormat format = image.Metadata.DecodedImageFormat;
        ImageFrameMetadata frameMeta = image.Frames[frameIndex].Metadata;

        if (format == GifFormat.Instance)
        {
            // GIF frame delay is in hundredths of a second
            GifFrameMetadata gifMeta = frameMeta.GetFormatMetadata(GifFormat.Instance);
            return gifMeta.FrameDelay / 100.0;
        }

        if (format == PngFormat.Instance)
        {
            // PNG animated frame delay is in seconds (as double)
            PngFrameMetadata pngMeta = frameMeta.GetFormatMetadata(PngFormat.Instance);
            return pngMeta.FrameDelay.ToDouble();
        }

        if (format == WebpFormat.Instance)
        {
            // WEBP animated frame delay is in milliseconds
            WebpFrameMetadata webpMeta = frameMeta.GetFormatMetadata(WebpFormat.Instance);
            return webpMeta.FrameDelay / 1000.0;
        }

        // Default: assume 1/60th second (~16.67 ms) if unknown
        return 1.0 / 60.0;
    }

    protected SKBitmap GetFrameForTimestamp(TimeSpan timestamp)
    {
        if (_scaledFrames.Count <= 1)
        {
            return _scaledFrames[0];
        }

        if (_repeatCount > 0 && timestamp.TotalSeconds >= _animatedDurationSeconds * _repeatCount)
        {
            return _scaledFrames.Last();
        }

        double currentTime = timestamp.TotalSeconds % _animatedDurationSeconds;

        double frameTime = 0;
        for (var i = 0; i < _sourceImage.Frames.Count; i++)
        {
            frameTime += _frameDelays[i];
            if (currentTime <= frameTime)
            {
                return _scaledFrames[i];
            }
        }

        return _scaledFrames.Last();
    }
}