nfo and memory fixes (#789)

* partial episode nfo metadata * nfo metadata reliability fixes * use recyclable memory streams
2022-05-07 20:32:57 -05:00
parent 6326189444
commit 44dd68fe59
32 changed files with 542 additions and 131 deletions
@@ -2,22 +2,43 @@
 using Bugsnag;
 using ErsatzTV.Core.Errors;
 using ErsatzTV.Core.Interfaces.Metadata.Nfo;
+using Microsoft.Extensions.Logging;
+using Microsoft.IO;

 namespace ErsatzTV.Core.Metadata.Nfo;

 public class ArtistNfoReader : NfoReader<ArtistNfo>, IArtistNfoReader
 {
    private readonly IClient _client;
+    private readonly ILogger<ArtistNfoReader> _logger;

-    public ArtistNfoReader(IClient client) => _client = client;
-
-    public async Task<Either<BaseError, ArtistNfo>> Read(Stream input)
+    public ArtistNfoReader(
+        RecyclableMemoryStreamManager recyclableMemoryStreamManager,
+        IClient client,
+        ILogger<ArtistNfoReader> logger)
+        : base(recyclableMemoryStreamManager, logger)
    {
+        _client = client;
+        _logger = logger;
+    }
+
+    public async Task<Either<BaseError, ArtistNfo>> ReadFromFile(string fileName)
+    {
+        // ReSharper disable once ConvertToUsingDeclaration
+        await using (Stream s = await SanitizedStreamForFile(fileName))
+        {
+            return await Read(s);
+        }
+    }
+
+    internal async Task<Either<BaseError, ArtistNfo>> Read(Stream input)
+    {
+        ArtistNfo nfo = null;
+
        try
        {
            var settings = new XmlReaderSettings { Async = true, ConformanceLevel = ConformanceLevel.Fragment };
            using var reader = XmlReader.Create(input, settings);
-            ArtistNfo nfo = null;
            var done = false;

            while (!done && await reader.ReadAsync())
@@ -74,6 +95,13 @@ public class ArtistNfoReader : NfoReader<ArtistNfo>, IArtistNfoReader

            return Optional(nfo).ToEither((BaseError)new FailedToReadNfo());
        }
+        catch (XmlException ex) when (ex.Message.Contains(
+                                          "invalid character",
+                                          StringComparison.InvariantCultureIgnoreCase))
+        {
+            _logger.LogWarning("Invalid XML detected; returning incomplete metadata");
+            return Optional(nfo).ToEither((BaseError)new FailedToReadNfo());
+        }
        catch (Exception ex)
        {
            _client.Notify(ex);
@@ -2,23 +2,42 @@
 using Bugsnag;
 using ErsatzTV.Core.Errors;
 using ErsatzTV.Core.Interfaces.Metadata.Nfo;
+using Microsoft.Extensions.Logging;
+using Microsoft.IO;

 namespace ErsatzTV.Core.Metadata.Nfo;

 public class EpisodeNfoReader : NfoReader<TvShowEpisodeNfo>, IEpisodeNfoReader
 {
    private readonly IClient _client;
+    private readonly ILogger<EpisodeNfoReader> _logger;

-    public EpisodeNfoReader(IClient client) => _client = client;
-
-    public async Task<Either<BaseError, List<TvShowEpisodeNfo>>> Read(Stream input)
+    public EpisodeNfoReader(
+        RecyclableMemoryStreamManager recyclableMemoryStreamManager,
+        IClient client,
+        ILogger<EpisodeNfoReader> logger)
+        : base(recyclableMemoryStreamManager, logger)
    {
+        _client = client;
+        _logger = logger;
+    }
+
+    public async Task<Either<BaseError, List<TvShowEpisodeNfo>>> ReadFromFile(string fileName)
+    {
+        // ReSharper disable once ConvertToUsingDeclaration
+        await using (Stream s = await SanitizedStreamForFile(fileName))
+        {
+            return await Read(s);
+        }
+    }
+
+    internal async Task<Either<BaseError, List<TvShowEpisodeNfo>>> Read(Stream input)
+    {
+        var result = new List<TvShowEpisodeNfo>();
+
        try
        {
-            var result = new List<TvShowEpisodeNfo>();
-
-            var settings = new XmlReaderSettings { Async = true, ConformanceLevel = ConformanceLevel.Fragment };
-            using var reader = XmlReader.Create(input, settings);
+            using var reader = XmlReader.Create(input, Settings);
            TvShowEpisodeNfo nfo = null;

            while (await reader.ReadAsync())
@@ -36,6 +55,8 @@ public class EpisodeNfoReader : NfoReader<TvShowEpisodeNfo>, IEpisodeNfoReader
                                    Writers = new List<string>(),
                                    Directors = new List<string>()
                                };
+                                // immediately add so we have something to return if we encounter invalid characters
+                                result.Add(nfo);
                                break;
                            case "title":
                                await ReadStringContent(reader, nfo, (episode, title) => episode.Title = title);
@@ -87,25 +108,19 @@ public class EpisodeNfoReader : NfoReader<TvShowEpisodeNfo>, IEpisodeNfoReader
                                break;
                        }

-                        break;
-                    case XmlNodeType.EndElement:
-                        switch (reader.Name.ToLowerInvariant())
-                        {
-                            case "episodedetails":
-                                if (nfo != null)
-                                {
-                                    result.Add(nfo);
-                                }
-
-                                break;
-                        }
-
                        break;
                }
            }

            return result;
        }
+        catch (XmlException ex) when (ex.Message.Contains(
+                                          "invalid character",
+                                          StringComparison.InvariantCultureIgnoreCase))
+        {
+            _logger.LogWarning("Invalid XML detected; returning incomplete metadata");
+            return result;
+        }
        catch (Exception ex)
        {
            _client.Notify(ex);
@@ -2,22 +2,43 @@
 using Bugsnag;
 using ErsatzTV.Core.Errors;
 using ErsatzTV.Core.Interfaces.Metadata.Nfo;
+using Microsoft.Extensions.Logging;
+using Microsoft.IO;

 namespace ErsatzTV.Core.Metadata.Nfo;

 public class MovieNfoReader : NfoReader<MovieNfo>, IMovieNfoReader
 {
    private readonly IClient _client;
+    private readonly ILogger<MovieNfoReader> _logger;

-    public MovieNfoReader(IClient client) => _client = client;
-
-    public async Task<Either<BaseError, MovieNfo>> Read(Stream input)
+    public MovieNfoReader(
+        RecyclableMemoryStreamManager recyclableMemoryStreamManager,
+        IClient client,
+        ILogger<MovieNfoReader> logger)
+        : base(recyclableMemoryStreamManager, logger)
    {
+        _client = client;
+        _logger = logger;
+    }
+
+    public async Task<Either<BaseError, MovieNfo>> ReadFromFile(string fileName)
+    {
+        // ReSharper disable once ConvertToUsingDeclaration
+        await using (Stream s = await SanitizedStreamForFile(fileName))
+        {
+            return await Read(s);
+        }
+    }
+
+    internal async Task<Either<BaseError, MovieNfo>> Read(Stream input)
+    {
+        MovieNfo nfo = null;
+
        try
        {
            var settings = new XmlReaderSettings { Async = true, ConformanceLevel = ConformanceLevel.Fragment };
            using var reader = XmlReader.Create(input, settings);
-            MovieNfo nfo = null;
            var done = false;

            while (!done && await reader.ReadAsync())
@@ -105,6 +126,13 @@ public class MovieNfoReader : NfoReader<MovieNfo>, IMovieNfoReader

            return Optional(nfo).ToEither((BaseError)new FailedToReadNfo());
        }
+        catch (XmlException ex) when (ex.Message.Contains(
+                                          "invalid character",
+                                          StringComparison.InvariantCultureIgnoreCase))
+        {
+            _logger.LogWarning("Invalid XML detected; returning incomplete metadata");
+            return Optional(nfo).ToEither((BaseError)new FailedToReadNfo());
+        }
        catch (Exception ex)
        {
            _client.Notify(ex);
@@ -2,22 +2,43 @@
 using Bugsnag;
 using ErsatzTV.Core.Errors;
 using ErsatzTV.Core.Interfaces.Metadata.Nfo;
+using Microsoft.Extensions.Logging;
+using Microsoft.IO;

 namespace ErsatzTV.Core.Metadata.Nfo;

 public class MusicVideoNfoReader : NfoReader<MusicVideoNfo>, IMusicVideoNfoReader
 {
    private readonly IClient _client;
+    private readonly ILogger<MusicVideoNfoReader> _logger;

-    public MusicVideoNfoReader(IClient client) => _client = client;
-
-    public async Task<Either<BaseError, MusicVideoNfo>> Read(Stream input)
+    public MusicVideoNfoReader(
+        RecyclableMemoryStreamManager recyclableMemoryStreamManager,
+        IClient client,
+        ILogger<MusicVideoNfoReader> logger)
+        : base(recyclableMemoryStreamManager, logger)
    {
+        _client = client;
+        _logger = logger;
+    }
+
+    public async Task<Either<BaseError, MusicVideoNfo>> ReadFromFile(string fileName)
+    {
+        // ReSharper disable once ConvertToUsingDeclaration
+        await using (Stream s = await SanitizedStreamForFile(fileName))
+        {
+            return await Read(s);
+        }
+    }
+
+    internal async Task<Either<BaseError, MusicVideoNfo>> Read(Stream input)
+    {
+        MusicVideoNfo nfo = null;
+
        try
        {
            var settings = new XmlReaderSettings { Async = true, ConformanceLevel = ConformanceLevel.Fragment };
            using var reader = XmlReader.Create(input, settings);
-            MusicVideoNfo nfo = null;
            var done = false;

            while (!done && await reader.ReadAsync())
@@ -87,6 +108,13 @@ public class MusicVideoNfoReader : NfoReader<MusicVideoNfo>, IMusicVideoNfoReade

            return Optional(nfo).ToEither((BaseError)new FailedToReadNfo());
        }
+        catch (XmlException ex) when (ex.Message.Contains(
+                                          "invalid character",
+                                          StringComparison.InvariantCultureIgnoreCase))
+        {
+            _logger.LogWarning("Invalid XML detected; returning incomplete metadata");
+            return Optional(nfo).ToEither((BaseError)new FailedToReadNfo());
+        }
        catch (Exception ex)
        {
            _client.Notify(ex);
@@ -1,83 +1,166 @@
-using System.Xml;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Xml;
 using System.Xml.Linq;
+using Microsoft.Extensions.Logging;
+using Microsoft.IO;

 namespace ErsatzTV.Core.Metadata.Nfo;

 public abstract class NfoReader<T>
 {
-    protected static async Task ReadStringContent(XmlReader reader, T nfo, Action<T, string> action)
-    {
-        if (nfo != null)
+    private static readonly byte[] Buffer = new byte[8 * 1024 * 1024];
+    private static readonly Regex Pattern = new(@"[\p{C}-[\r\n\t]]+");
+
+    protected static readonly XmlReaderSettings Settings =
+        new()
        {
-            string result = await reader.ReadElementContentAsStringAsync();
-            action(nfo, result);
+            Async = true,
+            ConformanceLevel = ConformanceLevel.Fragment,
+            ValidationType = ValidationType.None,
+            CheckCharacters = false,
+            IgnoreProcessingInstructions = true,
+            IgnoreComments = true
+        };
+
+    private readonly ILogger _logger;
+
+    private readonly RecyclableMemoryStreamManager _recyclableMemoryStreamManager;
+
+    protected NfoReader(RecyclableMemoryStreamManager recyclableMemoryStreamManager, ILogger logger)
+    {
+        _recyclableMemoryStreamManager = recyclableMemoryStreamManager;
+        _logger = logger;
+    }
+
+    protected async Task<Stream> SanitizedStreamForFile(string fileName)
+    {
+        using (var fs = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read, Buffer.Length, true))
+        {
+            while (await fs.ReadAsync(Buffer) > 0)
+            {
+                // read the file
+            }
+
+            string text = Encoding.UTF8.GetString(Buffer);
+            // trim BOM and zero width space, replace controls with replacement character
+            string stripped = Pattern.Replace(text.Trim('\uFEFF', '\u200B'), "\ufffd");
+
+            MemoryStream ms = _recyclableMemoryStreamManager.GetStream();
+            await ms.WriteAsync(Encoding.UTF8.GetBytes(stripped));
+            ms.Position = 0;
+            return ms;
        }
    }

-    protected static async Task ReadIntContent(XmlReader reader, T nfo, Action<T, int> action)
+    protected async Task ReadStringContent(XmlReader reader, T nfo, Action<T, string> action)
    {
-        if (nfo != null && int.TryParse(await reader.ReadElementContentAsStringAsync(), out int result))
+        try
        {
-            action(nfo, result);
+            if (nfo != null)
+            {
+                string result = await reader.ReadElementContentAsStringAsync();
+                action(nfo, result);
+            }
+        }
+        catch (XmlException ex)
+        {
+            _logger.LogWarning(ex, "Error reading string content from NFO {ElementName}", reader.Name);
        }
    }

-    protected static async Task ReadDateTimeContent(XmlReader reader, T nfo, Action<T, DateTime> action)
+    protected async Task ReadIntContent(XmlReader reader, T nfo, Action<T, int> action)
    {
-        if (nfo != null && DateTime.TryParse(await reader.ReadElementContentAsStringAsync(), out DateTime result))
+        try
        {
-            action(nfo, result);
+            if (nfo != null && int.TryParse(await reader.ReadElementContentAsStringAsync(), out int result))
+            {
+                action(nfo, result);
+            }
+        }
+        catch (XmlException ex)
+        {
+            _logger.LogWarning(ex, "Error reading int content from NFO {ElementName}", reader.Name);
        }
    }

-    protected static void ReadActor(XmlReader reader, T nfo, Action<T, ActorNfo> action)
+    protected async Task ReadDateTimeContent(XmlReader reader, T nfo, Action<T, DateTime> action)
    {
-        if (nfo != null)
+        try
        {
-            var actor = new ActorNfo();
-            var element = (XElement)XNode.ReadFrom(reader);
-
-            XElement name = element.Element("name");
-            if (name != null)
+            if (nfo != null && DateTime.TryParse(await reader.ReadElementContentAsStringAsync(), out DateTime result))
            {
-                actor.Name = name.Value;
+                action(nfo, result);
            }
-
-            XElement role = element.Element("role");
-            if (role != null)
-            {
-                actor.Role = role.Value;
-            }
-
-            XElement order = element.Element("order");
-            if (order != null && int.TryParse(order.Value, out int orderValue))
-            {
-                actor.Order = orderValue;
-            }
-
-            XElement thumb = element.Element("thumb");
-            if (thumb != null)
-            {
-                actor.Thumb = thumb.Value;
-            }
-
-            action(nfo, actor);
+        }
+        catch (XmlException ex)
+        {
+            _logger.LogWarning(ex, "Error reading date content from NFO {ElementName}", reader.Name);
        }
    }

-    protected static async Task ReadUniqueId(XmlReader reader, T nfo, Action<T, UniqueIdNfo> action)
+    protected void ReadActor(XmlReader reader, T nfo, Action<T, ActorNfo> action)
    {
-        if (nfo != null)
+        try
        {
-            var uniqueId = new UniqueIdNfo();
-            reader.MoveToAttribute("default");
-            uniqueId.Default = bool.TryParse(reader.Value, out bool def) && def;
-            reader.MoveToAttribute("type");
-            uniqueId.Type = reader.Value;
-            reader.MoveToElement();
-            uniqueId.Guid = await reader.ReadElementContentAsStringAsync();
+            if (nfo != null)
+            {
+                var actor = new ActorNfo();
+                var element = (XElement)XNode.ReadFrom(reader);

-            action(nfo, uniqueId);
+                XElement name = element.Element("name");
+                if (name != null)
+                {
+                    actor.Name = name.Value;
+                }
+
+                XElement role = element.Element("role");
+                if (role != null)
+                {
+                    actor.Role = role.Value;
+                }
+
+                XElement order = element.Element("order");
+                if (order != null && int.TryParse(order.Value, out int orderValue))
+                {
+                    actor.Order = orderValue;
+                }
+
+                XElement thumb = element.Element("thumb");
+                if (thumb != null)
+                {
+                    actor.Thumb = thumb.Value;
+                }
+
+                action(nfo, actor);
+            }
+        }
+        catch (XmlException ex)
+        {
+            _logger.LogWarning(ex, "Error reading actor content from NFO {ElementName}", reader.Name);
+        }
+    }
+
+    protected async Task ReadUniqueId(XmlReader reader, T nfo, Action<T, UniqueIdNfo> action)
+    {
+        try
+        {
+            if (nfo != null)
+            {
+                var uniqueId = new UniqueIdNfo();
+                reader.MoveToAttribute("default");
+                uniqueId.Default = bool.TryParse(reader.Value, out bool def) && def;
+                reader.MoveToAttribute("type");
+                uniqueId.Type = reader.Value;
+                reader.MoveToElement();
+                uniqueId.Guid = await reader.ReadElementContentAsStringAsync();
+
+                action(nfo, uniqueId);
+            }
+        }
+        catch (XmlException ex)
+        {
+            _logger.LogWarning(ex, "Error reading uniqueid content from NFO {ElementName}", reader.Name);
        }
    }
 }
@@ -2,22 +2,43 @@
 using Bugsnag;
 using ErsatzTV.Core.Errors;
 using ErsatzTV.Core.Interfaces.Metadata.Nfo;
+using Microsoft.Extensions.Logging;
+using Microsoft.IO;

 namespace ErsatzTV.Core.Metadata.Nfo;

 public class OtherVideoNfoReader : NfoReader<OtherVideoNfo>, IOtherVideoNfoReader
 {
    private readonly IClient _client;
+    private readonly ILogger<OtherVideoNfoReader> _logger;

-    public OtherVideoNfoReader(IClient client) => _client = client;
-
-    public async Task<Either<BaseError, OtherVideoNfo>> Read(Stream input)
+    public OtherVideoNfoReader(
+        RecyclableMemoryStreamManager recyclableMemoryStreamManager,
+        IClient client,
+        ILogger<OtherVideoNfoReader> logger)
+        : base(recyclableMemoryStreamManager, logger)
    {
+        _client = client;
+        _logger = logger;
+    }
+
+    public async Task<Either<BaseError, OtherVideoNfo>> ReadFromFile(string fileName)
+    {
+        // ReSharper disable once ConvertToUsingDeclaration
+        await using (Stream s = await SanitizedStreamForFile(fileName))
+        {
+            return await Read(s);
+        }
+    }
+
+    internal async Task<Either<BaseError, OtherVideoNfo>> Read(Stream input)
+    {
+        OtherVideoNfo nfo = null;
+
        try
        {
            var settings = new XmlReaderSettings { Async = true, ConformanceLevel = ConformanceLevel.Fragment };
            using var reader = XmlReader.Create(input, settings);
-            OtherVideoNfo nfo = null;
            var done = false;

            while (!done && await reader.ReadAsync())
@@ -105,6 +126,13 @@ public class OtherVideoNfoReader : NfoReader<OtherVideoNfo>, IOtherVideoNfoReade

            return Optional(nfo).ToEither((BaseError)new FailedToReadNfo());
        }
+        catch (XmlException ex) when (ex.Message.Contains(
+                                          "invalid character",
+                                          StringComparison.InvariantCultureIgnoreCase))
+        {
+            _logger.LogWarning("Invalid XML detected; returning incomplete metadata");
+            return Optional(nfo).ToEither((BaseError)new FailedToReadNfo());
+        }
        catch (Exception ex)
        {
            _client.Notify(ex);
@@ -2,22 +2,43 @@
 using Bugsnag;
 using ErsatzTV.Core.Errors;
 using ErsatzTV.Core.Interfaces.Metadata.Nfo;
+using Microsoft.Extensions.Logging;
+using Microsoft.IO;

 namespace ErsatzTV.Core.Metadata.Nfo;

 public class TvShowNfoReader : NfoReader<TvShowNfo>, ITvShowNfoReader
 {
    private readonly IClient _client;
+    private readonly ILogger<TvShowNfoReader> _logger;

-    public TvShowNfoReader(IClient client) => _client = client;
-
-    public async Task<Either<BaseError, TvShowNfo>> Read(Stream input)
+    public TvShowNfoReader(
+        RecyclableMemoryStreamManager recyclableMemoryStreamManager,
+        IClient client,
+        ILogger<TvShowNfoReader> logger)
+        : base(recyclableMemoryStreamManager, logger)
    {
+        _client = client;
+        _logger = logger;
+    }
+
+    public async Task<Either<BaseError, TvShowNfo>> ReadFromFile(string fileName)
+    {
+        // ReSharper disable once ConvertToUsingDeclaration
+        await using (Stream s = await SanitizedStreamForFile(fileName))
+        {
+            return await Read(s);
+        }
+    }
+
+    internal async Task<Either<BaseError, TvShowNfo>> Read(Stream input)
+    {
+        TvShowNfo nfo = null;
+
        try
        {
            var settings = new XmlReaderSettings { Async = true, ConformanceLevel = ConformanceLevel.Fragment };
            using var reader = XmlReader.Create(input, settings);
-            TvShowNfo nfo = null;
            var done = false;

            while (!done && await reader.ReadAsync())
@@ -91,6 +112,13 @@ public class TvShowNfoReader : NfoReader<TvShowNfo>, ITvShowNfoReader

            return Optional(nfo).ToEither((BaseError)new FailedToReadNfo());
        }
+        catch (XmlException ex) when (ex.Message.Contains(
+                                          "invalid character",
+                                          StringComparison.InvariantCultureIgnoreCase))
+        {
+            _logger.LogWarning("Invalid XML detected; returning incomplete metadata");
+            return Optional(nfo).ToEither((BaseError)new FailedToReadNfo());
+        }
        catch (Exception ex)
        {
            _client.Notify(ex);