fix(491): unique index on LibraryFolder(LibraryPathId, PathHash) + tolerate concurrent insert
GetOrAddFolder was a check-then-insert with no unique constraint behind it, so two callers racing the same folder could both miss the lookup and both insert. Enforce identity in the schema and make the loser adopt the winner. - LibraryFolder gains a SHA-256 PathHash (the MediaFile.Path/PathHash precedent): Path is MySQL longtext, which cannot be indexed without a prefix length and collates case-insensitively, so the unique index is on (LibraryPathId, PathHash) instead. - GetOrAddFolder and SetEtag catch a classified unique violation via the existing TvContext.IsUniqueConstraintViolation seam (#308) and re-read. - Dual-provider migration audits and collapses pre-existing duplicates (repointing MediaFile, ParentId and ImageFolderDuration) before creating the index; legacy rows keep a null hash and heal on the next scan. - Tests: deterministic cross-connection race, 8x10 barrier stress with an insert-attempt vacuity guard, classifier-inversion negative control, and a real-migration dedupe test. Refs #488 #308 fix #491
This commit is contained in:
@@ -1,9 +1,19 @@
|
|||||||
namespace ErsatzTV.Core.Domain;
|
namespace ErsatzTV.Core.Domain;
|
||||||
|
|
||||||
public class LibraryFolder
|
public class LibraryFolder
|
||||||
{
|
{
|
||||||
public int Id { get; set; }
|
public int Id { get; set; }
|
||||||
public string Path { get; set; }
|
public string Path { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// SHA-256 hex of <see cref="Path" /> (<see cref="ErsatzTV.Core.PathUtils.GetPathHash" />), the
|
||||||
|
/// indexable stand-in for the unbounded <see cref="Path" /> column that backs the unique
|
||||||
|
/// <c>(LibraryPathId, PathHash)</c> constraint — the same shape as <c>MediaFile.PathHash</c>.
|
||||||
|
/// Nullable: rows created before ersatztv#491 carry <c>null</c> until a scan heals them, and a
|
||||||
|
/// unique index treats nulls as distinct so those legacy rows never collide.
|
||||||
|
/// </summary>
|
||||||
|
public string PathHash { get; set; }
|
||||||
|
|
||||||
public int LibraryPathId { get; set; }
|
public int LibraryPathId { get; set; }
|
||||||
public LibraryPath LibraryPath { get; set; }
|
public LibraryPath LibraryPath { get; set; }
|
||||||
public int? ParentId { get; set; }
|
public int? ParentId { get; set; }
|
||||||
|
|||||||
+7314
File diff suppressed because it is too large
Load Diff
+153
@@ -0,0 +1,153 @@
|
|||||||
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
|
#nullable disable
|
||||||
|
|
||||||
|
namespace ErsatzTV.Infrastructure.MySql.Migrations
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
public partial class Add_LibraryFolder_PathHash_UniqueIndex : Migration
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Up(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
// ersatztv#491 — audit/clean pre-existing duplicate LibraryFolder rows before the unique index.
|
||||||
|
// Mirrors the Sqlite migration; see it for the full rationale. The one provider difference is
|
||||||
|
// the explicit utf8mb4_bin collation on every Path comparison: MySql's default collation is
|
||||||
|
// case-INsensitive, and grouping under it would treat sibling folders differing only in case
|
||||||
|
// (legal on a case-sensitive filesystem) as duplicates and delete one. The unique index is on
|
||||||
|
// the SHA-256 PathHash precisely so identity stays byte-exact on both providers.
|
||||||
|
// DROP TABLE IF EXISTS makes a retry after a partial failure safe (DDL implicitly commits on
|
||||||
|
// MySql, so the migration is not atomic).
|
||||||
|
migrationBuilder.Sql("DROP TABLE IF EXISTS `__LibraryFolderDedupe`");
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"""
|
||||||
|
CREATE TABLE `__LibraryFolderDedupe` (
|
||||||
|
LoserId INT NOT NULL PRIMARY KEY,
|
||||||
|
KeeperId INT NOT NULL
|
||||||
|
)
|
||||||
|
""");
|
||||||
|
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"""
|
||||||
|
INSERT INTO `__LibraryFolderDedupe` (LoserId, KeeperId)
|
||||||
|
SELECT l.Id, k.KeeperId
|
||||||
|
FROM LibraryFolder l
|
||||||
|
INNER JOIN (
|
||||||
|
SELECT LibraryPathId, Path COLLATE utf8mb4_bin AS BinPath, MIN(Id) AS KeeperId
|
||||||
|
FROM LibraryFolder
|
||||||
|
GROUP BY LibraryPathId, Path COLLATE utf8mb4_bin
|
||||||
|
) k ON k.LibraryPathId = l.LibraryPathId AND k.BinPath = l.Path COLLATE utf8mb4_bin
|
||||||
|
WHERE l.Id <> k.KeeperId
|
||||||
|
""");
|
||||||
|
|
||||||
|
// media files recorded against a duplicate folder follow the keeper (MediaFile.LibraryFolderId
|
||||||
|
// is Restrict, so the delete below would fail otherwise)
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"""
|
||||||
|
UPDATE MediaFile
|
||||||
|
SET LibraryFolderId = (
|
||||||
|
SELECT KeeperId FROM `__LibraryFolderDedupe` WHERE LoserId = MediaFile.LibraryFolderId)
|
||||||
|
WHERE LibraryFolderId IN (SELECT LoserId FROM `__LibraryFolderDedupe`)
|
||||||
|
""");
|
||||||
|
|
||||||
|
// child folders parented on a duplicate follow the keeper (ParentId is Restrict as well)
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"""
|
||||||
|
UPDATE LibraryFolder
|
||||||
|
SET ParentId = (
|
||||||
|
SELECT KeeperId FROM `__LibraryFolderDedupe` WHERE LoserId = LibraryFolder.ParentId)
|
||||||
|
WHERE ParentId IN (SELECT LoserId FROM `__LibraryFolderDedupe`)
|
||||||
|
""");
|
||||||
|
|
||||||
|
// ImageFolderDuration is 1:1 with LibraryFolder (unique index on LibraryFolderId), so the
|
||||||
|
// duplicates' rows cannot all be repointed. Keep the keeper's own setting when it has one;
|
||||||
|
// otherwise promote exactly one loser's (lowest Id) and drop the rest.
|
||||||
|
migrationBuilder.Sql("DROP TABLE IF EXISTS `__LibraryFolderDedupeIfd`");
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"""
|
||||||
|
CREATE TABLE `__LibraryFolderDedupeIfd` (
|
||||||
|
KeeperId INT NOT NULL PRIMARY KEY,
|
||||||
|
IfdId INT NOT NULL
|
||||||
|
)
|
||||||
|
""");
|
||||||
|
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"""
|
||||||
|
INSERT INTO `__LibraryFolderDedupeIfd` (KeeperId, IfdId)
|
||||||
|
SELECT d.KeeperId, MIN(i.Id)
|
||||||
|
FROM `__LibraryFolderDedupe` d
|
||||||
|
INNER JOIN ImageFolderDuration i ON i.LibraryFolderId = d.LoserId
|
||||||
|
WHERE NOT EXISTS (
|
||||||
|
SELECT 1 FROM ImageFolderDuration ki WHERE ki.LibraryFolderId = d.KeeperId)
|
||||||
|
GROUP BY d.KeeperId
|
||||||
|
""");
|
||||||
|
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"""
|
||||||
|
DELETE FROM ImageFolderDuration
|
||||||
|
WHERE LibraryFolderId IN (SELECT LoserId FROM `__LibraryFolderDedupe`)
|
||||||
|
AND Id NOT IN (SELECT IfdId FROM `__LibraryFolderDedupeIfd`)
|
||||||
|
""");
|
||||||
|
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"""
|
||||||
|
UPDATE ImageFolderDuration
|
||||||
|
SET LibraryFolderId = (
|
||||||
|
SELECT KeeperId FROM `__LibraryFolderDedupeIfd` WHERE IfdId = ImageFolderDuration.Id)
|
||||||
|
WHERE Id IN (SELECT IfdId FROM `__LibraryFolderDedupeIfd`)
|
||||||
|
""");
|
||||||
|
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"DELETE FROM LibraryFolder WHERE Id IN (SELECT LoserId FROM `__LibraryFolderDedupe`)");
|
||||||
|
|
||||||
|
migrationBuilder.Sql("DROP TABLE `__LibraryFolderDedupeIfd`");
|
||||||
|
migrationBuilder.Sql("DROP TABLE `__LibraryFolderDedupe`");
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<string>(
|
||||||
|
name: "PathHash",
|
||||||
|
table: "LibraryFolder",
|
||||||
|
type: "varchar(64)",
|
||||||
|
maxLength: 64,
|
||||||
|
nullable: true)
|
||||||
|
.Annotation("MySql:CharSet", "utf8mb4");
|
||||||
|
|
||||||
|
// Existing rows keep a null hash on purpose: a unique index treats nulls as distinct, so the
|
||||||
|
// index applies cleanly to any database, and LibraryRepository.GetOrAddFolder heals each row
|
||||||
|
// (SHA-256 of Path) the first time a scan touches it. Those rows are deduplicated above and are
|
||||||
|
// still found by the Path lookup, so no insert can race them in the meantime.
|
||||||
|
//
|
||||||
|
// Order matters on MySql, and EF scaffolds it the other way round: InnoDB refuses to drop the
|
||||||
|
// FK's only backing index ("Cannot drop index 'IX_LibraryFolder_LibraryPathId': needed in a
|
||||||
|
// foreign key constraint"). Create the composite first — LibraryPathId is its leftmost column,
|
||||||
|
// so it takes over as the FK's backing index — then drop the now-redundant single-column one.
|
||||||
|
migrationBuilder.CreateIndex(
|
||||||
|
name: "IX_LibraryFolder_LibraryPathId_PathHash",
|
||||||
|
table: "LibraryFolder",
|
||||||
|
columns: new[] { "LibraryPathId", "PathHash" },
|
||||||
|
unique: true);
|
||||||
|
|
||||||
|
migrationBuilder.DropIndex(
|
||||||
|
name: "IX_LibraryFolder_LibraryPathId",
|
||||||
|
table: "LibraryFolder");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Down(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
// mirror of Up: restore the single-column index before dropping the composite one, so the
|
||||||
|
// foreign key is never left without a backing index
|
||||||
|
migrationBuilder.CreateIndex(
|
||||||
|
name: "IX_LibraryFolder_LibraryPathId",
|
||||||
|
table: "LibraryFolder",
|
||||||
|
column: "LibraryPathId");
|
||||||
|
|
||||||
|
migrationBuilder.DropIndex(
|
||||||
|
name: "IX_LibraryFolder_LibraryPathId_PathHash",
|
||||||
|
table: "LibraryFolder");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "PathHash",
|
||||||
|
table: "LibraryFolder");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1353,12 +1353,17 @@ namespace ErsatzTV.Infrastructure.MySql.Migrations
|
|||||||
b.Property<string>("Path")
|
b.Property<string>("Path")
|
||||||
.HasColumnType("longtext");
|
.HasColumnType("longtext");
|
||||||
|
|
||||||
|
b.Property<string>("PathHash")
|
||||||
|
.HasMaxLength(64)
|
||||||
|
.HasColumnType("varchar(64)");
|
||||||
|
|
||||||
b.HasKey("Id");
|
b.HasKey("Id");
|
||||||
|
|
||||||
b.HasIndex("LibraryPathId");
|
|
||||||
|
|
||||||
b.HasIndex("ParentId");
|
b.HasIndex("ParentId");
|
||||||
|
|
||||||
|
b.HasIndex("LibraryPathId", "PathHash")
|
||||||
|
.IsUnique();
|
||||||
|
|
||||||
b.ToTable("LibraryFolder", (string)null);
|
b.ToTable("LibraryFolder", (string)null);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
+7139
File diff suppressed because it is too large
Load Diff
+146
@@ -0,0 +1,146 @@
|
|||||||
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
|
#nullable disable
|
||||||
|
|
||||||
|
namespace ErsatzTV.Infrastructure.Sqlite.Migrations
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
public partial class Add_LibraryFolder_PathHash_UniqueIndex : Migration
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Up(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
// ersatztv#491 — audit/clean pre-existing duplicate LibraryFolder rows before the unique index.
|
||||||
|
// Duplicates were reachable before #488 (the folder lookup read a scan-start in-memory snapshot,
|
||||||
|
// so a folder created earlier in the SAME scan was invisible and inserted again) and via the
|
||||||
|
// check-then-insert race the index now closes. Keep the lowest Id per (LibraryPathId, Path) and
|
||||||
|
// repoint every dependent row at it before deleting the losers. The helper tables keep the
|
||||||
|
// statements readable; DROP TABLE IF EXISTS makes a retry after a partial failure safe.
|
||||||
|
migrationBuilder.Sql("DROP TABLE IF EXISTS __LibraryFolderDedupe");
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"""
|
||||||
|
CREATE TABLE __LibraryFolderDedupe (
|
||||||
|
LoserId INTEGER NOT NULL PRIMARY KEY,
|
||||||
|
KeeperId INTEGER NOT NULL
|
||||||
|
)
|
||||||
|
""");
|
||||||
|
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"""
|
||||||
|
INSERT INTO __LibraryFolderDedupe (LoserId, KeeperId)
|
||||||
|
SELECT l.Id, k.KeeperId
|
||||||
|
FROM LibraryFolder l
|
||||||
|
INNER JOIN (
|
||||||
|
SELECT LibraryPathId, Path, MIN(Id) AS KeeperId
|
||||||
|
FROM LibraryFolder
|
||||||
|
GROUP BY LibraryPathId, Path
|
||||||
|
) k ON k.LibraryPathId = l.LibraryPathId AND k.Path = l.Path
|
||||||
|
WHERE l.Id <> k.KeeperId
|
||||||
|
""");
|
||||||
|
|
||||||
|
// media files recorded against a duplicate folder follow the keeper (MediaFile.LibraryFolderId
|
||||||
|
// is Restrict, so the delete below would fail otherwise)
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"""
|
||||||
|
UPDATE MediaFile
|
||||||
|
SET LibraryFolderId = (
|
||||||
|
SELECT KeeperId FROM __LibraryFolderDedupe WHERE LoserId = MediaFile.LibraryFolderId)
|
||||||
|
WHERE LibraryFolderId IN (SELECT LoserId FROM __LibraryFolderDedupe)
|
||||||
|
""");
|
||||||
|
|
||||||
|
// child folders parented on a duplicate follow the keeper (ParentId is Restrict as well)
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"""
|
||||||
|
UPDATE LibraryFolder
|
||||||
|
SET ParentId = (
|
||||||
|
SELECT KeeperId FROM __LibraryFolderDedupe WHERE LoserId = LibraryFolder.ParentId)
|
||||||
|
WHERE ParentId IN (SELECT LoserId FROM __LibraryFolderDedupe)
|
||||||
|
""");
|
||||||
|
|
||||||
|
// ImageFolderDuration is 1:1 with LibraryFolder (unique index on LibraryFolderId), so the
|
||||||
|
// duplicates' rows cannot all be repointed. Keep the keeper's own setting when it has one;
|
||||||
|
// otherwise promote exactly one loser's (lowest Id) and drop the rest.
|
||||||
|
migrationBuilder.Sql("DROP TABLE IF EXISTS __LibraryFolderDedupeIfd");
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"""
|
||||||
|
CREATE TABLE __LibraryFolderDedupeIfd (
|
||||||
|
KeeperId INTEGER NOT NULL PRIMARY KEY,
|
||||||
|
IfdId INTEGER NOT NULL
|
||||||
|
)
|
||||||
|
""");
|
||||||
|
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"""
|
||||||
|
INSERT INTO __LibraryFolderDedupeIfd (KeeperId, IfdId)
|
||||||
|
SELECT d.KeeperId, MIN(i.Id)
|
||||||
|
FROM __LibraryFolderDedupe d
|
||||||
|
INNER JOIN ImageFolderDuration i ON i.LibraryFolderId = d.LoserId
|
||||||
|
WHERE NOT EXISTS (
|
||||||
|
SELECT 1 FROM ImageFolderDuration ki WHERE ki.LibraryFolderId = d.KeeperId)
|
||||||
|
GROUP BY d.KeeperId
|
||||||
|
""");
|
||||||
|
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"""
|
||||||
|
DELETE FROM ImageFolderDuration
|
||||||
|
WHERE LibraryFolderId IN (SELECT LoserId FROM __LibraryFolderDedupe)
|
||||||
|
AND Id NOT IN (SELECT IfdId FROM __LibraryFolderDedupeIfd)
|
||||||
|
""");
|
||||||
|
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"""
|
||||||
|
UPDATE ImageFolderDuration
|
||||||
|
SET LibraryFolderId = (
|
||||||
|
SELECT KeeperId FROM __LibraryFolderDedupeIfd WHERE IfdId = ImageFolderDuration.Id)
|
||||||
|
WHERE Id IN (SELECT IfdId FROM __LibraryFolderDedupeIfd)
|
||||||
|
""");
|
||||||
|
|
||||||
|
migrationBuilder.Sql(
|
||||||
|
"DELETE FROM LibraryFolder WHERE Id IN (SELECT LoserId FROM __LibraryFolderDedupe)");
|
||||||
|
|
||||||
|
migrationBuilder.Sql("DROP TABLE __LibraryFolderDedupeIfd");
|
||||||
|
migrationBuilder.Sql("DROP TABLE __LibraryFolderDedupe");
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<string>(
|
||||||
|
name: "PathHash",
|
||||||
|
table: "LibraryFolder",
|
||||||
|
type: "TEXT",
|
||||||
|
maxLength: 64,
|
||||||
|
nullable: true);
|
||||||
|
|
||||||
|
// Existing rows keep a null hash on purpose: a unique index treats nulls as distinct, so the
|
||||||
|
// index applies cleanly to any database, and LibraryRepository.GetOrAddFolder heals each row
|
||||||
|
// (SHA-256 of Path) the first time a scan touches it. Those rows are deduplicated above and are
|
||||||
|
// still found by the Path lookup, so no insert can race them in the meantime.
|
||||||
|
//
|
||||||
|
// Create-then-drop rather than EF's scaffolded drop-then-create, matching the MySql copy, where
|
||||||
|
// InnoDB refuses to drop the foreign key's only backing index.
|
||||||
|
migrationBuilder.CreateIndex(
|
||||||
|
name: "IX_LibraryFolder_LibraryPathId_PathHash",
|
||||||
|
table: "LibraryFolder",
|
||||||
|
columns: new[] { "LibraryPathId", "PathHash" },
|
||||||
|
unique: true);
|
||||||
|
|
||||||
|
migrationBuilder.DropIndex(
|
||||||
|
name: "IX_LibraryFolder_LibraryPathId",
|
||||||
|
table: "LibraryFolder");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Down(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
migrationBuilder.CreateIndex(
|
||||||
|
name: "IX_LibraryFolder_LibraryPathId",
|
||||||
|
table: "LibraryFolder",
|
||||||
|
column: "LibraryPathId");
|
||||||
|
|
||||||
|
migrationBuilder.DropIndex(
|
||||||
|
name: "IX_LibraryFolder_LibraryPathId_PathHash",
|
||||||
|
table: "LibraryFolder");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "PathHash",
|
||||||
|
table: "LibraryFolder");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1298,12 +1298,17 @@ namespace ErsatzTV.Infrastructure.Sqlite.Migrations
|
|||||||
b.Property<string>("Path")
|
b.Property<string>("Path")
|
||||||
.HasColumnType("TEXT");
|
.HasColumnType("TEXT");
|
||||||
|
|
||||||
|
b.Property<string>("PathHash")
|
||||||
|
.HasMaxLength(64)
|
||||||
|
.HasColumnType("TEXT");
|
||||||
|
|
||||||
b.HasKey("Id");
|
b.HasKey("Id");
|
||||||
|
|
||||||
b.HasIndex("LibraryPathId");
|
|
||||||
|
|
||||||
b.HasIndex("ParentId");
|
b.HasIndex("ParentId");
|
||||||
|
|
||||||
|
b.HasIndex("LibraryPathId", "PathHash")
|
||||||
|
.IsUnique();
|
||||||
|
|
||||||
b.ToTable("LibraryFolder", (string)null);
|
b.ToTable("LibraryFolder", (string)null);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
using ErsatzTV.Core.Domain;
|
using ErsatzTV.Core.Domain;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
||||||
|
|
||||||
@@ -10,6 +10,19 @@ public class LibraryFolderConfiguration : IEntityTypeConfiguration<LibraryFolder
|
|||||||
{
|
{
|
||||||
builder.ToTable("LibraryFolder");
|
builder.ToTable("LibraryFolder");
|
||||||
|
|
||||||
|
// ersatztv#491: GetOrAddFolder is a check-then-insert, so two callers racing the same folder both
|
||||||
|
// miss the lookup and both insert. The unique index makes the duplicate impossible at the storage
|
||||||
|
// layer (the loser gets a constraint violation the repository catches and re-reads).
|
||||||
|
// Indexed on PathHash rather than Path because Path is unbounded (MySQL longtext, which cannot be
|
||||||
|
// indexed without a prefix length, and whose default collation is case-INsensitive — a prefix
|
||||||
|
// index would also false-collide sibling folders differing only in case on a case-sensitive
|
||||||
|
// filesystem). This mirrors the existing MediaFile.Path/PathHash pair.
|
||||||
|
builder.Property(f => f.PathHash)
|
||||||
|
.HasMaxLength(64);
|
||||||
|
|
||||||
|
builder.HasIndex(f => new { f.LibraryPathId, f.PathHash })
|
||||||
|
.IsUnique();
|
||||||
|
|
||||||
builder.HasOne(f => f.Parent)
|
builder.HasOne(f => f.Parent)
|
||||||
.WithMany(p => p.Children)
|
.WithMany(p => p.Children)
|
||||||
.HasForeignKey(f => f.ParentId)
|
.HasForeignKey(f => f.ParentId)
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
using System.IO.Abstractions;
|
using System.IO.Abstractions;
|
||||||
using Dapper;
|
using Dapper;
|
||||||
|
using ErsatzTV.Core;
|
||||||
using ErsatzTV.Core.Domain;
|
using ErsatzTV.Core.Domain;
|
||||||
using ErsatzTV.Core.Interfaces.Repositories;
|
using ErsatzTV.Core.Interfaces.Repositories;
|
||||||
using ErsatzTV.Infrastructure.Extensions;
|
using ErsatzTV.Infrastructure.Extensions;
|
||||||
@@ -110,16 +111,36 @@ public class LibraryRepository(IFileSystem fileSystem, IDbContextFactory<TvConte
|
|||||||
|
|
||||||
if (knownFolder.IsNone)
|
if (knownFolder.IsNone)
|
||||||
{
|
{
|
||||||
await dbContext.LibraryFolders.AddAsync(
|
var newFolder = new LibraryFolder
|
||||||
new LibraryFolder
|
|
||||||
{
|
{
|
||||||
Path = path,
|
Path = path,
|
||||||
|
PathHash = PathUtils.GetPathHash(path),
|
||||||
Etag = etag,
|
Etag = etag,
|
||||||
LibraryPathId = libraryPath.Id
|
LibraryPathId = libraryPath.Id
|
||||||
});
|
};
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await dbContext.LibraryFolders.AddAsync(newFolder);
|
||||||
await dbContext.SaveChangesAsync();
|
await dbContext.SaveChangesAsync();
|
||||||
}
|
}
|
||||||
|
catch (DbUpdateException ex) when (TvContext.IsUniqueConstraintViolation(ex))
|
||||||
|
{
|
||||||
|
// ersatztv#491: a concurrent caller created this folder between the caller's lookup and
|
||||||
|
// this insert. The etag write is the whole point of the call, so apply it to the winner's
|
||||||
|
// row rather than failing the scan.
|
||||||
|
dbContext.Entry(newFolder).State = EntityState.Detached;
|
||||||
|
LibraryFolder winner = await GetFolder(dbContext, libraryPath.Id, path);
|
||||||
|
if (winner is null)
|
||||||
|
{
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
|
||||||
|
await dbContext.Connection.ExecuteAsync(
|
||||||
|
"UPDATE LibraryFolder SET Etag = @Etag WHERE Id = @Id",
|
||||||
|
new { winner.Id, Etag = etag });
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task CleanEtagsForLibraryPath(LibraryPath libraryPath)
|
public async Task CleanEtagsForLibraryPath(LibraryPath libraryPath)
|
||||||
@@ -174,11 +195,44 @@ public class LibraryRepository(IFileSystem fileSystem, IDbContextFactory<TvConte
|
|||||||
// local scan path (via GetLibrary) and is null on the remote (Jellyfin) sync path, which used
|
// local scan path (via GetLibrary) and is null on the remote (Jellyfin) sync path, which used
|
||||||
// to NRE every Jellyfin music-video scan here (ersatztv#488). The local scanners already hit
|
// to NRE every Jellyfin music-video scan here (ersatztv#488). The local scanners already hit
|
||||||
// the db once per folder via GetParentFolderId, so this adds no new query pattern.
|
// the db once per folder via GetParentFolderId, so this adds no new query pattern.
|
||||||
LibraryFolder knownFolder = await dbContext.LibraryFolders
|
LibraryFolder knownFolder = await GetFolder(dbContext, libraryPath.Id, folder);
|
||||||
.AsNoTracking()
|
|
||||||
.Filter(f => f.LibraryPathId == libraryPath.Id && f.Path == folder)
|
// add new folder to library path
|
||||||
.FirstOrDefaultAsync()
|
if (knownFolder is null)
|
||||||
?? CreateNewFolder(libraryPath, maybeParentFolder, folder);
|
{
|
||||||
|
LibraryFolder newFolder = CreateNewFolder(libraryPath, maybeParentFolder, folder);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await dbContext.LibraryFolders.AddAsync(newFolder);
|
||||||
|
await dbContext.SaveChangesAsync();
|
||||||
|
knownFolder = newFolder;
|
||||||
|
}
|
||||||
|
catch (DbUpdateException ex) when (TvContext.IsUniqueConstraintViolation(ex))
|
||||||
|
{
|
||||||
|
// ersatztv#491: the lookup above is not atomic with this insert, so a concurrent caller
|
||||||
|
// scanning the same folder can slip its row in between. The unique index on
|
||||||
|
// (LibraryPathId, PathHash) turns that lost race into a constraint violation instead of a
|
||||||
|
// duplicate row; adopt the winner's row rather than failing the scan. Detach first so the
|
||||||
|
// failed insert is not retried by anything reusing this context.
|
||||||
|
dbContext.Entry(newFolder).State = EntityState.Detached;
|
||||||
|
knownFolder = await GetFolder(dbContext, libraryPath.Id, folder);
|
||||||
|
if (knownFolder is null)
|
||||||
|
{
|
||||||
|
// no winner to adopt — the violation came from somewhere else, so don't swallow it
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (string.IsNullOrEmpty(knownFolder.PathHash))
|
||||||
|
{
|
||||||
|
// heal a row created before the PathHash column existed, so it participates in the unique
|
||||||
|
// index from here on (a null hash is distinct from every other value, so it does not).
|
||||||
|
knownFolder.PathHash = PathUtils.GetPathHash(folder);
|
||||||
|
|
||||||
|
await dbContext.Connection.ExecuteAsync(
|
||||||
|
"UPDATE LibraryFolder SET PathHash = @PathHash WHERE Id = @Id",
|
||||||
|
new { knownFolder.PathHash, knownFolder.Id });
|
||||||
|
}
|
||||||
|
|
||||||
// update parent folder if not present
|
// update parent folder if not present
|
||||||
foreach (int parentFolder in maybeParentFolder)
|
foreach (int parentFolder in maybeParentFolder)
|
||||||
@@ -193,13 +247,6 @@ public class LibraryRepository(IFileSystem fileSystem, IDbContextFactory<TvConte
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// add new folder to library path
|
|
||||||
if (knownFolder.Id < 1)
|
|
||||||
{
|
|
||||||
await dbContext.LibraryFolders.AddAsync(knownFolder);
|
|
||||||
await dbContext.SaveChangesAsync();
|
|
||||||
}
|
|
||||||
|
|
||||||
return knownFolder;
|
return knownFolder;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -221,6 +268,12 @@ public class LibraryRepository(IFileSystem fileSystem, IDbContextFactory<TvConte
|
|||||||
new { Path = normalizedLibraryPath, libraryPath.Id });
|
new { Path = normalizedLibraryPath, libraryPath.Id });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static Task<LibraryFolder> GetFolder(TvContext dbContext, int libraryPathId, string folder) =>
|
||||||
|
dbContext.LibraryFolders
|
||||||
|
.AsNoTracking()
|
||||||
|
.Filter(f => f.LibraryPathId == libraryPathId && f.Path == folder)
|
||||||
|
.FirstOrDefaultAsync();
|
||||||
|
|
||||||
private static LibraryFolder CreateNewFolder(LibraryPath libraryPath, Option<int> maybeParentFolder, string folder)
|
private static LibraryFolder CreateNewFolder(LibraryPath libraryPath, Option<int> maybeParentFolder, string folder)
|
||||||
{
|
{
|
||||||
int? parentId = null;
|
int? parentId = null;
|
||||||
@@ -232,6 +285,7 @@ public class LibraryRepository(IFileSystem fileSystem, IDbContextFactory<TvConte
|
|||||||
return new LibraryFolder
|
return new LibraryFolder
|
||||||
{
|
{
|
||||||
Path = folder,
|
Path = folder,
|
||||||
|
PathHash = PathUtils.GetPathHash(folder),
|
||||||
Etag = null,
|
Etag = null,
|
||||||
LibraryPathId = libraryPath.Id,
|
LibraryPathId = libraryPath.Id,
|
||||||
ParentId = parentId
|
ParentId = parentId
|
||||||
|
|||||||
@@ -0,0 +1,302 @@
|
|||||||
|
using ErsatzTV.Core;
|
||||||
|
using ErsatzTV.Core.Domain;
|
||||||
|
using ErsatzTV.Infrastructure.Data;
|
||||||
|
using ErsatzTV.Infrastructure.Data.Repositories;
|
||||||
|
using ErsatzTV.Infrastructure.Sqlite.Data;
|
||||||
|
using ErsatzTV.Tests.Support;
|
||||||
|
using LanguageExt;
|
||||||
|
using Microsoft.Data.Sqlite;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.EntityFrameworkCore.Diagnostics;
|
||||||
|
using NSubstitute;
|
||||||
|
using NUnit.Framework;
|
||||||
|
using Shouldly;
|
||||||
|
using IFileSystem = System.IO.Abstractions.IFileSystem;
|
||||||
|
|
||||||
|
namespace ErsatzTV.Tests.Integration;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// ersatztv#491: <c>ILibraryRepository.GetOrAddFolder</c> is a check-then-insert, so two callers
|
||||||
|
/// racing the same <c>(LibraryPathId, Path)</c> both miss the lookup and both insert. The fix is a
|
||||||
|
/// unique index on <c>(LibraryPathId, PathHash)</c> plus a catch-and-re-read in the repository, so
|
||||||
|
/// the loser adopts the winner's row instead of creating a duplicate.
|
||||||
|
/// </summary>
|
||||||
|
[TestFixture]
|
||||||
|
public class LibraryFolderConcurrencyTests
|
||||||
|
{
|
||||||
|
private const string LibraryPathValue = "/data/music";
|
||||||
|
private const string FolderPath = "/data/music/artist1";
|
||||||
|
|
||||||
|
private static LibraryRepository Repository(IDbContextFactory<TvContext> factory) =>
|
||||||
|
new(Substitute.For<IFileSystem>(), factory);
|
||||||
|
|
||||||
|
private static async Task<int> SeedLibraryPath(Func<TvContext> createContext)
|
||||||
|
{
|
||||||
|
await using TvContext context = createContext();
|
||||||
|
var libraryPath = new LibraryPath { Path = LibraryPathValue };
|
||||||
|
await context.LibraryPaths.AddAsync(libraryPath);
|
||||||
|
await context.SaveChangesAsync();
|
||||||
|
return libraryPath.Id;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<int> FolderCount(Func<TvContext> createContext, int libraryPathId, string path)
|
||||||
|
{
|
||||||
|
await using TvContext context = createContext();
|
||||||
|
return await context.LibraryFolders.CountAsync(f => f.LibraryPathId == libraryPathId && f.Path == path);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task InsertFolderRaw(
|
||||||
|
SharedCacheTvContext db,
|
||||||
|
int libraryPathId,
|
||||||
|
string path,
|
||||||
|
string pathHash,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
await using SqliteConnection connection = db.OpenConnection();
|
||||||
|
await using SqliteCommand command = connection.CreateCommand();
|
||||||
|
command.CommandText =
|
||||||
|
"INSERT INTO \"LibraryFolder\" (\"LibraryPathId\", \"Path\", \"PathHash\", \"Etag\", \"ParentId\") " +
|
||||||
|
"VALUES ($libraryPathId, $path, $pathHash, NULL, NULL)";
|
||||||
|
command.Parameters.AddWithValue("$libraryPathId", libraryPathId);
|
||||||
|
command.Parameters.AddWithValue("$path", path);
|
||||||
|
command.Parameters.AddWithValue("$pathHash", (object?)pathHash ?? DBNull.Value);
|
||||||
|
await command.ExecuteNonQueryAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Simulates the concurrent "winner": exactly once, on a SEPARATE connection, insert the same
|
||||||
|
/// folder and commit — AFTER the intercepted context read the (stale) absent lookup but BEFORE its
|
||||||
|
/// own INSERT runs. This interposes the race deterministically instead of hoping for a timing
|
||||||
|
/// window. <see cref="Fired" /> proves the race actually happened (non-vacuity).
|
||||||
|
/// </summary>
|
||||||
|
private sealed class InsertConflictingFolderOnce(SharedCacheTvContext db, int libraryPathId, string path)
|
||||||
|
: SaveChangesInterceptor
|
||||||
|
{
|
||||||
|
private int _fired;
|
||||||
|
|
||||||
|
public int Fired => _fired;
|
||||||
|
|
||||||
|
public override async ValueTask<InterceptionResult<int>> SavingChangesAsync(
|
||||||
|
DbContextEventData eventData,
|
||||||
|
InterceptionResult<int> result,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
if (Interlocked.Exchange(ref _fired, 1) == 0)
|
||||||
|
{
|
||||||
|
await InsertFolderRaw(db, libraryPathId, path, PathUtils.GetPathHash(path), cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Counts insert attempts so the multi-threaded test can prove it really raced.</summary>
|
||||||
|
private sealed class CountSaveAttempts : SaveChangesInterceptor
|
||||||
|
{
|
||||||
|
private int _attempts;
|
||||||
|
|
||||||
|
public int Attempts => _attempts;
|
||||||
|
|
||||||
|
public override ValueTask<InterceptionResult<int>> SavingChangesAsync(
|
||||||
|
DbContextEventData eventData,
|
||||||
|
InterceptionResult<int> result,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
Interlocked.Increment(ref _attempts);
|
||||||
|
return ValueTask.FromResult(result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----- Negative control #1: the index itself. Without the new unique index this test fails, because
|
||||||
|
// the second insert simply succeeds and there is no violation to classify. -----
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public async Task Duplicate_LibraryFolder_Insert_Throws_A_Classified_UniqueViolation()
|
||||||
|
{
|
||||||
|
await using SharedCacheTvContext db = await SharedCacheTvContext.CreateAsync("etv491-index");
|
||||||
|
int libraryPathId = await SeedLibraryPath(db.CreateContext);
|
||||||
|
string hash = PathUtils.GetPathHash(FolderPath);
|
||||||
|
|
||||||
|
await InsertFolderRaw(db, libraryPathId, FolderPath, hash);
|
||||||
|
|
||||||
|
await using TvContext context = db.CreateContext();
|
||||||
|
await context.LibraryFolders.AddAsync(
|
||||||
|
new LibraryFolder { LibraryPathId = libraryPathId, Path = FolderPath, PathHash = hash });
|
||||||
|
|
||||||
|
DbUpdateException ex = await Should.ThrowAsync<DbUpdateException>(() => context.SaveChangesAsync());
|
||||||
|
SqliteErrorClassifier.IsUniqueConstraintViolation(ex).ShouldBeTrue();
|
||||||
|
|
||||||
|
// and the classifier is not a blanket "true"
|
||||||
|
SqliteErrorClassifier.IsUniqueConstraintViolation(
|
||||||
|
new DbUpdateException("nope", new InvalidOperationException())).ShouldBeFalse();
|
||||||
|
}
|
||||||
|
|
||||||
|
// The migration leaves pre-#491 rows with a null hash; a unique index treats nulls as distinct, so
|
||||||
|
// applying the index to an existing database can never fail on them. Documents that premise.
|
||||||
|
[Test]
|
||||||
|
public async Task Legacy_Null_PathHash_Rows_Do_Not_Collide()
|
||||||
|
{
|
||||||
|
await using SharedCacheTvContext db = await SharedCacheTvContext.CreateAsync("etv491-nulls");
|
||||||
|
int libraryPathId = await SeedLibraryPath(db.CreateContext);
|
||||||
|
|
||||||
|
await InsertFolderRaw(db, libraryPathId, "/data/music/a", null);
|
||||||
|
await InsertFolderRaw(db, libraryPathId, "/data/music/b", null);
|
||||||
|
|
||||||
|
await using TvContext context = db.CreateContext();
|
||||||
|
(await context.LibraryFolders.CountAsync(f => f.LibraryPathId == libraryPathId)).ShouldBe(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----- The deterministic cross-connection race through the real repository -----
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public async Task GetOrAddFolder_Losing_The_Race_Adopts_The_Winner_Instead_Of_Duplicating()
|
||||||
|
{
|
||||||
|
await using SharedCacheTvContext db = await SharedCacheTvContext.CreateAsync("etv491-race");
|
||||||
|
int libraryPathId = await SeedLibraryPath(db.CreateContext);
|
||||||
|
var libraryPath = new LibraryPath { Id = libraryPathId, Path = LibraryPathValue, LibraryFolders = null };
|
||||||
|
|
||||||
|
var racer = new InsertConflictingFolderOnce(db, libraryPathId, FolderPath);
|
||||||
|
LibraryRepository repository = Repository(db.Factory(racer));
|
||||||
|
|
||||||
|
LibraryFolder result = await repository.GetOrAddFolder(libraryPath, Option<int>.None, FolderPath);
|
||||||
|
|
||||||
|
racer.Fired.ShouldBe(1); // the race genuinely occurred — this assertion is the vacuity guard
|
||||||
|
result.ShouldNotBeNull();
|
||||||
|
result.Id.ShouldBeGreaterThan(0);
|
||||||
|
result.Path.ShouldBe(FolderPath);
|
||||||
|
(await FolderCount(db.CreateContext, libraryPathId, FolderPath)).ShouldBe(1);
|
||||||
|
|
||||||
|
// the returned row is the winner's persisted row, not a phantom
|
||||||
|
await using TvContext context = db.CreateContext();
|
||||||
|
LibraryFolder persisted = await context.LibraryFolders.SingleAsync(f => f.LibraryPathId == libraryPathId);
|
||||||
|
persisted.Id.ShouldBe(result.Id);
|
||||||
|
persisted.PathHash.ShouldBe(PathUtils.GetPathHash(FolderPath));
|
||||||
|
}
|
||||||
|
|
||||||
|
// The loser must still apply the parent id it was asked to set — to the WINNER's row.
|
||||||
|
[Test]
|
||||||
|
public async Task GetOrAddFolder_Losing_The_Race_Still_Persists_The_ParentId()
|
||||||
|
{
|
||||||
|
await using SharedCacheTvContext db = await SharedCacheTvContext.CreateAsync("etv491-race-parent");
|
||||||
|
int libraryPathId = await SeedLibraryPath(db.CreateContext);
|
||||||
|
var libraryPath = new LibraryPath { Id = libraryPathId, Path = LibraryPathValue, LibraryFolders = null };
|
||||||
|
|
||||||
|
LibraryFolder parent = await Repository(db.Factory())
|
||||||
|
.GetOrAddFolder(libraryPath, Option<int>.None, LibraryPathValue);
|
||||||
|
|
||||||
|
var racer = new InsertConflictingFolderOnce(db, libraryPathId, FolderPath);
|
||||||
|
LibraryFolder result = await Repository(db.Factory(racer))
|
||||||
|
.GetOrAddFolder(libraryPath, Option<int>.Some(parent.Id), FolderPath);
|
||||||
|
|
||||||
|
racer.Fired.ShouldBe(1);
|
||||||
|
result.ParentId.ShouldBe(parent.Id);
|
||||||
|
|
||||||
|
await using TvContext context = db.CreateContext();
|
||||||
|
LibraryFolder persisted = await context.LibraryFolders.SingleAsync(f => f.Path == FolderPath);
|
||||||
|
persisted.Id.ShouldBe(result.Id);
|
||||||
|
persisted.ParentId.ShouldBe(parent.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----- Negative control #2: invert the real condition (the provider classifier) and the SAME race
|
||||||
|
// must blow up, proving the catch in GetOrAddFolder is load-bearing rather than decorative. -----
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public async Task GetOrAddFolder_Rethrows_When_The_Provider_Does_Not_Classify_The_Violation()
|
||||||
|
{
|
||||||
|
Func<DbUpdateException, bool> original = TvContext.IsUniqueConstraintViolation;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await using SharedCacheTvContext db = await SharedCacheTvContext.CreateAsync("etv491-negctl");
|
||||||
|
int libraryPathId = await SeedLibraryPath(db.CreateContext);
|
||||||
|
var libraryPath = new LibraryPath { Id = libraryPathId, Path = LibraryPathValue, LibraryFolders = null };
|
||||||
|
|
||||||
|
var racer = new InsertConflictingFolderOnce(db, libraryPathId, FolderPath);
|
||||||
|
LibraryRepository repository = Repository(db.Factory(racer));
|
||||||
|
|
||||||
|
TvContext.IsUniqueConstraintViolation = _ => false;
|
||||||
|
await Should.ThrowAsync<DbUpdateException>(
|
||||||
|
() => repository.GetOrAddFolder(libraryPath, Option<int>.None, FolderPath));
|
||||||
|
|
||||||
|
racer.Fired.ShouldBe(1);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
TvContext.IsUniqueConstraintViolation = original;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----- N threads x rounds over a single (LibraryPathId, Path) -----
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public async Task Concurrent_GetOrAddFolder_Never_Produces_Duplicate_Rows()
|
||||||
|
{
|
||||||
|
const int threads = 8;
|
||||||
|
const int rounds = 10;
|
||||||
|
|
||||||
|
await using SharedCacheTvContext db = await SharedCacheTvContext.CreateAsync("etv491-threads");
|
||||||
|
int libraryPathId = await SeedLibraryPath(db.CreateContext);
|
||||||
|
|
||||||
|
var counter = new CountSaveAttempts();
|
||||||
|
IDbContextFactory<TvContext> factory = db.Factory(counter);
|
||||||
|
|
||||||
|
for (var round = 0; round < rounds; round++)
|
||||||
|
{
|
||||||
|
string path = $"{LibraryPathValue}/round{round}";
|
||||||
|
using var gate = new Barrier(threads);
|
||||||
|
|
||||||
|
var tasks = new Task<LibraryFolder>[threads];
|
||||||
|
for (var thread = 0; thread < threads; thread++)
|
||||||
|
{
|
||||||
|
tasks[thread] = Task.Run(async () =>
|
||||||
|
{
|
||||||
|
// every thread carries its own detached LibraryPath, as the scanners do
|
||||||
|
var libraryPath = new LibraryPath
|
||||||
|
{
|
||||||
|
Id = libraryPathId, Path = LibraryPathValue, LibraryFolders = null
|
||||||
|
};
|
||||||
|
|
||||||
|
gate.SignalAndWait();
|
||||||
|
return await Repository(factory).GetOrAddFolder(libraryPath, Option<int>.None, path);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
LibraryFolder[] results = await Task.WhenAll(tasks);
|
||||||
|
|
||||||
|
// every caller got the same single row back...
|
||||||
|
results.Select(f => f.Id).Distinct().Count().ShouldBe(1);
|
||||||
|
results[0].Id.ShouldBeGreaterThan(0);
|
||||||
|
|
||||||
|
// ...and exactly one row exists for it
|
||||||
|
(await FolderCount(db.CreateContext, libraryPathId, path)).ShouldBe(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
await using TvContext context = db.CreateContext();
|
||||||
|
(await context.LibraryFolders.CountAsync(f => f.LibraryPathId == libraryPathId)).ShouldBe(rounds);
|
||||||
|
|
||||||
|
// Vacuity guard: one insert attempt per round would mean the threads never actually collided and
|
||||||
|
// the test proved nothing. More attempts than rounds means at least one caller lost the race and
|
||||||
|
// was rescued by the index + catch.
|
||||||
|
counter.Attempts.ShouldBeGreaterThan(rounds);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----- SetEtag is the repository's other check-then-insert on LibraryFolder -----
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public async Task SetEtag_Losing_The_Race_Updates_The_Winner_Instead_Of_Duplicating()
|
||||||
|
{
|
||||||
|
await using SharedCacheTvContext db = await SharedCacheTvContext.CreateAsync("etv491-setetag");
|
||||||
|
int libraryPathId = await SeedLibraryPath(db.CreateContext);
|
||||||
|
var libraryPath = new LibraryPath { Id = libraryPathId, Path = LibraryPathValue, LibraryFolders = null };
|
||||||
|
|
||||||
|
var racer = new InsertConflictingFolderOnce(db, libraryPathId, FolderPath);
|
||||||
|
LibraryRepository repository = Repository(db.Factory(racer));
|
||||||
|
|
||||||
|
await repository.SetEtag(libraryPath, Option<LibraryFolder>.None, FolderPath, "etag-1");
|
||||||
|
|
||||||
|
racer.Fired.ShouldBe(1);
|
||||||
|
|
||||||
|
await using TvContext context = db.CreateContext();
|
||||||
|
LibraryFolder persisted = await context.LibraryFolders.SingleAsync(f => f.Path == FolderPath);
|
||||||
|
persisted.Etag.ShouldBe("etag-1");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,180 @@
|
|||||||
|
using Dapper;
|
||||||
|
using ErsatzTV.Infrastructure;
|
||||||
|
using ErsatzTV.Infrastructure.Data;
|
||||||
|
using ErsatzTV.Infrastructure.Sqlite.Data;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
using NUnit.Framework;
|
||||||
|
using Shouldly;
|
||||||
|
|
||||||
|
namespace ErsatzTV.Tests.Integration;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// ersatztv#491: the unique index on <c>LibraryFolder(LibraryPathId, PathHash)</c> ships with an
|
||||||
|
/// audit/cleanup of pre-existing duplicate rows (reachable before #488, when the folder lookup read a
|
||||||
|
/// scan-start in-memory snapshot and could not see a folder created earlier in the same scan). This
|
||||||
|
/// drives the REAL Sqlite migration against a database seeded at the previous migration, so the
|
||||||
|
/// cleanup SQL is exercised rather than restated.
|
||||||
|
/// </summary>
|
||||||
|
[TestFixture]
|
||||||
|
public class LibraryFolderDedupeMigrationTests
|
||||||
|
{
|
||||||
|
// the migration immediately preceding Add_LibraryFolder_PathHash_UniqueIndex
|
||||||
|
private const string PreviousMigration = "Add_Channel_Origin";
|
||||||
|
|
||||||
|
private string _databasePath = null!;
|
||||||
|
private DbContextOptions<TvContext> _options = null!;
|
||||||
|
|
||||||
|
[SetUp]
|
||||||
|
public void SetUp()
|
||||||
|
{
|
||||||
|
TvContext.IsSqlite = true;
|
||||||
|
TvContext.IsUniqueConstraintViolation = SqliteErrorClassifier.IsUniqueConstraintViolation;
|
||||||
|
|
||||||
|
_databasePath = Path.Combine(Path.GetTempPath(), $"etv491-{Guid.NewGuid():N}.sqlite3");
|
||||||
|
_options = new DbContextOptionsBuilder<TvContext>()
|
||||||
|
.UseSqlite(
|
||||||
|
$"Data Source={_databasePath};Foreign Keys=False",
|
||||||
|
o => o.MigrationsAssembly("ErsatzTV.Infrastructure.Sqlite"))
|
||||||
|
.Options;
|
||||||
|
}
|
||||||
|
|
||||||
|
[TearDown]
|
||||||
|
public void TearDown()
|
||||||
|
{
|
||||||
|
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
|
||||||
|
foreach (string path in new[] { _databasePath, $"{_databasePath}-wal", $"{_databasePath}-shm" })
|
||||||
|
{
|
||||||
|
if (File.Exists(path))
|
||||||
|
{
|
||||||
|
File.Delete(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public async Task Migration_Collapses_Duplicate_Folders_And_Repoints_Their_Dependents()
|
||||||
|
{
|
||||||
|
await using (TvContext context = CreateContext())
|
||||||
|
{
|
||||||
|
await context.Database.MigrateAsync(PreviousMigration);
|
||||||
|
|
||||||
|
// one library path with the SAME folder recorded three times (ids 1, 2, 3) plus an unrelated
|
||||||
|
// folder (id 4) and a child parented on one of the duplicates (id 5)
|
||||||
|
await context.Connection.ExecuteAsync(
|
||||||
|
"INSERT INTO LibraryPath (Id, LibraryId, Path) VALUES (1, 1, '/data/music')");
|
||||||
|
await context.Connection.ExecuteAsync(
|
||||||
|
"""
|
||||||
|
INSERT INTO LibraryFolder (Id, LibraryPathId, Path, ParentId, Etag) VALUES
|
||||||
|
(1, 1, '/data/music/artist1', NULL, 'etag-keeper'),
|
||||||
|
(2, 1, '/data/music/artist1', NULL, 'etag-dupe-a'),
|
||||||
|
(3, 1, '/data/music/artist1', NULL, 'etag-dupe-b'),
|
||||||
|
(4, 1, '/data/music/artist2', NULL, NULL),
|
||||||
|
(5, 1, '/data/music/artist1/album', 3, NULL)
|
||||||
|
""");
|
||||||
|
|
||||||
|
// a media file on each duplicate, and an image-folder-duration on a duplicate only
|
||||||
|
await context.Connection.ExecuteAsync(
|
||||||
|
"""
|
||||||
|
INSERT INTO MediaFile (Id, Path, PathHash, MediaVersionId, LibraryFolderId) VALUES
|
||||||
|
(1, '/data/music/artist1/a.mkv', 'hash-a', 1, 1),
|
||||||
|
(2, '/data/music/artist1/b.mkv', 'hash-b', 2, 2),
|
||||||
|
(3, '/data/music/artist1/c.mkv', 'hash-c', 3, 3)
|
||||||
|
""");
|
||||||
|
await context.Connection.ExecuteAsync(
|
||||||
|
"""
|
||||||
|
INSERT INTO ImageFolderDuration (Id, LibraryFolderId, DurationSeconds) VALUES
|
||||||
|
(1, 2, 30.0),
|
||||||
|
(2, 3, 45.0)
|
||||||
|
""");
|
||||||
|
}
|
||||||
|
|
||||||
|
await using (TvContext context = CreateContext())
|
||||||
|
{
|
||||||
|
await context.Database.MigrateAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
await using (TvContext context = CreateContext())
|
||||||
|
{
|
||||||
|
// the duplicates are gone; the lowest id survives
|
||||||
|
List<int> folderIds =
|
||||||
|
(await context.Connection.QueryAsync<int>(
|
||||||
|
"SELECT Id FROM LibraryFolder WHERE Path = '/data/music/artist1'")).ToList();
|
||||||
|
folderIds.ShouldBe([1]);
|
||||||
|
|
||||||
|
// the untouched folder and the child are still there
|
||||||
|
(await context.Connection.ExecuteScalarAsync<int>("SELECT COUNT(*) FROM LibraryFolder")).ShouldBe(3);
|
||||||
|
|
||||||
|
// every media file follows the keeper — nothing orphaned, nothing deleted
|
||||||
|
List<int> mediaFolderIds =
|
||||||
|
(await context.Connection.QueryAsync<int>(
|
||||||
|
"SELECT LibraryFolderId FROM MediaFile ORDER BY Id")).ToList();
|
||||||
|
mediaFolderIds.ShouldBe([1, 1, 1]);
|
||||||
|
|
||||||
|
// the child folder is reparented off the deleted duplicate onto the keeper
|
||||||
|
(await context.Connection.ExecuteScalarAsync<int>(
|
||||||
|
"SELECT ParentId FROM LibraryFolder WHERE Id = 5")).ShouldBe(1);
|
||||||
|
|
||||||
|
// the keeper had no ImageFolderDuration, so exactly one duplicate's setting is promoted to it
|
||||||
|
// (the lowest id) and the rest are dropped — the 1:1 unique index cannot hold both
|
||||||
|
List<string> durations =
|
||||||
|
(await context.Connection.QueryAsync<string>(
|
||||||
|
"SELECT Id || ':' || LibraryFolderId FROM ImageFolderDuration ORDER BY Id")).ToList();
|
||||||
|
durations.ShouldBe(["1:1"]);
|
||||||
|
|
||||||
|
// the helper tables the cleanup used are not left behind
|
||||||
|
(await context.Connection.ExecuteScalarAsync<int>(
|
||||||
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name LIKE '__LibraryFolderDedupe%'"))
|
||||||
|
.ShouldBe(0);
|
||||||
|
|
||||||
|
// and the unique index is now in place and enforcing
|
||||||
|
(await context.Connection.ExecuteScalarAsync<int>(
|
||||||
|
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'IX_LibraryFolder_LibraryPathId_PathHash'"))
|
||||||
|
.ShouldBe(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Test]
|
||||||
|
public async Task Migration_Leaves_A_Database_Without_Duplicates_Alone()
|
||||||
|
{
|
||||||
|
// positive control: the cleanup must not touch rows that were already unique
|
||||||
|
await using (TvContext context = CreateContext())
|
||||||
|
{
|
||||||
|
await context.Database.MigrateAsync(PreviousMigration);
|
||||||
|
await context.Connection.ExecuteAsync(
|
||||||
|
"INSERT INTO LibraryPath (Id, LibraryId, Path) VALUES (1, 1, '/data/music')");
|
||||||
|
await context.Connection.ExecuteAsync(
|
||||||
|
"""
|
||||||
|
INSERT INTO LibraryFolder (Id, LibraryPathId, Path, ParentId, Etag) VALUES
|
||||||
|
(1, 1, '/data/music/artist1', NULL, 'etag-1'),
|
||||||
|
(2, 1, '/data/music/artist2', 1, 'etag-2')
|
||||||
|
""");
|
||||||
|
await context.Connection.ExecuteAsync(
|
||||||
|
"INSERT INTO ImageFolderDuration (Id, LibraryFolderId, DurationSeconds) VALUES (1, 2, 30.0)");
|
||||||
|
}
|
||||||
|
|
||||||
|
await using (TvContext context = CreateContext())
|
||||||
|
{
|
||||||
|
await context.Database.MigrateAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
await using (TvContext context = CreateContext())
|
||||||
|
{
|
||||||
|
(await context.Connection.ExecuteScalarAsync<int>("SELECT COUNT(*) FROM LibraryFolder")).ShouldBe(2);
|
||||||
|
(await context.Connection.ExecuteScalarAsync<int>("SELECT ParentId FROM LibraryFolder WHERE Id = 2"))
|
||||||
|
.ShouldBe(1);
|
||||||
|
(await context.Connection.ExecuteScalarAsync<int>(
|
||||||
|
"SELECT LibraryFolderId FROM ImageFolderDuration WHERE Id = 1")).ShouldBe(2);
|
||||||
|
|
||||||
|
// existing rows keep a null hash — the index applies because nulls are distinct
|
||||||
|
(await context.Connection.ExecuteScalarAsync<int>(
|
||||||
|
"SELECT COUNT(*) FROM LibraryFolder WHERE PathHash IS NULL")).ShouldBe(2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private TvContext CreateContext() =>
|
||||||
|
new(
|
||||||
|
_options,
|
||||||
|
NullLoggerFactory.Instance,
|
||||||
|
new SlowQueryInterceptor(NullLogger<SlowQueryInterceptor>.Instance));
|
||||||
|
}
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
using ErsatzTV.Core;
|
||||||
using ErsatzTV.Core.Domain;
|
using ErsatzTV.Core.Domain;
|
||||||
using ErsatzTV.Infrastructure.Data;
|
using ErsatzTV.Infrastructure.Data;
|
||||||
using ErsatzTV.Infrastructure.Data.Repositories;
|
using ErsatzTV.Infrastructure.Data.Repositories;
|
||||||
@@ -105,6 +106,54 @@ public class LibraryRepositoryTests
|
|||||||
count.ShouldBe(1);
|
count.ShouldBe(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ersatztv#491: the unique index is on (LibraryPathId, PathHash) because Path is unbounded, so every
|
||||||
|
// new row must carry the hash or the constraint is unenforceable for it.
|
||||||
|
[Test]
|
||||||
|
public async Task GetOrAddFolder_Should_Populate_PathHash_On_New_Folder()
|
||||||
|
{
|
||||||
|
int libraryPathId = await SeedLibraryPath("/data/music");
|
||||||
|
var libraryPath = new LibraryPath { Id = libraryPathId, Path = "/data/music", LibraryFolders = null };
|
||||||
|
|
||||||
|
LibraryFolder result = await _repository.GetOrAddFolder(libraryPath, Option<int>.None, "/data/music/artist1");
|
||||||
|
|
||||||
|
result.PathHash.ShouldBe(PathUtils.GetPathHash("/data/music/artist1"));
|
||||||
|
|
||||||
|
await using TvContext context = _db.CreateContext();
|
||||||
|
LibraryFolder persisted = await context.LibraryFolders.SingleAsync(f => f.Id == result.Id);
|
||||||
|
persisted.PathHash.ShouldBe(PathUtils.GetPathHash("/data/music/artist1"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ersatztv#491: rows that predate the PathHash column are left null by the migration (nulls are
|
||||||
|
// distinct in a unique index, so the index applies cleanly); the first scan that touches one heals it.
|
||||||
|
[Test]
|
||||||
|
public async Task GetOrAddFolder_Should_Heal_A_Legacy_Null_PathHash()
|
||||||
|
{
|
||||||
|
int libraryPathId = await SeedLibraryPath("/data/music");
|
||||||
|
var libraryPath = new LibraryPath { Id = libraryPathId, Path = "/data/music", LibraryFolders = null };
|
||||||
|
|
||||||
|
int legacyId;
|
||||||
|
await using (TvContext seed = _db.CreateContext())
|
||||||
|
{
|
||||||
|
var legacy = new LibraryFolder
|
||||||
|
{
|
||||||
|
LibraryPathId = libraryPathId, Path = "/data/music/artist1", PathHash = null
|
||||||
|
};
|
||||||
|
await seed.LibraryFolders.AddAsync(legacy);
|
||||||
|
await seed.SaveChangesAsync();
|
||||||
|
legacyId = legacy.Id;
|
||||||
|
}
|
||||||
|
|
||||||
|
LibraryFolder result = await _repository.GetOrAddFolder(libraryPath, Option<int>.None, "/data/music/artist1");
|
||||||
|
|
||||||
|
result.Id.ShouldBe(legacyId);
|
||||||
|
result.PathHash.ShouldBe(PathUtils.GetPathHash("/data/music/artist1"));
|
||||||
|
|
||||||
|
await using TvContext context = _db.CreateContext();
|
||||||
|
LibraryFolder persisted = await context.LibraryFolders.SingleAsync(f => f.Id == legacyId);
|
||||||
|
persisted.PathHash.ShouldBe(PathUtils.GetPathHash("/data/music/artist1"));
|
||||||
|
(await context.LibraryFolders.CountAsync(f => f.LibraryPathId == libraryPathId)).ShouldBe(1);
|
||||||
|
}
|
||||||
|
|
||||||
private async Task<int> SeedLibraryPath(string path)
|
private async Task<int> SeedLibraryPath(string path)
|
||||||
{
|
{
|
||||||
await using TvContext context = _db.CreateContext();
|
await using TvContext context = _db.CreateContext();
|
||||||
|
|||||||
Reference in New Issue
Block a user