Cross-family review of 1b4dd6d6 found that utf8mb4_bin - chosen to keep the dedupe case-exact - is a PAD SPACE collation, so trailing spaces are insignificant under it. Verified on MySQL 8.4: '/media/Foo' = '/media/Foo ' is TRUE, while case correctly compares unequal. Two distinct legal directories therefore grouped together and the second was DELETED irreversibly, even though PathUtils.GetPathHash hashes them differently and the unique index about to be created would have accepted both. The dedupe destroyed data the constraint never required it to destroy. Group and join on CONVERT(Path USING binary) instead - NO PAD and byte-exact, matching the hash. utf8mb4_0900_bin is also NO PAD but carries a server-version floor. This is the only path comparison in either migration (every other predicate keys off an integer id), so there is no mix of padded and unpadded comparisons across the keeper-selection, repoint and delete steps. SQLite's = on TEXT is byte-exact with no padding, so that migration was already correct - which is exactly why a SQLite-only test could not see the divergence. The two providers are now semantically equivalent, and the dedupe fixture is shared: same rows, same expected survivors (1,4,5,6,7,9,10), asserted by the SQLite test and reproduced by hand on MySQL 8.4. Runtime was never affected, and this is now stated and tested rather than assumed: GetFolder's SQL equality is a superset narrowing (both collation quirks make it more permissive, never less, so it cannot miss a byte-exact match) and ResolveExact settles identity with StringComparison.Ordinal, which compares length first. Added ResolveExact coverage for the trailing-space axis. Refs #488 #308 fix #491
173 lines
8.7 KiB
C#
173 lines
8.7 KiB
C#
using Microsoft.EntityFrameworkCore.Migrations;
|
|
|
|
#nullable disable
|
|
|
|
namespace ErsatzTV.Infrastructure.MySql.Migrations
|
|
{
|
|
/// <inheritdoc />
|
|
public partial class Add_LibraryFolder_PathHash_UniqueIndex : Migration
|
|
{
|
|
/// <inheritdoc />
|
|
protected override void Up(MigrationBuilder migrationBuilder)
|
|
{
|
|
// ersatztv#491 — audit/clean pre-existing duplicate LibraryFolder rows before the unique index.
|
|
// Mirrors the Sqlite migration; see it for the full rationale. The one provider difference is
|
|
// that every Path comparison here is forced BYTE-EXACT with CONVERT(... USING binary), because
|
|
// MySql's string comparison differs from Sqlite's on two independent axes and the dedupe
|
|
// deletes rows irreversibly:
|
|
// * case — the server default (utf8mb4_general_ci) is case-INsensitive, so grouping under it
|
|
// would collapse sibling folders differing only in case, legal on a case-sensitive fs;
|
|
// * trailing spaces — utf8mb4_bin, the obvious fix for the case half, is a PAD SPACE
|
|
// collation (verified on 8.4: '/media/Foo' = '/media/Foo ' is TRUE under it), so it would
|
|
// still collapse "/media/Foo" and "/media/Foo ", two distinct legal directories.
|
|
// Binary comparison is NO PAD and byte-exact, which is exactly what PathUtils.GetPathHash does
|
|
// — so the dedupe now destroys only rows the unique index would actually have rejected, and
|
|
// the two providers' migrations are semantically equivalent. (utf8mb4_0900_bin is also NO PAD
|
|
// but carries a server-version floor; CONVERT USING binary does not.)
|
|
// DROP TABLE IF EXISTS makes a retry after a partial failure safe (DDL implicitly commits on
|
|
// MySql, so the migration is not atomic).
|
|
migrationBuilder.Sql("DROP TABLE IF EXISTS `__LibraryFolderDedupe`");
|
|
migrationBuilder.Sql(
|
|
"""
|
|
CREATE TABLE `__LibraryFolderDedupe` (
|
|
LoserId INT NOT NULL PRIMARY KEY,
|
|
KeeperId INT NOT NULL
|
|
)
|
|
""");
|
|
|
|
migrationBuilder.Sql(
|
|
"""
|
|
INSERT INTO `__LibraryFolderDedupe` (LoserId, KeeperId)
|
|
SELECT l.Id, k.KeeperId
|
|
FROM LibraryFolder l
|
|
INNER JOIN (
|
|
SELECT LibraryPathId, CONVERT(Path USING binary) AS BinPath, MIN(Id) AS KeeperId
|
|
FROM LibraryFolder
|
|
GROUP BY LibraryPathId, CONVERT(Path USING binary)
|
|
) k ON k.LibraryPathId = l.LibraryPathId AND k.BinPath = CONVERT(l.Path USING binary)
|
|
WHERE l.Id <> k.KeeperId
|
|
""");
|
|
|
|
// media files recorded against a duplicate folder follow the keeper (MediaFile.LibraryFolderId
|
|
// is Restrict, so the delete below would fail otherwise)
|
|
migrationBuilder.Sql(
|
|
"""
|
|
UPDATE MediaFile
|
|
SET LibraryFolderId = (
|
|
SELECT KeeperId FROM `__LibraryFolderDedupe` WHERE LoserId = MediaFile.LibraryFolderId)
|
|
WHERE LibraryFolderId IN (SELECT LoserId FROM `__LibraryFolderDedupe`)
|
|
""");
|
|
|
|
// child folders parented on a duplicate follow the keeper (ParentId is Restrict as well)
|
|
migrationBuilder.Sql(
|
|
"""
|
|
UPDATE LibraryFolder
|
|
SET ParentId = (
|
|
SELECT KeeperId FROM `__LibraryFolderDedupe` WHERE LoserId = LibraryFolder.ParentId)
|
|
WHERE ParentId IN (SELECT LoserId FROM `__LibraryFolderDedupe`)
|
|
""");
|
|
|
|
// A folder parented on its OWN duplicate would become its own parent above. Unreachable from
|
|
// any code path today, but this is a tree the scanner walks, so remove the cycle class rather
|
|
// than reason about it.
|
|
migrationBuilder.Sql("UPDATE LibraryFolder SET ParentId = NULL WHERE ParentId = Id");
|
|
|
|
// Clear the survivor's etag. Which duplicate the scanner was actually writing to was
|
|
// arbitrary, so MIN(Id)'s etag may describe a stale view of the folder and would suppress the
|
|
// next rescan. A null etag costs exactly one rescan and cannot be wrong.
|
|
migrationBuilder.Sql(
|
|
"UPDATE LibraryFolder SET Etag = NULL WHERE Id IN (SELECT KeeperId FROM `__LibraryFolderDedupe`)");
|
|
|
|
// ImageFolderDuration is 1:1 with LibraryFolder (unique index on LibraryFolderId), so the
|
|
// duplicates' rows cannot all be repointed. Keep the keeper's own setting when it has one;
|
|
// otherwise promote exactly one loser's (lowest Id) and drop the rest.
|
|
migrationBuilder.Sql("DROP TABLE IF EXISTS `__LibraryFolderDedupeIfd`");
|
|
migrationBuilder.Sql(
|
|
"""
|
|
CREATE TABLE `__LibraryFolderDedupeIfd` (
|
|
KeeperId INT NOT NULL PRIMARY KEY,
|
|
IfdId INT NOT NULL
|
|
)
|
|
""");
|
|
|
|
migrationBuilder.Sql(
|
|
"""
|
|
INSERT INTO `__LibraryFolderDedupeIfd` (KeeperId, IfdId)
|
|
SELECT d.KeeperId, MIN(i.Id)
|
|
FROM `__LibraryFolderDedupe` d
|
|
INNER JOIN ImageFolderDuration i ON i.LibraryFolderId = d.LoserId
|
|
WHERE NOT EXISTS (
|
|
SELECT 1 FROM ImageFolderDuration ki WHERE ki.LibraryFolderId = d.KeeperId)
|
|
GROUP BY d.KeeperId
|
|
""");
|
|
|
|
migrationBuilder.Sql(
|
|
"""
|
|
DELETE FROM ImageFolderDuration
|
|
WHERE LibraryFolderId IN (SELECT LoserId FROM `__LibraryFolderDedupe`)
|
|
AND Id NOT IN (SELECT IfdId FROM `__LibraryFolderDedupeIfd`)
|
|
""");
|
|
|
|
migrationBuilder.Sql(
|
|
"""
|
|
UPDATE ImageFolderDuration
|
|
SET LibraryFolderId = (
|
|
SELECT KeeperId FROM `__LibraryFolderDedupeIfd` WHERE IfdId = ImageFolderDuration.Id)
|
|
WHERE Id IN (SELECT IfdId FROM `__LibraryFolderDedupeIfd`)
|
|
""");
|
|
|
|
migrationBuilder.Sql(
|
|
"DELETE FROM LibraryFolder WHERE Id IN (SELECT LoserId FROM `__LibraryFolderDedupe`)");
|
|
|
|
migrationBuilder.Sql("DROP TABLE `__LibraryFolderDedupeIfd`");
|
|
migrationBuilder.Sql("DROP TABLE `__LibraryFolderDedupe`");
|
|
|
|
migrationBuilder.AddColumn<string>(
|
|
name: "PathHash",
|
|
table: "LibraryFolder",
|
|
type: "varchar(64)",
|
|
maxLength: 64,
|
|
nullable: true)
|
|
.Annotation("MySql:CharSet", "utf8mb4");
|
|
|
|
// Existing rows keep a null hash on purpose: a unique index treats nulls as distinct, so the
|
|
// index applies cleanly to any database, and LibraryRepository.GetOrAddFolder heals each row
|
|
// (SHA-256 of Path) the first time a scan touches it. Those rows are deduplicated above and are
|
|
// still found by the Path lookup, so no insert can race them in the meantime.
|
|
//
|
|
// Order matters on MySql, and EF scaffolds it the other way round: InnoDB refuses to drop the
|
|
// FK's only backing index ("Cannot drop index 'IX_LibraryFolder_LibraryPathId': needed in a
|
|
// foreign key constraint"). Create the composite first — LibraryPathId is its leftmost column,
|
|
// so it takes over as the FK's backing index — then drop the now-redundant single-column one.
|
|
migrationBuilder.CreateIndex(
|
|
name: "IX_LibraryFolder_LibraryPathId_PathHash",
|
|
table: "LibraryFolder",
|
|
columns: new[] { "LibraryPathId", "PathHash" },
|
|
unique: true);
|
|
|
|
migrationBuilder.DropIndex(
|
|
name: "IX_LibraryFolder_LibraryPathId",
|
|
table: "LibraryFolder");
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
protected override void Down(MigrationBuilder migrationBuilder)
|
|
{
|
|
// mirror of Up: restore the single-column index before dropping the composite one, so the
|
|
// foreign key is never left without a backing index
|
|
migrationBuilder.CreateIndex(
|
|
name: "IX_LibraryFolder_LibraryPathId",
|
|
table: "LibraryFolder",
|
|
column: "LibraryPathId");
|
|
|
|
migrationBuilder.DropIndex(
|
|
name: "IX_LibraryFolder_LibraryPathId_PathHash",
|
|
table: "LibraryFolder");
|
|
|
|
migrationBuilder.DropColumn(
|
|
name: "PathHash",
|
|
table: "LibraryFolder");
|
|
}
|
|
}
|
|
}
|