diff --git a/packages/core/src/imports/parsers.ts b/packages/core/src/imports/parsers.ts index d3bd7a1..31eb4d2 100644 --- a/packages/core/src/imports/parsers.ts +++ b/packages/core/src/imports/parsers.ts @@ -1,3 +1,11 @@ +import type { z } from "zod"; + +import { + ImportEpisodeSchema, + ImportMovieSchema, + ImportRatingSchema, + ImportWatchlistItemSchema, +} from "@sofa/api/schemas"; import { createLogger } from "@sofa/logger"; const log = createLogger("imports"); @@ -89,6 +97,76 @@ export function countUnresolved(data: NormalizedImport): number { return count; } +/** Mirrors the `.max()` on each list in NormalizedImportSchema. */ +const MAX_ITEMS_PER_LIST = 50_000; + +/** Turn `null` / `NaN` field values into `undefined` (the API schemas allow optional, not null). */ +function withoutNulls(item: T): T { + const clean: Record = {}; + for (const [key, value] of Object.entries(item)) { + if (value === null || (typeof value === "number" && Number.isNaN(value))) continue; + clean[key] = value; + } + return clean as T; +} + +/** + * Make parser output conform to NormalizedImportSchema. Real exports contain + * nulls (Trakt sends `"imdb": null`, `"year": null`) and occasional invalid + * entries (e.g. episode number 0); one bad item must not fail the whole parse, + * so invalid items are dropped and summarized in a warning. + */ +export function finalizeParseResult( + data: NormalizedImport, + warnings: string[], + unsupported = 0, +): ParseResult { + let dropped = 0; + + function clean(list: string, items: T[], schema: z.ZodType): T[] { + const kept: T[] = []; + const examples: string[] = []; + let invalid = 0; + for (const raw of items) { + const result = schema.safeParse(withoutNulls(raw)); + if (result.success) { + kept.push(result.data); + } else { + invalid++; + if (examples.length < 3) { + const issue = result.error.issues[0]; + examples.push(`${issue?.path.join(".") || "item"}: ${issue?.message ?? "invalid"}`); + } + } + } + if (invalid > 0) { + dropped += invalid; + warnings.push(`Skipped ${invalid} invalid ${list} (${examples.join("; ")})`); + } + if (kept.length > MAX_ITEMS_PER_LIST) { + warnings.push( + `Only the first ${MAX_ITEMS_PER_LIST.toLocaleString("en-US")} ${list} were included (${kept.length.toLocaleString("en-US")} found)`, + ); + return kept.slice(0, MAX_ITEMS_PER_LIST); + } + return kept; + } + + const normalized: NormalizedImport = { + source: data.source, + movies: clean("movies", data.movies, ImportMovieSchema), + episodes: clean("episodes", data.episodes, ImportEpisodeSchema), + watchlist: clean("watchlist items", data.watchlist, ImportWatchlistItemSchema), + ratings: clean("ratings", data.ratings, ImportRatingSchema), + }; + + return { + data: normalized, + warnings, + diagnostics: { unresolved: countUnresolved(normalized), unsupported: unsupported + dropped }, + }; +} + // ─── Rating Conversion ────────────────────────────────────────────── /** Convert a 1-10 rating to Sofa's 1-5 integer scale. */ @@ -304,11 +382,7 @@ export function parseTraktPayload(data: { ratings, }; - return { - data: normalized, - warnings, - diagnostics: { unresolved: countUnresolved(normalized), unsupported }, - }; + return finalizeParseResult(normalized, warnings, unsupported); } // ─── Simkl Parser ─────────────────────────────────────────────────── @@ -500,11 +574,7 @@ export function parseSimklPayload(data: { ratings, }; - return { - data: normalized, - warnings, - diagnostics: { unresolved: countUnresolved(normalized), unsupported: 0 }, - }; + return finalizeParseResult(normalized, warnings); } // ─── Letterboxd Parser ────────────────────────────────────────────── @@ -668,12 +738,9 @@ export async function parseLetterboxdExport(zipFile: Blob): Promise `Parsed Letterboxd export: ${movies.length} movies, ${watchlist.length} watchlist, ${ratings.length} ratings`, ); - // Letterboxd has no IDs — all items need title-based resolution - const unresolved = movies.length + watchlist.length + ratings.length; - - return { - data: { source: "letterboxd", movies, episodes: [], watchlist, ratings }, + // Letterboxd has no IDs — all items need title-based resolution (counted by the finalizer) + return finalizeParseResult( + { source: "letterboxd", movies, episodes: [], watchlist, ratings }, warnings, - diagnostics: { unresolved, unsupported: 0 }, - }; + ); } diff --git a/packages/core/src/imports/sofa-parser.ts b/packages/core/src/imports/sofa-parser.ts index 2047f37..dd05d06 100644 --- a/packages/core/src/imports/sofa-parser.ts +++ b/packages/core/src/imports/sofa-parser.ts @@ -2,7 +2,7 @@ import { SofaExportSchema } from "@sofa/api/schemas"; import { createLogger } from "@sofa/logger"; import type { NormalizedImport, ParseResult } from "./parsers"; -import { countUnresolved } from "./parsers"; +import { finalizeParseResult } from "./parsers"; const log = createLogger("imports"); @@ -59,9 +59,5 @@ export function parseSofaExport(data: unknown): ParseResult { `Parsed Sofa export: ${normalized.movies.length} movies, ${normalized.episodes.length} episodes, ${normalized.watchlist.length} library, ${normalized.ratings.length} ratings`, ); - return { - data: normalized, - warnings, - diagnostics: { unresolved: countUnresolved(normalized), unsupported: 0 }, - }; + return finalizeParseResult(normalized, warnings); } diff --git a/packages/core/test/import-parsers.test.ts b/packages/core/test/import-parsers.test.ts index 9ead527..6b42b2b 100644 --- a/packages/core/test/import-parsers.test.ts +++ b/packages/core/test/import-parsers.test.ts @@ -1,6 +1,8 @@ import AdmZip from "adm-zip"; import { describe, expect, test } from "vitest"; +import { NormalizedImportSchema } from "@sofa/api/schemas"; + import { type ParseResult, parseLetterboxdExport, @@ -1072,3 +1074,102 @@ describe("parseLetterboxdExport", () => { } }); }); + +// ─── Output always matches the API schema ──────────────────────────── + +describe("parser output always matches the API schema", () => { + test("Trakt nulls become undefined", () => { + const result = parseTraktPayload({ + history: { + movies: [ + { + watched_at: "2024-01-15T20:00:00.000Z", + movie: { title: "No IDs", year: null, ids: { trakt: 1, tmdb: null, imdb: null } }, + }, + ], + shows: [ + { + watched_at: "2024-01-15T21:00:00.000Z", + show: { title: "Show", year: null, ids: { tmdb: 1396, imdb: null, tvdb: null } }, + episode: { season: 1, number: 1 }, + }, + ], + }, + watchlist: [ + { type: "movie", movie: { title: "W", year: null, ids: { tmdb: 5, imdb: null } } }, + ], + ratings: [ + { + type: "movie", + rating: 8, + rated_at: null, + movie: { title: "R", year: 2000, ids: { tmdb: 6, imdb: null } }, + }, + ], + } as never); + + expect(NormalizedImportSchema.safeParse(result.data).success).toBe(true); + expect(result.data.movies).toHaveLength(1); + expect(result.data.episodes).toHaveLength(1); + expect(result.data.watchlist).toHaveLength(1); + expect(result.data.ratings).toHaveLength(1); + expect(result.data.movies[0].imdbId).toBeUndefined(); + }); + + test("invalid items are dropped with a warning, not fatal", () => { + const result = parseTraktPayload({ + history: { + shows: [ + { + watched_at: "2024-01-15T21:00:00.000Z", + show: { title: "Show", ids: { tmdb: 1396 } }, + episode: { season: 0, number: 0 }, + }, + { + watched_at: "2024-01-15T21:00:00.000Z", + show: { title: "Show", ids: { tmdb: 1396 } }, + episode: { season: 1, number: 2 }, + }, + ], + }, + }); + + expect(result.data.episodes).toHaveLength(1); + expect(NormalizedImportSchema.safeParse(result.data).success).toBe(true); + expect(result.warnings.some((w) => w.includes("episodes"))).toBe(true); + expect(result.diagnostics?.unsupported).toBeGreaterThanOrEqual(1); + }); + + test("Simkl nulls become undefined", () => { + const result = parseSimklPayload({ + movies: [ + { + title: "M", + year: null, + status: "completed", + last_watched_at: null, + ids: { tmdb: 7, imdb: null }, + }, + ], + } as never); + + expect(result.data.movies).toHaveLength(1); + expect(NormalizedImportSchema.safeParse(result.data).success).toBe(true); + expect(result.data.movies[0].watchedAt).toBeUndefined(); + }); + + test("oversized lists are truncated with a warning", () => { + const result = parseTraktPayload({ + history: { + movies: Array.from({ length: 50_001 }, (_, i) => ({ + watched_at: "2024-01-15T20:00:00.000Z", + movie: { title: `M${i}`, ids: { tmdb: i + 1 } }, + })), + }, + }); + + expect(result.data.movies).toHaveLength(50_000); + expect(NormalizedImportSchema.safeParse(result.data).success).toBe(true); + expect(result.warnings.some((w) => w.includes("50,000"))).toBe(true); + }); +});