fix(imports): normalize nulls and drop invalid items instead of failing the parse

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

packages/core/src/imports/parsers.ts

@@ -1,3 +1,11 @@
+import type { z } from "zod";
+
+import {
+  ImportEpisodeSchema,
+  ImportMovieSchema,
+  ImportRatingSchema,
+  ImportWatchlistItemSchema,
+} from "@sofa/api/schemas";
 import { createLogger } from "@sofa/logger";
 
 const log = createLogger("imports");
@@ -89,6 +97,76 @@ export function countUnresolved(data: NormalizedImport): number {
   return count;
 }
 
+/** Mirrors the `.max()` on each list in NormalizedImportSchema. */
+const MAX_ITEMS_PER_LIST = 50_000;
+
+/** Turn `null` / `NaN` field values into `undefined` (the API schemas allow optional, not null). */
+function withoutNulls<T extends object>(item: T): T {
+  const clean: Record<string, unknown> = {};
+  for (const [key, value] of Object.entries(item)) {
+    if (value === null || (typeof value === "number" && Number.isNaN(value))) continue;
+    clean[key] = value;
+  }
+  return clean as T;
+}
+
+/**
+ * Make parser output conform to NormalizedImportSchema. Real exports contain
+ * nulls (Trakt sends `"imdb": null`, `"year": null`) and occasional invalid
+ * entries (e.g. episode number 0); one bad item must not fail the whole parse,
+ * so invalid items are dropped and summarized in a warning.
+ */
+export function finalizeParseResult(
+  data: NormalizedImport,
+  warnings: string[],
+  unsupported = 0,
+): ParseResult {
+  let dropped = 0;
+
+  function clean<T extends object>(list: string, items: T[], schema: z.ZodType<T>): T[] {
+    const kept: T[] = [];
+    const examples: string[] = [];
+    let invalid = 0;
+    for (const raw of items) {
+      const result = schema.safeParse(withoutNulls(raw));
+      if (result.success) {
+        kept.push(result.data);
+      } else {
+        invalid++;
+        if (examples.length < 3) {
+          const issue = result.error.issues[0];
+          examples.push(`${issue?.path.join(".") || "item"}: ${issue?.message ?? "invalid"}`);
+        }
+      }
+    }
+    if (invalid > 0) {
+      dropped += invalid;
+      warnings.push(`Skipped ${invalid} invalid ${list} (${examples.join("; ")})`);
+    }
+    if (kept.length > MAX_ITEMS_PER_LIST) {
+      warnings.push(
+        `Only the first ${MAX_ITEMS_PER_LIST.toLocaleString("en-US")} ${list} were included (${kept.length.toLocaleString("en-US")} found)`,
+      );
+      return kept.slice(0, MAX_ITEMS_PER_LIST);
+    }
+    return kept;
+  }
+
+  const normalized: NormalizedImport = {
+    source: data.source,
+    movies: clean("movies", data.movies, ImportMovieSchema),
+    episodes: clean("episodes", data.episodes, ImportEpisodeSchema),
+    watchlist: clean("watchlist items", data.watchlist, ImportWatchlistItemSchema),
+    ratings: clean("ratings", data.ratings, ImportRatingSchema),
+  };
+
+  return {
+    data: normalized,
+    warnings,
+    diagnostics: { unresolved: countUnresolved(normalized), unsupported: unsupported + dropped },
+  };
+}
+
 // ─── Rating Conversion ──────────────────────────────────────────────
 
 /** Convert a 1-10 rating to Sofa's 1-5 integer scale. */
@@ -304,11 +382,7 @@ export function parseTraktPayload(data: {
     ratings,
   };
 
-  return {
-    data: normalized,
-    warnings,
-    diagnostics: { unresolved: countUnresolved(normalized), unsupported },
-  };
+  return finalizeParseResult(normalized, warnings, unsupported);
 }
 
 // ─── Simkl Parser ───────────────────────────────────────────────────
@@ -500,11 +574,7 @@ export function parseSimklPayload(data: {
     ratings,
   };
 
-  return {
-    data: normalized,
-    warnings,
-    diagnostics: { unresolved: countUnresolved(normalized), unsupported: 0 },
-  };
+  return finalizeParseResult(normalized, warnings);
 }
 
 // ─── Letterboxd Parser ──────────────────────────────────────────────
@@ -668,12 +738,9 @@ export async function parseLetterboxdExport(zipFile: Blob): Promise<ParseResult>
     `Parsed Letterboxd export: ${movies.length} movies, ${watchlist.length} watchlist, ${ratings.length} ratings`,
   );
 
-  // Letterboxd has no IDs — all items need title-based resolution
-  const unresolved = movies.length + watchlist.length + ratings.length;
-
-  return {
-    data: { source: "letterboxd", movies, episodes: [], watchlist, ratings },
+  // Letterboxd has no IDs — all items need title-based resolution (counted by the finalizer)
+  return finalizeParseResult(
+    { source: "letterboxd", movies, episodes: [], watchlist, ratings },
     warnings,
-    diagnostics: { unresolved, unsupported: 0 },
-  };
+  );
 }

packages/core/src/imports/sofa-parser.ts

@@ -2,7 +2,7 @@ import { SofaExportSchema } from "@sofa/api/schemas";
 import { createLogger } from "@sofa/logger";
 
 import type { NormalizedImport, ParseResult } from "./parsers";
-import { countUnresolved } from "./parsers";
+import { finalizeParseResult } from "./parsers";
 
 const log = createLogger("imports");
 
@@ -59,9 +59,5 @@ export function parseSofaExport(data: unknown): ParseResult {
     `Parsed Sofa export: ${normalized.movies.length} movies, ${normalized.episodes.length} episodes, ${normalized.watchlist.length} library, ${normalized.ratings.length} ratings`,
   );
 
-  return {
-    data: normalized,
-    warnings,
-    diagnostics: { unresolved: countUnresolved(normalized), unsupported: 0 },
-  };
+  return finalizeParseResult(normalized, warnings);
 }

packages/core/test/import-parsers.test.ts

@@ -1,6 +1,8 @@
 import AdmZip from "adm-zip";
 import { describe, expect, test } from "vitest";
 
+import { NormalizedImportSchema } from "@sofa/api/schemas";
+
 import {
   type ParseResult,
   parseLetterboxdExport,
@@ -1072,3 +1074,102 @@ describe("parseLetterboxdExport", () => {
     }
   });
 });
+
+// ─── Output always matches the API schema ────────────────────────────
+
+describe("parser output always matches the API schema", () => {
+  test("Trakt nulls become undefined", () => {
+    const result = parseTraktPayload({
+      history: {
+        movies: [
+          {
+            watched_at: "2024-01-15T20:00:00.000Z",
+            movie: { title: "No IDs", year: null, ids: { trakt: 1, tmdb: null, imdb: null } },
+          },
+        ],
+        shows: [
+          {
+            watched_at: "2024-01-15T21:00:00.000Z",
+            show: { title: "Show", year: null, ids: { tmdb: 1396, imdb: null, tvdb: null } },
+            episode: { season: 1, number: 1 },
+          },
+        ],
+      },
+      watchlist: [
+        { type: "movie", movie: { title: "W", year: null, ids: { tmdb: 5, imdb: null } } },
+      ],
+      ratings: [
+        {
+          type: "movie",
+          rating: 8,
+          rated_at: null,
+          movie: { title: "R", year: 2000, ids: { tmdb: 6, imdb: null } },
+        },
+      ],
+    } as never);
+
+    expect(NormalizedImportSchema.safeParse(result.data).success).toBe(true);
+    expect(result.data.movies).toHaveLength(1);
+    expect(result.data.episodes).toHaveLength(1);
+    expect(result.data.watchlist).toHaveLength(1);
+    expect(result.data.ratings).toHaveLength(1);
+    expect(result.data.movies[0].imdbId).toBeUndefined();
+  });
+
+  test("invalid items are dropped with a warning, not fatal", () => {
+    const result = parseTraktPayload({
+      history: {
+        shows: [
+          {
+            watched_at: "2024-01-15T21:00:00.000Z",
+            show: { title: "Show", ids: { tmdb: 1396 } },
+            episode: { season: 0, number: 0 },
+          },
+          {
+            watched_at: "2024-01-15T21:00:00.000Z",
+            show: { title: "Show", ids: { tmdb: 1396 } },
+            episode: { season: 1, number: 2 },
+          },
+        ],
+      },
+    });
+
+    expect(result.data.episodes).toHaveLength(1);
+    expect(NormalizedImportSchema.safeParse(result.data).success).toBe(true);
+    expect(result.warnings.some((w) => w.includes("episodes"))).toBe(true);
+    expect(result.diagnostics?.unsupported).toBeGreaterThanOrEqual(1);
+  });
+
+  test("Simkl nulls become undefined", () => {
+    const result = parseSimklPayload({
+      movies: [
+        {
+          title: "M",
+          year: null,
+          status: "completed",
+          last_watched_at: null,
+          ids: { tmdb: 7, imdb: null },
+        },
+      ],
+    } as never);
+
+    expect(result.data.movies).toHaveLength(1);
+    expect(NormalizedImportSchema.safeParse(result.data).success).toBe(true);
+    expect(result.data.movies[0].watchedAt).toBeUndefined();
+  });
+
+  test("oversized lists are truncated with a warning", () => {
+    const result = parseTraktPayload({
+      history: {
+        movies: Array.from({ length: 50_001 }, (_, i) => ({
+          watched_at: "2024-01-15T20:00:00.000Z",
+          movie: { title: `M${i}`, ids: { tmdb: i + 1 } },
+        })),
+      },
+    });
+
+    expect(result.data.movies).toHaveLength(50_000);
+    expect(NormalizedImportSchema.safeParse(result.data).success).toBe(true);
+    expect(result.warnings.some((w) => w.includes("50,000"))).toBe(true);
+  });
+});