fix(public-api): paginate Trakt sign-in import and keep it under the response limit

Follow Trakt's pagination headers, slim items to the fields Sofa reads,
keep the most recent plays when the payload would exceed Vercel's 4.5 MB
response limit, surface failed/truncated lists as warnings, and return a
readable fetch_error instead of an oversized response for every provider.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

apps/public-api/src/app.ts

@@ -6,6 +6,7 @@ import { logger } from "hono/logger";
 import { z } from "zod";
 
 import { getImporter, getImporterConfig, ProviderEnum } from "./importers";
+import { authorizedPollBody } from "./poll-response";
 
 const GITHUB_RELEASES_URL = "https://api.github.com/repos/jakejarvis/sofa/releases/latest";
 const VERSION_PREFIX_RE = /^v/;
@@ -192,7 +193,11 @@ app.post(
       // Fetch user data and return it inline
       try {
         const data = await importer.fetchUserData(result.accessToken, config.clientId);
-        return c.json({ status: "authorized", data });
+        const body = authorizedPollBody(data);
+        if (body.status === "fetch_error") {
+          console.warn(`Import payload for ${provider} exceeded the response size limit`);
+        }
+        return c.json(body);
       } catch (e) {
         // Auth succeeded but data fetch failed. Return a distinct status so
         // the client can show a meaningful error instead of polling forever.

apps/public-api/src/importers/trakt.test.ts

@@ -0,0 +1,216 @@
+import { afterEach, describe, expect, test, vi } from "vitest";
+
+import {
+  fitToBudget,
+  jsonBytes,
+  MAX_PAGES_PER_LIST,
+  slimTraktItem,
+  trakt,
+  type TraktPayload,
+} from "./trakt";
+
+type FetchFn = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+
+interface StubReply {
+  body?: unknown;
+  status?: number;
+  pageCount?: number;
+}
+
+/** Stub fetch; `reply` answers by Trakt path and page number. */
+function stubTrakt(reply: (path: string, page: number) => StubReply | undefined) {
+  const fetchMock = vi.fn<FetchFn>(async (input, _init) => {
+    const url = new URL(String(input));
+    const page = Number(url.searchParams.get("page") ?? "1");
+    const r = reply(url.pathname, page) ?? {};
+    const headers: Record<string, string> = {};
+    if (r.pageCount !== undefined) headers["X-Pagination-Page-Count"] = String(r.pageCount);
+    return new Response(JSON.stringify(r.body ?? []), { status: r.status ?? 200, headers });
+  });
+  vi.stubGlobal("fetch", fetchMock);
+  return fetchMock;
+}
+
+function requestsFor(fetchMock: ReturnType<typeof stubTrakt>, path: string) {
+  return fetchMock.mock.calls
+    .map(([input]) => new URL(String(input)))
+    .filter((url) => url.pathname === path);
+}
+
+function movieWatch(i: number) {
+  return {
+    id: i,
+    watched_at: new Date(Date.UTC(2024, 0, 1) + i * 86_400_000).toISOString(),
+    action: "watch",
+    type: "movie",
+    movie: {
+      title: `M${i}`,
+      year: 2000,
+      ids: { trakt: i, slug: `m-${i}`, tmdb: i + 1, imdb: null },
+    },
+  };
+}
+
+function emptyPayload(): TraktPayload {
+  return { history: { movies: [], shows: [] }, watchlist: [], ratings: [], warnings: [] };
+}
+
+interface FetchedPayload {
+  history: { movies: unknown[]; shows: unknown[] };
+  watchlist: unknown[];
+  ratings: unknown[];
+  warnings: string[];
+}
+
+async function fetchData(): Promise<FetchedPayload> {
+  return (await trakt.fetchUserData("token", "client")) as FetchedPayload;
+}
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+});
+
+describe("trakt.fetchUserData", () => {
+  test("follows X-Pagination-Page-Count", async () => {
+    const fetchMock = stubTrakt((path, page) =>
+      path === "/sync/history/movies" ? { body: [movieWatch(page)], pageCount: 3 } : undefined,
+    );
+    const data = await fetchData();
+    expect(data.history.movies).toHaveLength(3);
+    const pages = requestsFor(fetchMock, "/sync/history/movies").map((u) =>
+      u.searchParams.get("page"),
+    );
+    expect(pages).toContain("2");
+    expect(pages).toContain("3");
+  });
+
+  test("treats a response without pagination headers as one page", async () => {
+    const fetchMock = stubTrakt((path) =>
+      path === "/sync/ratings"
+        ? {
+            body: [
+              {
+                rated_at: "2024-01-01T00:00:00.000Z",
+                rating: 8,
+                type: "movie",
+                movie: { title: "A" },
+              },
+              {
+                rated_at: "2024-01-02T00:00:00.000Z",
+                rating: 9,
+                type: "movie",
+                movie: { title: "B" },
+              },
+            ],
+          }
+        : undefined,
+    );
+    const data = await fetchData();
+    expect(data.ratings).toHaveLength(2);
+    expect(requestsFor(fetchMock, "/sync/ratings")).toHaveLength(1);
+  });
+
+  test("warns instead of silently dropping a failed list", async () => {
+    stubTrakt((path) => (path === "/sync/watchlist" ? { status: 500 } : undefined));
+    const data = await fetchData();
+    expect(data.watchlist).toEqual([]);
+    expect(data.warnings).toHaveLength(1);
+    expect(data.warnings[0]).toContain("watchlist");
+  });
+
+  test("throws when every list fails", async () => {
+    stubTrakt(() => ({ status: 500 }));
+    await expect(trakt.fetchUserData("token", "client")).rejects.toThrow("every list");
+  });
+
+  test("keeps a partial list when a later page fails", async () => {
+    stubTrakt((path, page) => {
+      if (path !== "/sync/history/shows") return undefined;
+      if (page === 1) {
+        return {
+          body: [{ watched_at: "2024-01-01T00:00:00.000Z", type: "episode" }],
+          pageCount: 3,
+        };
+      }
+      if (page === 2) return { status: 500 };
+      return { body: [{ watched_at: "2024-01-03T00:00:00.000Z", type: "episode" }] };
+    });
+    const data = await fetchData();
+    expect(data.history.shows).toHaveLength(1);
+    expect(data.warnings.some((w) => w.includes("page 2 of 3"))).toBe(true);
+  });
+
+  test("stops at MAX_PAGES_PER_LIST", async () => {
+    stubTrakt((path, page) =>
+      path === "/sync/history/movies" ? { body: [movieWatch(page)], pageCount: 1000 } : undefined,
+    );
+    const data = await fetchData();
+    expect(data.history.movies).toHaveLength(MAX_PAGES_PER_LIST);
+    expect(data.warnings.some((w) => w.includes(`first ${MAX_PAGES_PER_LIST} pages`))).toBe(true);
+  });
+});
+
+describe("slimTraktItem", () => {
+  test("slims items to the fields Sofa reads", () => {
+    expect(
+      slimTraktItem({
+        id: 1,
+        watched_at: "2024-01-01T00:00:00.000Z",
+        action: "watch",
+        type: "movie",
+        movie: { title: "M", year: null, ids: { trakt: 9, slug: "m", imdb: null, tmdb: 5 } },
+      }),
+    ).toEqual({
+      watched_at: "2024-01-01T00:00:00.000Z",
+      type: "movie",
+      movie: { title: "M", ids: { tmdb: 5 } },
+    });
+
+    const episodePlay = slimTraktItem({
+      watched_at: "2024-01-01T00:00:00.000Z",
+      type: "episode",
+      episode: { season: 1, number: 2, title: "x", ids: { trakt: 3 } },
+      show: { title: "S", year: 2020, ids: { tvdb: 7 } },
+    });
+    expect(episodePlay).toMatchObject({ episode: { season: 1, number: 2 } });
+    expect(episodePlay?.episode).toEqual({ season: 1, number: 2 });
+  });
+
+  test("returns null for non-object input", () => {
+    expect(slimTraktItem(null)).toBeNull();
+    expect(slimTraktItem("x")).toBeNull();
+  });
+});
+
+describe("fitToBudget", () => {
+  test("keeps the most recent plays", () => {
+    const payload = emptyPayload();
+    for (let i = 0; i < 60; i++) {
+      payload.history.movies.push(slimTraktItem(movieWatch(i)));
+    }
+    const result = fitToBudget(payload, 3_000);
+    expect(jsonBytes(result)).toBeLessThanOrEqual(3_000);
+    const kept = result.history.movies as { watched_at: string }[];
+    expect(kept.length).toBeGreaterThan(0);
+    expect(kept.length).toBeLessThan(60);
+
+    const keptDates = new Set(kept.map((m) => m.watched_at));
+    const dropped = (payload.history.movies as { watched_at: string }[]).filter(
+      (m) => !keptDates.has(m.watched_at),
+    );
+    const oldestKept = [...keptDates].sort()[0] as string;
+    for (const d of dropped) {
+      expect(oldestKept > d.watched_at).toBe(true);
+    }
+    expect(result.warnings).toHaveLength(1);
+    expect(result.warnings[0]).toContain("60 plays");
+  });
+
+  test("leaves a small payload untouched", () => {
+    const payload = emptyPayload();
+    payload.history.movies.push(slimTraktItem(movieWatch(1)));
+    const result = fitToBudget(payload, 1_000_000);
+    expect(result).toEqual(payload);
+    expect(result.warnings).toEqual([]);
+  });
+});

apps/public-api/src/importers/trakt.ts

@@ -2,6 +2,19 @@ import type { DeviceCodeResponse, ImportProvider, PollResult } from "./types";
 
 const API_BASE = "https://api.trakt.tv";
 
+/** Items requested per page. Trakt may cap this; we always follow X-Pagination-Page-Count. */
+const PAGE_LIMIT = 1000;
+/** Hard stop per list (keeps us well inside Trakt's 1,000 GETs / 5 min and Vercel's max duration). */
+export const MAX_PAGES_PER_LIST = 100;
+const PAGE_CONCURRENCY = 4;
+const REQUEST_TIMEOUT_MS = 20_000;
+/**
+ * Target size of the payload returned to the browser. Vercel Functions reject
+ * response bodies over 4.5 MB (413 FUNCTION_PAYLOAD_TOO_LARGE); the poll route
+ * adds a small envelope, so stay well below.
+ */
+export const TRAKT_PAYLOAD_BUDGET_BYTES = 4_000_000;
+
 function traktHeaders(clientId: string, token?: string): Record<string, string> {
   const headers: Record<string, string> = {
     "Content-Type": "application/json",
@@ -12,6 +25,200 @@ function traktHeaders(clientId: string, token?: string): Record<string, string>
   return headers;
 }
 
+export function jsonBytes(value: unknown): number {
+  return new TextEncoder().encode(JSON.stringify(value)).byteLength;
+}
+
+// ─── Slimming ────────────────────────────────────────────────
+
+type Obj = Record<string, unknown>;
+
+function isObj(value: unknown): value is Obj {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function pickString(out: Obj, src: Obj, key: string): void {
+  if (typeof src[key] === "string") out[key] = src[key];
+}
+
+function pickNumber(out: Obj, src: Obj, key: string): void {
+  if (typeof src[key] === "number") out[key] = src[key];
+}
+
+function slimMedia(src: unknown): Obj | undefined {
+  if (!isObj(src)) return undefined;
+  const out: Obj = {};
+  pickString(out, src, "title");
+  pickNumber(out, src, "year");
+  if (isObj(src.ids)) {
+    const ids: Obj = {};
+    pickNumber(ids, src.ids, "tmdb");
+    pickString(ids, src.ids, "imdb");
+    pickNumber(ids, src.ids, "tvdb");
+    if (Object.keys(ids).length > 0) out.ids = ids;
+  }
+  return out;
+}
+
+/**
+ * Keep only the fields every released Sofa `parseTraktPayload` reads, dropping
+ * nulls and Trakt-internal data. Returns null for non-object input.
+ */
+export function slimTraktItem(item: unknown): Obj | null {
+  if (!isObj(item)) return null;
+  const out: Obj = {};
+  pickString(out, item, "type");
+  pickString(out, item, "watched_at");
+  pickString(out, item, "listed_at");
+  pickString(out, item, "rated_at");
+  pickNumber(out, item, "rating");
+  for (const key of ["movie", "show"] as const) {
+    const media = slimMedia(item[key]);
+    if (media) out[key] = media;
+  }
+  if (isObj(item.episode)) {
+    const episode: Obj = {};
+    pickNumber(episode, item.episode, "season");
+    pickNumber(episode, item.episode, "number");
+    out.episode = episode;
+  }
+  return out;
+}
+
+// ─── Pagination ──────────────────────────────────────────────
+
+interface ListResult {
+  items: Obj[];
+  ok: boolean;
+  warning?: string;
+}
+
+async function fetchPage(url: string, headers: Record<string, string>): Promise<Response | null> {
+  try {
+    return await fetch(url, { headers, signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) });
+  } catch {
+    return null;
+  }
+}
+
+async function readArray(res: Response): Promise<unknown[] | null> {
+  try {
+    const body: unknown = await res.json();
+    return Array.isArray(body) ? body : null;
+  } catch {
+    return null;
+  }
+}
+
+async function fetchAllPages(
+  path: string,
+  label: string,
+  headers: Record<string, string>,
+): Promise<ListResult> {
+  const pageUrl = (page: number) =>
+    `${API_BASE}${path}${path.includes("?") ? "&" : "?"}page=${page}&limit=${PAGE_LIMIT}`;
+
+  const first = await fetchPage(pageUrl(1), headers);
+  if (!first?.ok) {
+    const status = first?.status;
+    return {
+      items: [],
+      ok: false,
+      warning: `Couldn't fetch your Trakt ${label}${status ? ` (HTTP ${status})` : ""}; it was skipped.`,
+    };
+  }
+
+  const raw: unknown[] = (await readArray(first)) ?? [];
+  const headerCount = Number(first.headers.get("X-Pagination-Page-Count"));
+  const pageCount = Number.isFinite(headerCount) && headerCount >= 1 ? headerCount : 1;
+  const lastPage = Math.min(pageCount, MAX_PAGES_PER_LIST);
+
+  let warning: string | undefined;
+  for (let start = 2; start <= lastPage && !warning; start += PAGE_CONCURRENCY) {
+    const pages = Array.from(
+      { length: Math.min(PAGE_CONCURRENCY, lastPage - start + 1) },
+      (_, i) => start + i,
+    );
+    const results = await Promise.all(
+      pages.map(async (page) => {
+        const res = await fetchPage(pageUrl(page), headers);
+        return res?.ok ? await readArray(res) : null;
+      }),
+    );
+    for (const [i, items] of results.entries()) {
+      if (!items) {
+        warning = `Only part of your Trakt ${label} could be fetched (stopped at page ${pages[i]} of ${pageCount}).`;
+        break;
+      }
+      raw.push(...items);
+    }
+  }
+
+  if (!warning && pageCount > MAX_PAGES_PER_LIST) {
+    warning = `Only the first ${MAX_PAGES_PER_LIST} pages of your Trakt ${label} were fetched.`;
+  }
+
+  const items = raw.map(slimTraktItem).filter((i): i is Obj => i !== null);
+  return { items, ok: true, warning };
+}
+
+// ─── Size budget ─────────────────────────────────────────────
+
+export interface TraktPayload {
+  history: { movies: unknown[]; shows: unknown[] };
+  watchlist: unknown[];
+  ratings: unknown[];
+  warnings: string[];
+}
+
+function watchedAtOf(item: unknown): string {
+  return isObj(item) && typeof item.watched_at === "string" ? item.watched_at : "";
+}
+
+/**
+ * If the payload exceeds the budget, keep the most recent plays (watchlist and
+ * ratings are always kept) and add a warning pointing at the export upload.
+ */
+export function fitToBudget(payload: TraktPayload, budgetBytes: number): TraktPayload {
+  if (jsonBytes(payload) <= budgetBytes) return payload;
+
+  const tagged = [
+    ...payload.history.movies.map((item) => ({ item, kind: "movies" as const })),
+    ...payload.history.shows.map((item) => ({ item, kind: "shows" as const })),
+  ];
+  // Most recent first; items without watched_at ("") sort last.
+  tagged.sort((a, b) => {
+    const wa = watchedAtOf(a.item);
+    const wb = watchedAtOf(b.item);
+    if (wa === wb) return 0;
+    if (!wa) return 1;
+    if (!wb) return -1;
+    return wa < wb ? 1 : -1;
+  });
+
+  const result: TraktPayload = {
+    history: { movies: [], shows: [] },
+    watchlist: payload.watchlist,
+    ratings: payload.ratings,
+    warnings: [...payload.warnings],
+  };
+  const limit = budgetBytes - 1_000;
+  let size = jsonBytes(result);
+  let kept = 0;
+  for (const { item, kind } of tagged) {
+    const cost = jsonBytes(item) + 1;
+    if (size + cost > limit) break;
+    result.history[kind].push(item);
+    size += cost;
+    kept++;
+  }
+
+  result.warnings.push(
+    `Your Trakt history has ${tagged.length.toLocaleString("en-US")} plays, but only the most recent ${kept.toLocaleString("en-US")} fit in a sign-in import. To import all of it, upload your Trakt data export instead (Settings → Import → Trakt).`,
+  );
+  return result;
+}
+
 // ─── Provider ────────────────────────────────────────────────
 
 export const trakt: ImportProvider = {
@@ -56,35 +263,30 @@ export const trakt: ImportProvider = {
   async fetchUserData(accessToken, clientId): Promise<unknown> {
     const headers = traktHeaders(clientId, accessToken);
 
-    // Fetch all data in parallel
-    const [moviesRes, showsRes, watchlistRes, ratingsRes] = await Promise.all([
-      fetch(`${API_BASE}/sync/history/movies?limit=10000`, { headers }),
-      fetch(`${API_BASE}/sync/history/shows?limit=10000`, { headers }),
-      fetch(`${API_BASE}/sync/watchlist?extended=metadata&limit=10000`, {
-        headers,
-      }),
-      fetch(`${API_BASE}/sync/ratings`, { headers }),
+    const [movies, shows, watchlist, ratings] = await Promise.all([
+      fetchAllPages("/sync/history/movies", "movie history", headers),
+      fetchAllPages("/sync/history/shows", "episode history", headers),
+      fetchAllPages("/sync/watchlist", "watchlist", headers),
+      fetchAllPages("/sync/ratings", "ratings", headers),
     ]);
 
-    // If all endpoints failed, throw so the caller gets a clear error
-    if (!moviesRes.ok && !showsRes.ok && !watchlistRes.ok && !ratingsRes.ok) {
-      throw new Error(
-        `Trakt API returned errors: movies ${moviesRes.status}, shows ${showsRes.status}, watchlist ${watchlistRes.status}, ratings ${ratingsRes.status}`,
-      );
+    const lists = [movies, shows, watchlist, ratings];
+    const warnings = lists.flatMap((l) => (l.warning ? [l.warning] : []));
+
+    // If every list failed, throw so the caller gets a clear error
+    if (lists.every((l) => !l.ok)) {
+      throw new Error(`Trakt API returned errors for every list: ${warnings.join(" ")}`);
     }
 
-    const [moviesData, showsData, watchlistData, ratingsData] = await Promise.all([
-      moviesRes.ok ? moviesRes.json() : [],
-      showsRes.ok ? showsRes.json() : [],
-      watchlistRes.ok ? watchlistRes.json() : [],
-      ratingsRes.ok ? ratingsRes.json() : [],
-    ]);
-
-    // Return raw aggregated API response — parsing happens on the self-hosted server
-    return {
-      history: { movies: moviesData, shows: showsData },
-      watchlist: watchlistData,
-      ratings: ratingsData,
-    };
+    // Slimmed, aggregated API response — parsing happens on the self-hosted server
+    return fitToBudget(
+      {
+        history: { movies: movies.items, shows: shows.items },
+        watchlist: watchlist.items,
+        ratings: ratings.items,
+        warnings,
+      },
+      TRAKT_PAYLOAD_BUDGET_BYTES,
+    );
   },
 };

apps/public-api/src/poll-response.test.ts

@@ -0,0 +1,17 @@
+import { describe, expect, test } from "vitest";
+
+import { authorizedPollBody, MAX_POLL_RESPONSE_BYTES } from "./poll-response";
+
+describe("authorizedPollBody", () => {
+  test("passes small payloads through", () => {
+    expect(authorizedPollBody({ a: 1 })).toEqual({ status: "authorized", data: { a: 1 } });
+  });
+
+  test("turns an oversized payload into a fetch_error", () => {
+    const body = authorizedPollBody({ blob: "x".repeat(MAX_POLL_RESPONSE_BYTES) });
+    expect(body).toMatchObject({
+      status: "fetch_error",
+      error: expect.stringContaining("export file"),
+    });
+  });
+});

apps/public-api/src/poll-response.ts

@@ -0,0 +1,22 @@
+/**
+ * Vercel Functions reject response bodies over 4.5 MB with 413 FUNCTION_PAYLOAD_TOO_LARGE.
+ * The web client silently retries non-2xx polls with an already-used device code and ends
+ * up showing "Device code expired", so return a readable fetch_error instead.
+ */
+export const MAX_POLL_RESPONSE_BYTES = 4_400_000;
+
+export type AuthorizedPollBody =
+  | { status: "authorized"; data: unknown }
+  | { status: "fetch_error"; error: string };
+
+export function authorizedPollBody(data: unknown): AuthorizedPollBody {
+  const body = { status: "authorized" as const, data };
+  const bytes = new TextEncoder().encode(JSON.stringify(body)).byteLength;
+  if (bytes > MAX_POLL_RESPONSE_BYTES) {
+    return {
+      status: "fetch_error",
+      error: "Your library is too large to import by signing in. Upload your export file instead.",
+    };
+  }
+  return body;
+}

packages/core/src/imports/parsers.ts

@@ -277,8 +277,15 @@ export function parseTraktPayload(data: {
   history?: { movies?: TraktHistoryMovie[]; shows?: TraktHistoryEpisode[] };
   watchlist?: TraktWatchlistItem[];
   ratings?: TraktRatingItem[];
+  // Notes from the OAuth proxy (e.g. "history truncated"); ignored by older servers
+  warnings?: unknown;
 }): ParseResult {
   const warnings: string[] = [];
+  if (Array.isArray(data.warnings)) {
+    for (const warning of data.warnings) {
+      if (typeof warning === "string") warnings.push(warning);
+    }
+  }
   const movies: ImportMovie[] = [];
   const episodes: ImportEpisode[] = [];
   const watchlist: ImportWatchlistItem[] = [];

packages/core/test/import-parsers.test.ts

@@ -24,6 +24,15 @@ function createLetterboxdZip(files: Record<string, string>): Blob {
 // ─── parseTraktPayload ───────────────────────────────────────────────
 
 describe("parseTraktPayload", () => {
+  test("passes through warnings from the OAuth proxy", () => {
+    const result = parseTraktPayload({
+      history: { movies: [] },
+      warnings: ["Only some history fit", 42],
+    } as never);
+    expect(result.warnings).toContain("Only some history fit");
+    expect(result.warnings.every((w) => typeof w === "string")).toBe(true);
+  });
+
   test("parses valid data with movies, episodes, watchlist, and ratings", () => {
     const result = parseTraktPayload({
       history: {