perf(metadata): remember empty TMDB lookups instead of refetching every view

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

apps/server/src/cron.ts

@@ -6,16 +6,14 @@ import { refreshCredits, syncCastProfileThumbHashes } from "@sofa/core/credits";
 import {
   completeCronRun,
   failCronRun,
-  getCastEntryForTitle,
   getLibraryTitleIds,
   getReturningTvShows,
-  getStaleAvailabilityTitles,
   getStaleLibraryTitles,
   getStaleNonLibraryTitlesForRefresh,
   getThumbhashBackfillTitleIds,
   getTitleByIdForCron,
+  getTitleIdsCheckedBefore,
   getTitleIdsWithStaleSeasons,
-  getTitlesWithFreshRecommendations,
   runIsolated,
   startCronRun,
 } from "@sofa/core/cron";
@@ -138,10 +136,10 @@ async function refreshAvailabilityJob() {
   log.debug(`Checking availability for ${libraryIds.length} library titles`);
   const stale = new Date(Date.now() - DAY);
 
-  const { withOffers, withStaleOffers } = getStaleAvailabilityTitles(libraryIds, stale);
+  const staleIds = getTitleIdsCheckedBefore(libraryIds, "availabilityCheckedAt", stale);
 
   await runIsolated(
-    libraryIds.filter((id) => withStaleOffers.has(id) || !withOffers.has(id)),
+    staleIds,
     async (id) => {
       await refreshAvailability(id);
       await Bun.sleep(RATE_LIMIT_MS);
@@ -153,8 +151,7 @@ async function refreshAvailabilityJob() {
 async function refreshRecommendationsJob() {
   const libraryIds = getLibraryTitleIds();
   const stale = new Date(Date.now() - 7 * DAY);
-  const fresh = getTitlesWithFreshRecommendations(libraryIds, stale);
-  const staleIds = libraryIds.filter((id) => !fresh.has(id));
+  const staleIds = getTitleIdsCheckedBefore(libraryIds, "recommendationsCheckedAt", stale);
   log.debug(
     `Refreshing recommendations for ${staleIds.length} of ${libraryIds.length} library titles`,
   );
@@ -239,15 +236,9 @@ async function refreshCreditsJob() {
   log.debug(`Checking credits for ${libraryIds.length} library titles`);
   const stale = new Date(Date.now() - 30 * DAY);
 
-  for (const titleId of libraryIds) {
-    const castEntry = getCastEntryForTitle(titleId);
-
-    const needsRefresh = !castEntry || !castEntry.lastFetchedAt || castEntry.lastFetchedAt < stale;
-
-    if (needsRefresh) {
-      await refreshCredits(titleId);
-      await Bun.sleep(RATE_LIMIT_MS);
-    }
+  for (const titleId of getTitleIdsCheckedBefore(libraryIds, "creditsCheckedAt", stale)) {
+    await refreshCredits(titleId);
+    await Bun.sleep(RATE_LIMIT_MS);
   }
 }
 

packages/core/src/availability.ts

@@ -3,6 +3,7 @@ import {
   getAvailabilityForTitle,
   replaceAvailabilityTransaction,
 } from "@sofa/db/queries/availability";
+import { markEnrichmentChecked } from "@sofa/db/queries/metadata";
 import { getTitleById } from "@sofa/db/queries/title";
 import type { titleAvailability } from "@sofa/db/schema";
 import { createLogger } from "@sofa/logger";
@@ -42,6 +43,7 @@ export async function refreshAvailability(titleId: string) {
   }
 
   replaceAvailabilityTransaction(titleId, "US", allOfferRows);
+  markEnrichmentChecked(titleId, "availability", now);
 
   if (!us) {
     log.debug(`No US providers for title ${titleId}; cleared cached offers`);

packages/core/src/credits.ts

@@ -7,6 +7,7 @@ import {
   getPersonsForTitleCast,
   replaceTitleCastTransaction,
 } from "@sofa/db/queries/credits";
+import { markEnrichmentChecked } from "@sofa/db/queries/metadata";
 import { getTitleById } from "@sofa/db/queries/title";
 import type { titleCast } from "@sofa/db/schema";
 import { createLogger } from "@sofa/logger";
@@ -249,6 +250,7 @@ export async function refreshCredits(titleId: string) {
       if (allCastRows.length > 0) replaceTitleCastTransaction(titleId, allCastRows);
     }
 
+    markEnrichmentChecked(titleId, "credits");
     log.debug(`Credits refreshed for "${title.title}"`);
 
     (async () => {

packages/core/src/cron.ts

@@ -5,6 +5,7 @@ import {
   getStaleTitles,
   getStaleNonLibraryTitles,
   getTitleByIdForCron,
+  getTitleIdsCheckedBefore,
   getTitleIdsWithStaleSeasons,
   getTitlesWithFreshRecommendations,
   getTitlesWithStaleOffers,
@@ -82,6 +83,7 @@ export {
   getCastEntryForTitle,
   getReturningTvShows,
   getTitleByIdForCron,
+  getTitleIdsCheckedBefore,
   getTitleIdsWithStaleSeasons,
   getTitlesWithFreshRecommendations,
 };

packages/core/src/lists.ts

@@ -6,11 +6,17 @@ import {
   getSonarrShows,
   resolveListIntegration,
 } from "@sofa/db/queries/lists";
+import { markEnrichmentChecked } from "@sofa/db/queries/metadata";
 import { createLogger } from "@sofa/logger";
 import { getTvExternalIds } from "@sofa/tmdb/client";
 
+import { mapWithConcurrency } from "./concurrency";
+
 const log = createLogger("lists");
 
+const TVDB_RECHECK_MS = 30 * 24 * 60 * 60 * 1000;
+const TVDB_LOOKUP_CONCURRENCY = 4;
+
 /** Look up a list integration token and return the userId + provider, or null. */
 export function resolveListToken(
   token: string,
@@ -54,23 +60,32 @@ export async function getSonarrList(
   ] as Status[];
   const rows = getSonarrShows(userId, mappedStatuses);
 
-  // Resolve missing TVDB IDs in parallel instead of sequentially
-  const needsResolution = rows.filter((r) => r.tvdbId == null);
+  // Resolve missing TVDB IDs with bounded concurrency; skip shows TMDB recently had no answer for
+  const needsResolution = rows.filter(
+    (r) =>
+      r.tvdbId == null &&
+      (!r.tvdbCheckedAt || Date.now() - r.tvdbCheckedAt.getTime() >= TVDB_RECHECK_MS),
+  );
   if (needsResolution.length > 0) {
-    const resolved = await Promise.all(
-      needsResolution.map(async (row) => {
+    const settled = await mapWithConcurrency(
+      needsResolution,
+      async (row) => {
         try {
           const externalIds = await getTvExternalIds(row.tmdbId);
-          return { row, tvdbId: externalIds.tvdb_id };
+          return { row, tvdbId: externalIds.tvdb_id ?? null, ok: true };
         } catch (err) {
           log.warn(`Failed to resolve TVDB ID for TMDB ${row.tmdbId}:`, err);
-          return { row, tvdbId: null };
+          return { row, tvdbId: null, ok: false };
         }
-      }),
+      },
+      TVDB_LOOKUP_CONCURRENCY,
     );
     // Batch update resolved IDs
     const updates: { titleId: string; tvdbId: number }[] = [];
-    for (const { row, tvdbId } of resolved) {
+    for (const result of settled) {
+      if (result.status !== "fulfilled") continue;
+      const { row, tvdbId, ok } = result.value;
+      if (ok) markEnrichmentChecked(row.id, "tvdb");
       if (tvdbId != null) {
         row.tvdbId = tvdbId;
         updates.push({ titleId: row.id, tvdbId });

packages/core/src/metadata.ts

@@ -21,6 +21,7 @@ import {
   hasRecommendationsForTitle,
   hasSeasonForTitle,
   insertTitleReturning,
+  markEnrichmentChecked,
   nullifyEpisodeThumbHash,
   nullifySeasonThumbHash,
   updateTitleFields,
@@ -65,6 +66,13 @@ import {
 
 const log = createLogger("metadata");
 
+/** After an empty TMDB answer, wait this long before asking again. */
+const EMPTY_ENRICHMENT_RECHECK_MS = 7 * 24 * 60 * 60 * 1000;
+
+function checkedRecently(at: Date | null | undefined): boolean {
+  return !!at && Date.now() - at.getTime() < EMPTY_ENRICHMENT_RECHECK_MS;
+}
+
 const THUMBHASH_CONCURRENCY = 4;
 
 export function updateTitleWithArtInvalidation(
@@ -422,6 +430,7 @@ export async function refreshRecommendations(titleId: string) {
     getRecommendations(title.tmdbId, title.type),
     getSimilar(title.tmdbId, title.type),
   ]);
+  markEnrichmentChecked(titleId, "recommendations");
 
   const recsResults = recs.results ?? [];
   const similarResults = similar.results ?? [];
@@ -606,13 +615,13 @@ async function ensureEnriched(
 ): Promise<boolean> {
   const tasks: Promise<unknown>[] = [];
 
-  if (!existing.hasCast) {
+  if (!existing.hasCast && !checkedRecently(title.creditsCheckedAt)) {
     tasks.push(
       refreshCredits(titleId).catch((err) => log.debug("Credits enrichment failed:", err)),
     );
   }
 
-  if (!existing.hasAvailability) {
+  if (!existing.hasAvailability && !checkedRecently(title.availabilityCheckedAt)) {
     tasks.push(
       refreshAvailability(titleId).catch((err) =>
         log.debug("Availability enrichment failed:", err),
@@ -621,7 +630,7 @@ async function ensureEnriched(
   }
 
   // Recommendations are loaded separately (Suspense), so check here
-  if (!hasRecommendationsForTitle(titleId)) {
+  if (!hasRecommendationsForTitle(titleId) && !checkedRecently(title.recommendationsCheckedAt)) {
     tasks.push(
       refreshRecommendations(titleId).catch((err) =>
         log.debug("Recommendations enrichment failed:", err),
@@ -646,7 +655,7 @@ async function ensureEnriched(
     );
   }
 
-  if (!title.trailerVideoKey) {
+  if (!title.trailerVideoKey && !checkedRecently(title.trailerCheckedAt)) {
     tasks.push(
       refreshTrailer(titleId).catch((err) => log.debug("Trailer enrichment failed:", err)),
     );
@@ -866,6 +875,7 @@ export async function refreshTrailer(titleId: string) {
     const response = await getVideos(title.tmdbId, title.type);
     const key = pickBestTrailer(response.results ?? []);
     updateTrailerKey(titleId, key);
+    markEnrichmentChecked(titleId, "trailer");
     log.debug(`Trailer for "${title.title}": ${key ? `YouTube ${key}` : "none found"}`);
   } catch (err) {
     log.debug(`Failed to fetch trailer for title ${titleId}:`, err);

packages/core/test/availability.test.ts

@@ -1,6 +1,6 @@
 import { beforeEach, describe, expect, test, vi } from "vitest";
 
-import { titleAvailability } from "@sofa/db/schema";
+import { titleAvailability, titles } from "@sofa/db/schema";
 import {
   clearAllTables,
   eq,
@@ -43,4 +43,13 @@ describe("refreshAvailability", () => {
 
     expect(offers).toHaveLength(0);
   });
+
+  test("records availabilityCheckedAt even when TMDB has no providers", async () => {
+    insertTitle({ id: "movie-2", tmdbId: 102, type: "movie", title: "Movie 2" });
+
+    await refreshAvailability("movie-2");
+
+    const row = testDb.select().from(titles).where(eq(titles.id, "movie-2")).get();
+    expect(row?.availabilityCheckedAt).toBeInstanceOf(Date);
+  });
 });

packages/core/test/enrichment-checks.test.ts

@@ -0,0 +1,124 @@
+import { beforeEach, describe, expect, test, vi } from "vitest";
+
+import { getTitleIdsCheckedBefore } from "@sofa/db/queries/cron";
+import { titles } from "@sofa/db/schema";
+import { clearAllTables, eq, insertTitle, testDb } from "@sofa/test/db";
+
+const { getVideos } = vi.hoisted(() => ({
+  getVideos: vi.fn<() => Promise<{ results: unknown[] }>>(async () => ({ results: [] })),
+}));
+
+vi.mock("@sofa/tmdb/client", () => ({
+  getVideos,
+  getRecommendations: async () => ({ results: [] }),
+  getSimilar: async () => ({ results: [] }),
+}));
+
+vi.mock("../src/credits", () => ({
+  refreshCredits: async () => {},
+  getCastForTitle: () => [],
+}));
+
+vi.mock("../src/availability", () => ({
+  refreshAvailability: async () => {},
+}));
+
+vi.mock("../src/image-cache", () => ({
+  cacheEpisodeStills: async () => {},
+  cacheImagesForTitle: async () => {},
+  deleteOrphanedImage: async () => {},
+  imageCacheEnabled: () => false,
+  loadImageBuffer: async () => undefined,
+}));
+
+vi.mock("../src/thumbhash", () => ({
+  generateEpisodeThumbHash: async () => {},
+  generateSeasonThumbHash: async () => {},
+  generateTitleBackdropThumbHash: async () => {},
+  generateTitlePosterThumbHash: async () => {},
+}));
+
+vi.mock("../src/colors", () => ({
+  extractAndStoreColors: async () => {},
+  parseColorPalette: () => null,
+}));
+
+import { getOrFetchTitle, refreshTrailer } from "../src/metadata";
+
+const DAY = 24 * 60 * 60 * 1000;
+
+function getTitle(id: string) {
+  return testDb.select().from(titles).where(eq(titles.id, id)).get();
+}
+
+beforeEach(() => {
+  clearAllTables();
+  getVideos.mockReset();
+  getVideos.mockImplementation(async () => ({ results: [] }));
+});
+
+describe("refreshTrailer", () => {
+  test("stamps trailerCheckedAt even when TMDB has no trailer", async () => {
+    insertTitle({ id: "m1", tmdbId: 1, type: "movie", title: "M" });
+
+    await refreshTrailer("m1");
+
+    const row = getTitle("m1");
+    expect(row?.trailerVideoKey).toBeNull();
+    expect(row?.trailerCheckedAt).toBeInstanceOf(Date);
+  });
+
+  test("does not stamp when the TMDB call fails", async () => {
+    insertTitle({ id: "m1", tmdbId: 1, type: "movie", title: "M" });
+    getVideos.mockRejectedValue(new Error("network"));
+
+    await refreshTrailer("m1");
+
+    expect(getTitle("m1")?.trailerCheckedAt).toBeNull();
+  });
+});
+
+describe("getTitleIdsCheckedBefore", () => {
+  test("returns titles with a null or old stamp only", () => {
+    insertTitle({ id: "t-null", tmdbId: 1, type: "movie" });
+    insertTitle({ id: "t-old", tmdbId: 2, type: "movie" });
+    insertTitle({ id: "t-new", tmdbId: 3, type: "movie" });
+    testDb
+      .update(titles)
+      .set({ creditsCheckedAt: new Date(Date.now() - 40 * DAY) })
+      .where(eq(titles.id, "t-old"))
+      .run();
+    testDb.update(titles).set({ creditsCheckedAt: new Date() }).where(eq(titles.id, "t-new")).run();
+
+    const ids = getTitleIdsCheckedBefore(
+      ["t-null", "t-old", "t-new"],
+      "creditsCheckedAt",
+      new Date(Date.now() - 30 * DAY),
+    );
+
+    expect(ids.toSorted()).toEqual(["t-null", "t-old"]);
+  });
+});
+
+describe("getOrFetchTitle trailer gating", () => {
+  test("skips the trailer lookup until the empty answer is a week old", async () => {
+    insertTitle({ id: "m1", tmdbId: 1, type: "movie", title: "M" });
+    testDb
+      .update(titles)
+      .set({ lastFetchedAt: new Date(), trailerCheckedAt: new Date() })
+      .where(eq(titles.id, "m1"))
+      .run();
+
+    await getOrFetchTitle("m1");
+    expect(getVideos).not.toHaveBeenCalled();
+
+    testDb
+      .update(titles)
+      .set({ trailerCheckedAt: new Date(Date.now() - 8 * DAY) })
+      .where(eq(titles.id, "m1"))
+      .run();
+
+    await getOrFetchTitle("m1");
+    expect(getVideos).toHaveBeenCalled();
+  });
+});

packages/core/test/lists.test.ts

@@ -177,6 +177,18 @@ describe("getSonarrList", () => {
     expect(list).toEqual([]);
   });
 
+  test("remembers a null TVDB ID instead of looking it up again", async () => {
+    insertUser("user-1");
+    insertTitle({ id: "tv1", tmdbId: 300, type: "tv", title: "Show C" });
+    insertStatus("user-1", "tv1", "watchlist");
+
+    mockGetTvExternalIds.mockResolvedValueOnce({ tvdb_id: null, imdb_id: null });
+
+    expect(await getSonarrList("user-1")).toEqual([]);
+    expect(await getSonarrList("user-1")).toEqual([]);
+    expect(mockGetTvExternalIds).toHaveBeenCalledTimes(1);
+  });
+
   test("filters by status", async () => {
     insertUser("user-1");
     insertTitle({

packages/db/drizzle/20261005175931_enrichment_checked_at/migration.sql

@@ -0,0 +1,5 @@
+ALTER TABLE `titles` ADD `creditsCheckedAt` integer;--> statement-breakpoint
+ALTER TABLE `titles` ADD `availabilityCheckedAt` integer;--> statement-breakpoint
+ALTER TABLE `titles` ADD `recommendationsCheckedAt` integer;--> statement-breakpoint
+ALTER TABLE `titles` ADD `trailerCheckedAt` integer;--> statement-breakpoint
+ALTER TABLE `titles` ADD `tvdbCheckedAt` integer;

packages/db/src/queries/cron.ts

@@ -162,3 +162,21 @@ export function getTitlesWithFreshRecommendations(
       .map((r) => r.titleId),
   );
 }
+
+type CheckedAtColumn = "availabilityCheckedAt" | "recommendationsCheckedAt" | "creditsCheckedAt";
+
+/** Ids (from `titleIds`) whose `column` is null or older than `staleDate`. */
+export function getTitleIdsCheckedBefore(
+  titleIds: string[],
+  column: CheckedAtColumn,
+  staleDate: Date,
+): string[] {
+  if (titleIds.length === 0) return [];
+  const col = titles[column];
+  return db
+    .select({ id: titles.id })
+    .from(titles)
+    .where(and(inArray(titles.id, titleIds), or(isNull(col), lt(col, staleDate))))
+    .all()
+    .map((r) => r.id);
+}

packages/db/src/queries/lists.ts

@@ -47,6 +47,7 @@ export function getSonarrShows(
       id: titles.id,
       tmdbId: titles.tmdbId,
       tvdbId: titles.tvdbId,
+      tvdbCheckedAt: titles.tvdbCheckedAt,
       title: titles.title,
     })
     .from(userTitleStatus)

packages/db/src/queries/metadata.ts

@@ -53,6 +53,23 @@ export function updateTrailerKey(titleId: string, key: string | null) {
   db.update(titles).set({ trailerVideoKey: key }).where(eq(titles.id, titleId)).run();
 }
 
+export type EnrichmentKind = "credits" | "availability" | "recommendations" | "trailer" | "tvdb";
+
+/** Record that an enrichment lookup ran (even if TMDB returned nothing). */
+export function markEnrichmentChecked(titleId: string, kind: EnrichmentKind, at = new Date()) {
+  const values: Partial<typeof titles.$inferInsert> =
+    kind === "credits"
+      ? { creditsCheckedAt: at }
+      : kind === "availability"
+        ? { availabilityCheckedAt: at }
+        : kind === "recommendations"
+          ? { recommendationsCheckedAt: at }
+          : kind === "trailer"
+            ? { trailerCheckedAt: at }
+            : { tvdbCheckedAt: at };
+  db.update(titles).set(values).where(eq(titles.id, titleId)).run();
+}
+
 // ─── Genres ──────────────────────────────────────────────────────────
 
 interface TmdbGenreInput {

packages/db/src/schema.ts

@@ -115,6 +115,12 @@ export const titles = sqliteTable(
     runtimeMinutes: int("runtimeMinutes"),
     colorPalette: text("colorPalette"),
     trailerVideoKey: text("trailerVideoKey"),
+    // When each enrichment was last fetched from TMDB, even if TMDB returned nothing.
+    creditsCheckedAt: int("creditsCheckedAt", { mode: "timestamp" }),
+    availabilityCheckedAt: int("availabilityCheckedAt", { mode: "timestamp" }),
+    recommendationsCheckedAt: int("recommendationsCheckedAt", { mode: "timestamp" }),
+    trailerCheckedAt: int("trailerCheckedAt", { mode: "timestamp" }),
+    tvdbCheckedAt: int("tvdbCheckedAt", { mode: "timestamp" }),
     lastFetchedAt: int("lastFetchedAt", { mode: "timestamp" }),
   },
   (table) => [