perf(cron): refresh ended titles less often and only recent seasons of returning shows

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
jakeandClaude Sonnet 5.5 committed 2026-10-05 14:13:28 -04:00
1 parent 66a5e80544
commit 6576f21235
7 files changed
+224 -18

No files matched your search

+18 -9
View File
@@ -8,7 +8,8 @@ import {
failCronRun,
getLibraryTitleIds,
getReturningTvShows,
getStaleLibraryTitles,
getLibraryTitlesDueForRefresh,
getStoredSeasonNumbers,
getStaleNonLibraryTitlesForRefresh,
getThumbhashBackfillTitleIds,
getTitleByIdForCron,
@@ -28,6 +29,7 @@ import {
refreshRecommendations,
refreshTitle,
refreshTvChildren,
seasonsToRefresh,
syncTvChildArt,
} from "@sofa/core/metadata";
import { getSetting } from "@sofa/core/settings";
@@ -100,14 +102,13 @@ export async function triggerJob(name: string): Promise<boolean> {
async function nightlyRefreshLibrary() {
const libraryIds = getLibraryTitleIds();
log.debug(`Checking ${libraryIds.length} library titles for staleness`);
const libraryStale = new Date(Date.now() - 7 * DAY);
const nonLibraryStale = new Date(Date.now() - 30 * DAY);
// Library titles: 7 days
const staleLibrary = getStaleLibraryTitles(libraryIds, libraryStale);
// Library titles: 7 days, or 60 days for settled (ended/canceled/old) titles
const dueLibraryIds = getLibraryTitlesDueForRefresh(libraryIds);
await runIsolated(
staleLibrary.map((t) => t.id),
dueLibraryIds,
async (id) => {
await refreshTitle(id);
await Bun.sleep(RATE_LIMIT_MS);
@@ -169,7 +170,8 @@ async function refreshRecommendationsJob() {
async function refreshTvChildrenJob() {
const stale = new Date(Date.now() - 7 * DAY);
const tvShows = getReturningTvShows();
const libraryIds = new Set(getLibraryTitleIds());
const tvShows = getReturningTvShows().filter((s) => libraryIds.has(s.id));
log.debug(`Checking ${tvShows.length} returning TV shows for stale episodes`);
@@ -180,7 +182,12 @@ async function refreshTvChildrenJob() {
tvShows.filter((s) => titlesWithStaleSeasons.has(s.id)),
async (show) => {
const details = await getTvDetails(show.tmdbId);
await refreshTvChildren(show.id, show.tmdbId, details.number_of_seasons);
const onlySeasons = seasonsToRefresh(
details.status ?? null,
details.number_of_seasons,
getStoredSeasonNumbers(show.id),
);
await refreshTvChildren(show.id, show.tmdbId, details.number_of_seasons, { onlySeasons });
await syncTvChildArt(show.id, { warmCache: true });
await Bun.sleep(RATE_LIMIT_MS);
},
@@ -193,18 +200,20 @@ async function cacheImagesJob() {
log.debug(`Caching images for ${titleIds.length} titles needing art backfill`);
for (const titleId of titleIds) {
let downloads = 0;
try {
const title = getTitleByIdForCron(titleId);
if (!title) continue;
// Phase 1: warm the image cache so thumbhash generation can read from disk
if (imageCacheEnabled()) {
await Promise.all([
const counts = await Promise.all([
cacheImagesForTitle(titleId),
cacheEpisodeStills(titleId),
cacheProviderLogos(titleId),
cacheProfilePhotos(titleId),
]);
downloads = counts.reduce((a, b) => a + b, 0);
}
// Phase 2: generate thumbhashes (reads from warm cache, no duplicate downloads)
@@ -227,7 +236,7 @@ async function cacheImagesJob() {
} catch (err) {
log.warn(`Failed to cache images for title ${titleId}:`, err);
}
await Bun.sleep(RATE_LIMIT_MS);
if (downloads > 0) await Bun.sleep(RATE_LIMIT_MS);
}
}
+36
View File
@@ -1,6 +1,7 @@
import {
getCastEntryForTitle,
getLibraryTitleIds as queryGetLibraryTitleIds,
getRefreshCandidates,
getReturningTvShows,
getStaleTitles,
getStaleNonLibraryTitles,
@@ -14,6 +15,7 @@ import {
updateCronRunError,
updateCronRunSuccess,
} from "@sofa/db/queries/cron";
import { getSeasonsForTitle } from "@sofa/db/queries/metadata";
/**
* Run `fn` for each item in order, isolating failures so one bad item (e.g. a
@@ -69,6 +71,40 @@ export function getStaleLibraryTitles(libraryIds: string[], staleDate: Date) {
return getStaleTitles(libraryIds, staleDate);
}
const DAY_MS = 24 * 60 * 60 * 1000;
const SETTLED_TV_STATUSES = new Set(["Ended", "Canceled"]);
/** How long a library title's metadata stays fresh. Settled titles rarely change on TMDB. */
export function libraryRefreshIntervalMs(
title: { type: string; status: string | null; releaseDate: string | null },
now = new Date(),
): number {
if (title.type === "tv" && title.status && SETTLED_TV_STATUSES.has(title.status)) {
return 60 * DAY_MS;
}
if (title.type === "movie" && title.releaseDate) {
const released = Date.parse(`${title.releaseDate}T00:00:00Z`);
if (Number.isFinite(released) && now.getTime() - released > 365 * DAY_MS) return 60 * DAY_MS;
}
return 7 * DAY_MS;
}
/** Library title ids whose metadata is due for a refresh. */
export function getLibraryTitlesDueForRefresh(libraryIds: string[], now = new Date()): string[] {
return getRefreshCandidates(libraryIds)
.filter(
(t) =>
!t.lastFetchedAt ||
now.getTime() - t.lastFetchedAt.getTime() >= libraryRefreshIntervalMs(t, now),
)
.map((t) => t.id);
}
/** Season numbers Sofa has stored for a title. */
export function getStoredSeasonNumbers(titleId: string): number[] {
return getSeasonsForTitle(titleId).map((s) => s.seasonNumber);
}
export function getStaleNonLibraryTitlesForRefresh(staleDate: Date, limit: number) {
return getStaleNonLibraryTitles(staleDate, limit);
}
+9 -5
View File
@@ -167,9 +167,9 @@ export async function loadImageBuffer(
return remote?.buffer ?? null;
}
export async function cacheImagesForTitle(titleId: string) {
export async function cacheImagesForTitle(titleId: string): Promise<number> {
const title = getTitleWithPaths(titleId);
if (!title) return;
if (!title) return 0;
// Collect all candidate images, then check cache in parallel
const candidates: { imgPath: string; category: ImageCategory }[] = [];
@@ -200,9 +200,10 @@ export async function cacheImagesForTitle(titleId: string) {
}
await Promise.allSettled(tasks);
return tasks.length;
}
export async function cacheEpisodeStills(titleId: string) {
export async function cacheEpisodeStills(titleId: string): Promise<number> {
const allEps = getEpisodeStillsForTitle(titleId);
const epsWithStills = allEps.filter(
@@ -221,9 +222,10 @@ export async function cacheEpisodeStills(titleId: string) {
(c) => downloadAndCacheImage(c.stillPath, "stills"),
STILL_DOWNLOAD_CONCURRENCY,
);
return uncached.length;
}
export async function cacheProviderLogos(titleId: string) {
export async function cacheProviderLogos(titleId: string): Promise<number> {
const offers = getAvailabilityLogosForTitle(titleId);
// Deduplicate and parallel cache checks
@@ -245,9 +247,10 @@ export async function cacheProviderLogos(titleId: string) {
.map((c) => downloadAndCacheImage(c.logoPath, "logos"));
await Promise.allSettled(tasks);
return tasks.length;
}
export async function cacheProfilePhotos(titleId: string) {
export async function cacheProfilePhotos(titleId: string): Promise<number> {
const castRows = getCastProfilePathsForTitle(titleId);
// Deduplicate and parallel cache checks
@@ -272,4 +275,5 @@ export async function cacheProfilePhotos(titleId: string) {
log.debug(`Caching ${tasks.length} profile photos for title ${titleId}`);
}
await Promise.allSettled(tasks);
return tasks.length;
}
+37 -3
View File
@@ -334,7 +334,12 @@ export async function refreshTitle(titleId: string) {
lastFetchedAt: now,
});
upsertGenresTransaction(titleId, show.genres ?? []);
await refreshTvChildren(titleId, title.tmdbId, show.number_of_seasons);
const onlySeasons = seasonsToRefresh(
show.status ?? null,
show.number_of_seasons,
getSeasonsForTitle(titleId).map((s) => s.seasonNumber),
);
await refreshTvChildren(titleId, title.tmdbId, show.number_of_seasons, { onlySeasons });
}
const updated = getTitleById(titleId);
@@ -351,9 +356,38 @@ export async function refreshTitle(titleId: string) {
return updated;
}
export async function refreshTvChildren(titleId: string, tmdbId: number, numberOfSeasons: number) {
const RETURNING_TV_STATUSES = new Set(["Returning Series", "In Production"]);
/**
* Seasons worth re-fetching for a show we already have: for returning shows, the two latest
* plus any season never stored; otherwise every season.
*/
export function seasonsToRefresh(
status: string | null,
numberOfSeasons: number,
storedSeasonNumbers: number[],
): number[] | undefined {
if (!status || !RETURNING_TV_STATUSES.has(status) || storedSeasonNumbers.length === 0) {
return undefined; // all seasons
}
const stored = new Set(storedSeasonNumbers);
const result: number[] = [];
for (let sn = 1; sn <= numberOfSeasons; sn++) {
if (sn >= numberOfSeasons - 1 || !stored.has(sn)) result.push(sn);
}
return result;
}
export async function refreshTvChildren(
titleId: string,
tmdbId: number,
numberOfSeasons: number,
options?: { onlySeasons?: number[] },
) {
// Fetch seasons with limited concurrency to stay within TMDB's 40 req/s limit
const seasonNumbers = Array.from({ length: numberOfSeasons }, (_, i) => i + 1);
const allSeasons = Array.from({ length: numberOfSeasons }, (_, i) => i + 1);
const only = options?.onlySeasons;
const seasonNumbers = only ? allSeasons.filter((sn) => only.includes(sn)) : allSeasons;
const fetched = await mapWithConcurrency(
seasonNumbers,
(sn) => getTvSeasonDetails(tmdbId, sn),
+73
View File
@@ -6,7 +6,9 @@ import { clearAllTables, eq, insertTitle, testDb } from "@sofa/test/db";
import {
completeCronRun,
failCronRun,
getLibraryTitlesDueForRefresh,
getStaleLibraryTitles,
libraryRefreshIntervalMs,
runIsolated,
startCronRun,
} from "../src/cron";
@@ -125,3 +127,74 @@ describe("getStaleLibraryTitles", () => {
expect(getStaleLibraryTitles(["t-shell"], staleDate)).toEqual([{ id: "t-shell" }]);
});
});
describe("libraryRefreshIntervalMs", () => {
const DAY = 24 * 60 * 60 * 1000;
const now = new Date("2026-06-01T00:00:00Z");
test("ended TV shows refresh every 60 days", () => {
expect(libraryRefreshIntervalMs({ type: "tv", status: "Ended", releaseDate: null }, now)).toBe(
60 * DAY,
);
});
test("returning TV shows refresh every 7 days", () => {
expect(
libraryRefreshIntervalMs({ type: "tv", status: "Returning Series", releaseDate: null }, now),
).toBe(7 * DAY);
});
test("movies released over a year ago refresh every 60 days", () => {
expect(
libraryRefreshIntervalMs(
{ type: "movie", status: "Released", releaseDate: "2001-01-01" },
now,
),
).toBe(60 * DAY);
});
test("recent movies refresh every 7 days", () => {
expect(
libraryRefreshIntervalMs(
{ type: "movie", status: "Released", releaseDate: "2026-05-02" },
now,
),
).toBe(7 * DAY);
});
test("movies without a release date refresh every 7 days", () => {
expect(
libraryRefreshIntervalMs({ type: "movie", status: "Released", releaseDate: null }, now),
).toBe(7 * DAY);
});
});
describe("getLibraryTitlesDueForRefresh", () => {
const now = new Date("2026-06-01T00:00:00Z");
const tenDaysAgo = new Date(now.getTime() - 10 * 24 * 60 * 60 * 1000);
test("skips an ended show fetched 10 days ago", () => {
insertTitle({ id: "t-ended", tmdbId: 1, type: "tv" });
testDb
.update(titles)
.set({ status: "Ended", lastFetchedAt: tenDaysAgo })
.where(eq(titles.id, "t-ended"))
.run();
expect(getLibraryTitlesDueForRefresh(["t-ended"], now)).toEqual([]);
});
test("includes a returning show fetched 10 days ago", () => {
insertTitle({ id: "t-returning", tmdbId: 2, type: "tv" });
testDb
.update(titles)
.set({ status: "Returning Series", lastFetchedAt: tenDaysAgo })
.where(eq(titles.id, "t-returning"))
.run();
expect(getLibraryTitlesDueForRefresh(["t-returning"], now)).toEqual(["t-returning"]);
});
test("includes a never-fetched shell title", () => {
insertTitle({ id: "t-shell", tmdbId: 3 });
expect(getLibraryTitlesDueForRefresh(["t-shell"], now)).toEqual(["t-shell"]);
});
});
+36 -1
View File
@@ -56,7 +56,11 @@ vi.mock("@sofa/tmdb/client", () => ({
getVideos: async () => ({ results: [] }),
}));
import { refreshTvChildren, updateTitleWithArtInvalidation } from "../src/metadata";
import {
refreshTvChildren,
seasonsToRefresh,
updateTitleWithArtInvalidation,
} from "../src/metadata";
beforeEach(() => {
clearAllTables();
@@ -66,7 +70,38 @@ beforeEach(() => {
}));
});
describe("seasonsToRefresh", () => {
test("returning shows refresh only the two latest seasons", () => {
expect(seasonsToRefresh("Returning Series", 6, [1, 2, 3, 4, 5, 6])).toEqual([5, 6]);
});
test("returning shows also refresh seasons never stored", () => {
expect(seasonsToRefresh("Returning Series", 6, [1, 2, 3])).toEqual([4, 5, 6]);
});
test("ended shows refresh every season", () => {
expect(seasonsToRefresh("Ended", 6, [1, 2, 3, 4, 5, 6])).toBeUndefined();
});
test("shows with no stored seasons refresh every season", () => {
expect(seasonsToRefresh("Returning Series", 6, [])).toBeUndefined();
});
test("single-season returning show refreshes that season", () => {
expect(seasonsToRefresh("Returning Series", 1, [1])).toEqual([1]);
});
});
describe("refreshTvChildren", () => {
test("onlySeasons limits which seasons are fetched", async () => {
insertTitle({ id: "tv-1", tmdbId: 10, type: "tv", title: "Show" });
await refreshTvChildren("tv-1", 10, 3, { onlySeasons: [3] });
expect(mockGetTvSeasonDetails).toHaveBeenCalledTimes(1);
expect(mockGetTvSeasonDetails.mock.calls[0] as unknown[]).toEqual([10, 3]);
});
test("clears stale season and episode thumbhashes when artwork paths change", async () => {
insertTitle({ id: "tv-1", tmdbId: 10, type: "tv", title: "Show" });
testDb
+15
View File
@@ -180,3 +180,18 @@ export function getTitleIdsCheckedBefore(
.all()
.map((r) => r.id);
}
export function getRefreshCandidates(titleIds: string[]) {
if (titleIds.length === 0) return [];
return db
.select({
id: titles.id,
type: titles.type,
status: titles.status,
releaseDate: titles.releaseDate,
lastFetchedAt: titles.lastFetchedAt,
})
.from(titles)
.where(inArray(titles.id, titleIds))
.all();
}