mirror of
https://github.com/jakejarvis/sofa.git
synced 2026-10-15 22:55:42 -04:00
perf(cron): refresh ended titles less often and only recent seasons of returning shows
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
7 files changed
+224
-18
No files matched your search
+18
-9
@@ -8,7 +8,8 @@ import {
|
||||
failCronRun,
|
||||
getLibraryTitleIds,
|
||||
getReturningTvShows,
|
||||
getStaleLibraryTitles,
|
||||
getLibraryTitlesDueForRefresh,
|
||||
getStoredSeasonNumbers,
|
||||
getStaleNonLibraryTitlesForRefresh,
|
||||
getThumbhashBackfillTitleIds,
|
||||
getTitleByIdForCron,
|
||||
@@ -28,6 +29,7 @@ import {
|
||||
refreshRecommendations,
|
||||
refreshTitle,
|
||||
refreshTvChildren,
|
||||
seasonsToRefresh,
|
||||
syncTvChildArt,
|
||||
} from "@sofa/core/metadata";
|
||||
import { getSetting } from "@sofa/core/settings";
|
||||
@@ -100,14 +102,13 @@ export async function triggerJob(name: string): Promise<boolean> {
|
||||
async function nightlyRefreshLibrary() {
|
||||
const libraryIds = getLibraryTitleIds();
|
||||
log.debug(`Checking ${libraryIds.length} library titles for staleness`);
|
||||
const libraryStale = new Date(Date.now() - 7 * DAY);
|
||||
const nonLibraryStale = new Date(Date.now() - 30 * DAY);
|
||||
|
||||
// Library titles: 7 days
|
||||
const staleLibrary = getStaleLibraryTitles(libraryIds, libraryStale);
|
||||
// Library titles: 7 days, or 60 days for settled (ended/canceled/old) titles
|
||||
const dueLibraryIds = getLibraryTitlesDueForRefresh(libraryIds);
|
||||
|
||||
await runIsolated(
|
||||
staleLibrary.map((t) => t.id),
|
||||
dueLibraryIds,
|
||||
async (id) => {
|
||||
await refreshTitle(id);
|
||||
await Bun.sleep(RATE_LIMIT_MS);
|
||||
@@ -169,7 +170,8 @@ async function refreshRecommendationsJob() {
|
||||
async function refreshTvChildrenJob() {
|
||||
const stale = new Date(Date.now() - 7 * DAY);
|
||||
|
||||
const tvShows = getReturningTvShows();
|
||||
const libraryIds = new Set(getLibraryTitleIds());
|
||||
const tvShows = getReturningTvShows().filter((s) => libraryIds.has(s.id));
|
||||
|
||||
log.debug(`Checking ${tvShows.length} returning TV shows for stale episodes`);
|
||||
|
||||
@@ -180,7 +182,12 @@ async function refreshTvChildrenJob() {
|
||||
tvShows.filter((s) => titlesWithStaleSeasons.has(s.id)),
|
||||
async (show) => {
|
||||
const details = await getTvDetails(show.tmdbId);
|
||||
await refreshTvChildren(show.id, show.tmdbId, details.number_of_seasons);
|
||||
const onlySeasons = seasonsToRefresh(
|
||||
details.status ?? null,
|
||||
details.number_of_seasons,
|
||||
getStoredSeasonNumbers(show.id),
|
||||
);
|
||||
await refreshTvChildren(show.id, show.tmdbId, details.number_of_seasons, { onlySeasons });
|
||||
await syncTvChildArt(show.id, { warmCache: true });
|
||||
await Bun.sleep(RATE_LIMIT_MS);
|
||||
},
|
||||
@@ -193,18 +200,20 @@ async function cacheImagesJob() {
|
||||
log.debug(`Caching images for ${titleIds.length} titles needing art backfill`);
|
||||
|
||||
for (const titleId of titleIds) {
|
||||
let downloads = 0;
|
||||
try {
|
||||
const title = getTitleByIdForCron(titleId);
|
||||
if (!title) continue;
|
||||
|
||||
// Phase 1: warm the image cache so thumbhash generation can read from disk
|
||||
if (imageCacheEnabled()) {
|
||||
await Promise.all([
|
||||
const counts = await Promise.all([
|
||||
cacheImagesForTitle(titleId),
|
||||
cacheEpisodeStills(titleId),
|
||||
cacheProviderLogos(titleId),
|
||||
cacheProfilePhotos(titleId),
|
||||
]);
|
||||
downloads = counts.reduce((a, b) => a + b, 0);
|
||||
}
|
||||
|
||||
// Phase 2: generate thumbhashes (reads from warm cache, no duplicate downloads)
|
||||
@@ -227,7 +236,7 @@ async function cacheImagesJob() {
|
||||
} catch (err) {
|
||||
log.warn(`Failed to cache images for title ${titleId}:`, err);
|
||||
}
|
||||
await Bun.sleep(RATE_LIMIT_MS);
|
||||
if (downloads > 0) await Bun.sleep(RATE_LIMIT_MS);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
getCastEntryForTitle,
|
||||
getLibraryTitleIds as queryGetLibraryTitleIds,
|
||||
getRefreshCandidates,
|
||||
getReturningTvShows,
|
||||
getStaleTitles,
|
||||
getStaleNonLibraryTitles,
|
||||
@@ -14,6 +15,7 @@ import {
|
||||
updateCronRunError,
|
||||
updateCronRunSuccess,
|
||||
} from "@sofa/db/queries/cron";
|
||||
import { getSeasonsForTitle } from "@sofa/db/queries/metadata";
|
||||
|
||||
/**
|
||||
* Run `fn` for each item in order, isolating failures so one bad item (e.g. a
|
||||
@@ -69,6 +71,40 @@ export function getStaleLibraryTitles(libraryIds: string[], staleDate: Date) {
|
||||
return getStaleTitles(libraryIds, staleDate);
|
||||
}
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
const SETTLED_TV_STATUSES = new Set(["Ended", "Canceled"]);
|
||||
|
||||
/** How long a library title's metadata stays fresh. Settled titles rarely change on TMDB. */
|
||||
export function libraryRefreshIntervalMs(
|
||||
title: { type: string; status: string | null; releaseDate: string | null },
|
||||
now = new Date(),
|
||||
): number {
|
||||
if (title.type === "tv" && title.status && SETTLED_TV_STATUSES.has(title.status)) {
|
||||
return 60 * DAY_MS;
|
||||
}
|
||||
if (title.type === "movie" && title.releaseDate) {
|
||||
const released = Date.parse(`${title.releaseDate}T00:00:00Z`);
|
||||
if (Number.isFinite(released) && now.getTime() - released > 365 * DAY_MS) return 60 * DAY_MS;
|
||||
}
|
||||
return 7 * DAY_MS;
|
||||
}
|
||||
|
||||
/** Library title ids whose metadata is due for a refresh. */
|
||||
export function getLibraryTitlesDueForRefresh(libraryIds: string[], now = new Date()): string[] {
|
||||
return getRefreshCandidates(libraryIds)
|
||||
.filter(
|
||||
(t) =>
|
||||
!t.lastFetchedAt ||
|
||||
now.getTime() - t.lastFetchedAt.getTime() >= libraryRefreshIntervalMs(t, now),
|
||||
)
|
||||
.map((t) => t.id);
|
||||
}
|
||||
|
||||
/** Season numbers Sofa has stored for a title. */
|
||||
export function getStoredSeasonNumbers(titleId: string): number[] {
|
||||
return getSeasonsForTitle(titleId).map((s) => s.seasonNumber);
|
||||
}
|
||||
|
||||
export function getStaleNonLibraryTitlesForRefresh(staleDate: Date, limit: number) {
|
||||
return getStaleNonLibraryTitles(staleDate, limit);
|
||||
}
|
||||
|
||||
@@ -167,9 +167,9 @@ export async function loadImageBuffer(
|
||||
return remote?.buffer ?? null;
|
||||
}
|
||||
|
||||
export async function cacheImagesForTitle(titleId: string) {
|
||||
export async function cacheImagesForTitle(titleId: string): Promise<number> {
|
||||
const title = getTitleWithPaths(titleId);
|
||||
if (!title) return;
|
||||
if (!title) return 0;
|
||||
|
||||
// Collect all candidate images, then check cache in parallel
|
||||
const candidates: { imgPath: string; category: ImageCategory }[] = [];
|
||||
@@ -200,9 +200,10 @@ export async function cacheImagesForTitle(titleId: string) {
|
||||
}
|
||||
|
||||
await Promise.allSettled(tasks);
|
||||
return tasks.length;
|
||||
}
|
||||
|
||||
export async function cacheEpisodeStills(titleId: string) {
|
||||
export async function cacheEpisodeStills(titleId: string): Promise<number> {
|
||||
const allEps = getEpisodeStillsForTitle(titleId);
|
||||
|
||||
const epsWithStills = allEps.filter(
|
||||
@@ -221,9 +222,10 @@ export async function cacheEpisodeStills(titleId: string) {
|
||||
(c) => downloadAndCacheImage(c.stillPath, "stills"),
|
||||
STILL_DOWNLOAD_CONCURRENCY,
|
||||
);
|
||||
return uncached.length;
|
||||
}
|
||||
|
||||
export async function cacheProviderLogos(titleId: string) {
|
||||
export async function cacheProviderLogos(titleId: string): Promise<number> {
|
||||
const offers = getAvailabilityLogosForTitle(titleId);
|
||||
|
||||
// Deduplicate and parallel cache checks
|
||||
@@ -245,9 +247,10 @@ export async function cacheProviderLogos(titleId: string) {
|
||||
.map((c) => downloadAndCacheImage(c.logoPath, "logos"));
|
||||
|
||||
await Promise.allSettled(tasks);
|
||||
return tasks.length;
|
||||
}
|
||||
|
||||
export async function cacheProfilePhotos(titleId: string) {
|
||||
export async function cacheProfilePhotos(titleId: string): Promise<number> {
|
||||
const castRows = getCastProfilePathsForTitle(titleId);
|
||||
|
||||
// Deduplicate and parallel cache checks
|
||||
@@ -272,4 +275,5 @@ export async function cacheProfilePhotos(titleId: string) {
|
||||
log.debug(`Caching ${tasks.length} profile photos for title ${titleId}`);
|
||||
}
|
||||
await Promise.allSettled(tasks);
|
||||
return tasks.length;
|
||||
}
|
||||
@@ -334,7 +334,12 @@ export async function refreshTitle(titleId: string) {
|
||||
lastFetchedAt: now,
|
||||
});
|
||||
upsertGenresTransaction(titleId, show.genres ?? []);
|
||||
await refreshTvChildren(titleId, title.tmdbId, show.number_of_seasons);
|
||||
const onlySeasons = seasonsToRefresh(
|
||||
show.status ?? null,
|
||||
show.number_of_seasons,
|
||||
getSeasonsForTitle(titleId).map((s) => s.seasonNumber),
|
||||
);
|
||||
await refreshTvChildren(titleId, title.tmdbId, show.number_of_seasons, { onlySeasons });
|
||||
}
|
||||
|
||||
const updated = getTitleById(titleId);
|
||||
@@ -351,9 +356,38 @@ export async function refreshTitle(titleId: string) {
|
||||
return updated;
|
||||
}
|
||||
|
||||
export async function refreshTvChildren(titleId: string, tmdbId: number, numberOfSeasons: number) {
|
||||
const RETURNING_TV_STATUSES = new Set(["Returning Series", "In Production"]);
|
||||
|
||||
/**
|
||||
* Seasons worth re-fetching for a show we already have: for returning shows, the two latest
|
||||
* plus any season never stored; otherwise every season.
|
||||
*/
|
||||
export function seasonsToRefresh(
|
||||
status: string | null,
|
||||
numberOfSeasons: number,
|
||||
storedSeasonNumbers: number[],
|
||||
): number[] | undefined {
|
||||
if (!status || !RETURNING_TV_STATUSES.has(status) || storedSeasonNumbers.length === 0) {
|
||||
return undefined; // all seasons
|
||||
}
|
||||
const stored = new Set(storedSeasonNumbers);
|
||||
const result: number[] = [];
|
||||
for (let sn = 1; sn <= numberOfSeasons; sn++) {
|
||||
if (sn >= numberOfSeasons - 1 || !stored.has(sn)) result.push(sn);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function refreshTvChildren(
|
||||
titleId: string,
|
||||
tmdbId: number,
|
||||
numberOfSeasons: number,
|
||||
options?: { onlySeasons?: number[] },
|
||||
) {
|
||||
// Fetch seasons with limited concurrency to stay within TMDB's 40 req/s limit
|
||||
const seasonNumbers = Array.from({ length: numberOfSeasons }, (_, i) => i + 1);
|
||||
const allSeasons = Array.from({ length: numberOfSeasons }, (_, i) => i + 1);
|
||||
const only = options?.onlySeasons;
|
||||
const seasonNumbers = only ? allSeasons.filter((sn) => only.includes(sn)) : allSeasons;
|
||||
const fetched = await mapWithConcurrency(
|
||||
seasonNumbers,
|
||||
(sn) => getTvSeasonDetails(tmdbId, sn),
|
||||
|
||||
@@ -6,7 +6,9 @@ import { clearAllTables, eq, insertTitle, testDb } from "@sofa/test/db";
|
||||
import {
|
||||
completeCronRun,
|
||||
failCronRun,
|
||||
getLibraryTitlesDueForRefresh,
|
||||
getStaleLibraryTitles,
|
||||
libraryRefreshIntervalMs,
|
||||
runIsolated,
|
||||
startCronRun,
|
||||
} from "../src/cron";
|
||||
@@ -125,3 +127,74 @@ describe("getStaleLibraryTitles", () => {
|
||||
expect(getStaleLibraryTitles(["t-shell"], staleDate)).toEqual([{ id: "t-shell" }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("libraryRefreshIntervalMs", () => {
|
||||
const DAY = 24 * 60 * 60 * 1000;
|
||||
const now = new Date("2026-06-01T00:00:00Z");
|
||||
|
||||
test("ended TV shows refresh every 60 days", () => {
|
||||
expect(libraryRefreshIntervalMs({ type: "tv", status: "Ended", releaseDate: null }, now)).toBe(
|
||||
60 * DAY,
|
||||
);
|
||||
});
|
||||
|
||||
test("returning TV shows refresh every 7 days", () => {
|
||||
expect(
|
||||
libraryRefreshIntervalMs({ type: "tv", status: "Returning Series", releaseDate: null }, now),
|
||||
).toBe(7 * DAY);
|
||||
});
|
||||
|
||||
test("movies released over a year ago refresh every 60 days", () => {
|
||||
expect(
|
||||
libraryRefreshIntervalMs(
|
||||
{ type: "movie", status: "Released", releaseDate: "2001-01-01" },
|
||||
now,
|
||||
),
|
||||
).toBe(60 * DAY);
|
||||
});
|
||||
|
||||
test("recent movies refresh every 7 days", () => {
|
||||
expect(
|
||||
libraryRefreshIntervalMs(
|
||||
{ type: "movie", status: "Released", releaseDate: "2026-05-02" },
|
||||
now,
|
||||
),
|
||||
).toBe(7 * DAY);
|
||||
});
|
||||
|
||||
test("movies without a release date refresh every 7 days", () => {
|
||||
expect(
|
||||
libraryRefreshIntervalMs({ type: "movie", status: "Released", releaseDate: null }, now),
|
||||
).toBe(7 * DAY);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getLibraryTitlesDueForRefresh", () => {
|
||||
const now = new Date("2026-06-01T00:00:00Z");
|
||||
const tenDaysAgo = new Date(now.getTime() - 10 * 24 * 60 * 60 * 1000);
|
||||
|
||||
test("skips an ended show fetched 10 days ago", () => {
|
||||
insertTitle({ id: "t-ended", tmdbId: 1, type: "tv" });
|
||||
testDb
|
||||
.update(titles)
|
||||
.set({ status: "Ended", lastFetchedAt: tenDaysAgo })
|
||||
.where(eq(titles.id, "t-ended"))
|
||||
.run();
|
||||
expect(getLibraryTitlesDueForRefresh(["t-ended"], now)).toEqual([]);
|
||||
});
|
||||
|
||||
test("includes a returning show fetched 10 days ago", () => {
|
||||
insertTitle({ id: "t-returning", tmdbId: 2, type: "tv" });
|
||||
testDb
|
||||
.update(titles)
|
||||
.set({ status: "Returning Series", lastFetchedAt: tenDaysAgo })
|
||||
.where(eq(titles.id, "t-returning"))
|
||||
.run();
|
||||
expect(getLibraryTitlesDueForRefresh(["t-returning"], now)).toEqual(["t-returning"]);
|
||||
});
|
||||
|
||||
test("includes a never-fetched shell title", () => {
|
||||
insertTitle({ id: "t-shell", tmdbId: 3 });
|
||||
expect(getLibraryTitlesDueForRefresh(["t-shell"], now)).toEqual(["t-shell"]);
|
||||
});
|
||||
});
|
||||
@@ -56,7 +56,11 @@ vi.mock("@sofa/tmdb/client", () => ({
|
||||
getVideos: async () => ({ results: [] }),
|
||||
}));
|
||||
|
||||
import { refreshTvChildren, updateTitleWithArtInvalidation } from "../src/metadata";
|
||||
import {
|
||||
refreshTvChildren,
|
||||
seasonsToRefresh,
|
||||
updateTitleWithArtInvalidation,
|
||||
} from "../src/metadata";
|
||||
|
||||
beforeEach(() => {
|
||||
clearAllTables();
|
||||
@@ -66,7 +70,38 @@ beforeEach(() => {
|
||||
}));
|
||||
});
|
||||
|
||||
describe("seasonsToRefresh", () => {
|
||||
test("returning shows refresh only the two latest seasons", () => {
|
||||
expect(seasonsToRefresh("Returning Series", 6, [1, 2, 3, 4, 5, 6])).toEqual([5, 6]);
|
||||
});
|
||||
|
||||
test("returning shows also refresh seasons never stored", () => {
|
||||
expect(seasonsToRefresh("Returning Series", 6, [1, 2, 3])).toEqual([4, 5, 6]);
|
||||
});
|
||||
|
||||
test("ended shows refresh every season", () => {
|
||||
expect(seasonsToRefresh("Ended", 6, [1, 2, 3, 4, 5, 6])).toBeUndefined();
|
||||
});
|
||||
|
||||
test("shows with no stored seasons refresh every season", () => {
|
||||
expect(seasonsToRefresh("Returning Series", 6, [])).toBeUndefined();
|
||||
});
|
||||
|
||||
test("single-season returning show refreshes that season", () => {
|
||||
expect(seasonsToRefresh("Returning Series", 1, [1])).toEqual([1]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("refreshTvChildren", () => {
|
||||
test("onlySeasons limits which seasons are fetched", async () => {
|
||||
insertTitle({ id: "tv-1", tmdbId: 10, type: "tv", title: "Show" });
|
||||
|
||||
await refreshTvChildren("tv-1", 10, 3, { onlySeasons: [3] });
|
||||
|
||||
expect(mockGetTvSeasonDetails).toHaveBeenCalledTimes(1);
|
||||
expect(mockGetTvSeasonDetails.mock.calls[0] as unknown[]).toEqual([10, 3]);
|
||||
});
|
||||
|
||||
test("clears stale season and episode thumbhashes when artwork paths change", async () => {
|
||||
insertTitle({ id: "tv-1", tmdbId: 10, type: "tv", title: "Show" });
|
||||
testDb
|
||||
|
||||
@@ -180,3 +180,18 @@ export function getTitleIdsCheckedBefore(
|
||||
.all()
|
||||
.map((r) => r.id);
|
||||
}
|
||||
|
||||
export function getRefreshCandidates(titleIds: string[]) {
|
||||
if (titleIds.length === 0) return [];
|
||||
return db
|
||||
.select({
|
||||
id: titles.id,
|
||||
type: titles.type,
|
||||
status: titles.status,
|
||||
releaseDate: titles.releaseDate,
|
||||
lastFetchedAt: titles.lastFetchedAt,
|
||||
})
|
||||
.from(titles)
|
||||
.where(inArray(titles.id, titleIds))
|
||||
.all();
|
||||
}
|
||||
Reference in new issue
Block a user