mirror of
https://github.com/jakejarvis/sofa.git
synced 2026-07-14 18:15:56 -04:00
refactor: clean up orphaned cached images on cache purge and metadata updates
- Add `deleteOrphanedImage(category, path)` helper to `image-cache.ts` for fire-and-forget unlink of stale cached files - Extend `purgeShellTitlesTransaction` to collect poster, backdrop, still, and profile image paths from deleted titles/persons before cascade deletes, returning them as `orphanedImages` in `PurgeResult` - Wire orphaned image cleanup into `purgeMetadataCache` (cache purge), `batchUpsertPersons` (profile path changes), `updateTitleWithArtInvalidation` (poster/backdrop changes), and `refreshTvChildren` (season posters and episode stills) - Remove `getTitlesWithMissingThumbhashes`, `getTitleIdsWithMissingSeasonThumbhashes`, `getTitleIdsWithMissingEpisodeThumbhashes`, and `getTitleIdsWithMissingProfileThumbhashes` from `@sofa/db/queries/cron` — `getThumbhashBackfillTitleIds` now delegates to `getLibraryTitleIds` only - Fix episode still cleanup to call `deleteOrphanedImage` even when `stillThumbHash` is null (previously skipped)
This commit is contained in:
@@ -5,11 +5,13 @@ import { CACHE_DIR } from "@sofa/config";
|
||||
import { purgeShellTitlesTransaction } from "@sofa/db/queries/cache";
|
||||
import { createLogger } from "@sofa/logger";
|
||||
|
||||
import { deleteOrphanedImage } from "./image-cache";
|
||||
|
||||
const log = createLogger("purge");
|
||||
|
||||
/**
|
||||
* Delete un-enriched "shell" titles that aren't in any user's library,
|
||||
* then clean up orphaned person records.
|
||||
* then clean up orphaned person records and their cached images.
|
||||
*/
|
||||
export function purgeMetadataCache(): {
|
||||
deletedTitles: number;
|
||||
@@ -27,7 +29,16 @@ export function purgeMetadataCache(): {
|
||||
log.info(`Purged ${result.deletedPersons} orphaned persons`);
|
||||
}
|
||||
|
||||
return result;
|
||||
// Fire-and-forget cleanup of cached images for deleted titles/persons
|
||||
for (const img of result.orphanedImages) {
|
||||
deleteOrphanedImage(img.category, img.path);
|
||||
}
|
||||
|
||||
if (result.orphanedImages.length > 0) {
|
||||
log.info(`Queued deletion of ${result.orphanedImages.length} orphaned cached images`);
|
||||
}
|
||||
|
||||
return { deletedTitles: result.deletedTitles, deletedPersons: result.deletedPersons };
|
||||
}
|
||||
|
||||
/** Image cache subdirectories to scan */
|
||||
|
||||
@@ -13,7 +13,7 @@ import { createLogger } from "@sofa/logger";
|
||||
import { getMovieCredits, getTvAggregateCredits } from "@sofa/tmdb/client";
|
||||
import { tmdbImageUrl } from "@sofa/tmdb/image";
|
||||
|
||||
import { cacheProfilePhotos, imageCacheEnabled } from "./image-cache";
|
||||
import { cacheProfilePhotos, deleteOrphanedImage, imageCacheEnabled } from "./image-cache";
|
||||
import { generatePersonThumbHash } from "./thumbhash";
|
||||
|
||||
const log = createLogger("credits");
|
||||
@@ -50,6 +50,14 @@ function batchUpsertPersons(people: PersonData[]): Map<number, string> {
|
||||
|
||||
const idMap = batchUpsertPersonsTransaction(uniquePeople, existingMap);
|
||||
|
||||
// Clean up orphaned profile images when paths change
|
||||
for (const p of uniquePeople) {
|
||||
const prev = existingMap.get(p.tmdbId);
|
||||
if (prev && prev.profilePath && prev.profilePath !== p.profilePath) {
|
||||
deleteOrphanedImage("profiles", prev.profilePath);
|
||||
}
|
||||
}
|
||||
|
||||
// One fallback query for any concurrent inserts that conflicted
|
||||
const stillMissing = uniquePeople.filter((p) => !idMap.has(p.tmdbId)).map((p) => p.tmdbId);
|
||||
if (stillMissing.length > 0) {
|
||||
|
||||
@@ -5,11 +5,7 @@ import {
|
||||
getStaleTitles,
|
||||
getStaleNonLibraryTitles,
|
||||
getTitleByIdForCron,
|
||||
getTitleIdsWithMissingEpisodeThumbhashes,
|
||||
getTitleIdsWithMissingProfileThumbhashes,
|
||||
getTitleIdsWithMissingSeasonThumbhashes,
|
||||
getTitleIdsWithStaleSeasons,
|
||||
getTitlesWithMissingThumbhashes,
|
||||
getTitlesWithStaleOffers,
|
||||
getTitlesWithStaleOffersFetchedBefore,
|
||||
insertCronRunReturning,
|
||||
@@ -35,14 +31,7 @@ export function getLibraryTitleIds(): string[] {
|
||||
}
|
||||
|
||||
export function getThumbhashBackfillTitleIds(): string[] {
|
||||
const titleIds = new Set(getLibraryTitleIds());
|
||||
|
||||
for (const id of getTitlesWithMissingThumbhashes()) titleIds.add(id);
|
||||
for (const id of getTitleIdsWithMissingSeasonThumbhashes()) titleIds.add(id);
|
||||
for (const id of getTitleIdsWithMissingEpisodeThumbhashes()) titleIds.add(id);
|
||||
for (const id of getTitleIdsWithMissingProfileThumbhashes()) titleIds.add(id);
|
||||
|
||||
return [...titleIds];
|
||||
return getLibraryTitleIds();
|
||||
}
|
||||
|
||||
export function getStaleLibraryTitles(libraryIds: string[], staleDate: Date) {
|
||||
|
||||
@@ -20,6 +20,12 @@ export function imageCacheEnabled(): boolean {
|
||||
return process.env.IMAGE_CACHE_ENABLED !== "false";
|
||||
}
|
||||
|
||||
export function deleteOrphanedImage(category: ImageCategory, oldPath: string | null): void {
|
||||
if (!oldPath || !imageCacheEnabled()) return;
|
||||
const filePath = getLocalImagePath(category, path.basename(oldPath));
|
||||
unlink(filePath).catch(() => {});
|
||||
}
|
||||
|
||||
export async function ensureImageDirs() {
|
||||
for (const category of Object.keys(IMAGE_CATEGORY_SIZES) as ImageCategory[]) {
|
||||
await mkdir(path.join(CACHE_DIR, category), { recursive: true });
|
||||
|
||||
@@ -49,6 +49,7 @@ import { getCastForTitle, refreshCredits } from "./credits";
|
||||
import {
|
||||
cacheEpisodeStills,
|
||||
cacheImagesForTitle,
|
||||
deleteOrphanedImage,
|
||||
imageCacheEnabled,
|
||||
loadImageBuffer,
|
||||
} from "./image-cache";
|
||||
@@ -93,6 +94,9 @@ export function updateTitleWithArtInvalidation(
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
|
||||
if (posterPathChanged) deleteOrphanedImage("posters", title.posterPath);
|
||||
if (backdropPathChanged) deleteOrphanedImage("backdrops", title.backdropPath);
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
@@ -388,12 +392,14 @@ export async function refreshTvChildren(titleId: string, tmdbId: number, numberO
|
||||
// cache warming, so we only need to null out stale values here.
|
||||
if ((existingSeason?.posterPath ?? null) !== (seasonData.poster_path ?? null)) {
|
||||
nullifySeasonThumbHash(seasonRow.id);
|
||||
deleteOrphanedImage("posters", existingSeason?.posterPath ?? null);
|
||||
}
|
||||
const seasonEps = getSeasonEpisodesWithHashes(seasonRow.id);
|
||||
for (const ep of seasonEps) {
|
||||
const oldStill = oldEpStills.get(ep.episodeNumber);
|
||||
if (oldStill !== ep.stillPath && ep.stillThumbHash) {
|
||||
nullifyEpisodeThumbHash(ep.id);
|
||||
if (oldStill !== ep.stillPath) {
|
||||
if (ep.stillThumbHash) nullifyEpisodeThumbHash(ep.id);
|
||||
deleteOrphanedImage("stills", oldStill ?? null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,89 @@
|
||||
import { inArray, isNull, notInArray } from "drizzle-orm";
|
||||
|
||||
import { db } from "../client";
|
||||
import { persons, titleCast, titles, userTitleStatus } from "../schema";
|
||||
import { episodes, persons, seasons, titleCast, titles, userTitleStatus } from "../schema";
|
||||
|
||||
const BATCH_SIZE = 500;
|
||||
|
||||
export function purgeShellTitlesTransaction(): { deletedTitles: number; deletedPersons: number } {
|
||||
export interface OrphanedImage {
|
||||
category: "posters" | "backdrops" | "stills" | "profiles";
|
||||
path: string;
|
||||
}
|
||||
|
||||
export interface PurgeResult {
|
||||
deletedTitles: number;
|
||||
deletedPersons: number;
|
||||
orphanedImages: OrphanedImage[];
|
||||
}
|
||||
|
||||
/** Collect all cached image paths associated with a set of title IDs. */
|
||||
function collectTitleImagePaths(titleIds: string[]): OrphanedImage[] {
|
||||
const images: OrphanedImage[] = [];
|
||||
|
||||
for (let i = 0; i < titleIds.length; i += BATCH_SIZE) {
|
||||
const batch = titleIds.slice(i, i + BATCH_SIZE);
|
||||
|
||||
// Title posters + backdrops
|
||||
const titleRows = db
|
||||
.select({ posterPath: titles.posterPath, backdropPath: titles.backdropPath })
|
||||
.from(titles)
|
||||
.where(inArray(titles.id, batch))
|
||||
.all();
|
||||
for (const r of titleRows) {
|
||||
if (r.posterPath) images.push({ category: "posters", path: r.posterPath });
|
||||
if (r.backdropPath) images.push({ category: "backdrops", path: r.backdropPath });
|
||||
}
|
||||
|
||||
// Season posters
|
||||
const seasonRows = db
|
||||
.select({ posterPath: seasons.posterPath })
|
||||
.from(seasons)
|
||||
.where(inArray(seasons.titleId, batch))
|
||||
.all();
|
||||
for (const r of seasonRows) {
|
||||
if (r.posterPath) images.push({ category: "posters", path: r.posterPath });
|
||||
}
|
||||
|
||||
// Episode stills (via seasons)
|
||||
const seasonIds = db
|
||||
.select({ id: seasons.id })
|
||||
.from(seasons)
|
||||
.where(inArray(seasons.titleId, batch))
|
||||
.all()
|
||||
.map((s) => s.id);
|
||||
|
||||
for (let j = 0; j < seasonIds.length; j += BATCH_SIZE) {
|
||||
const sBatch = seasonIds.slice(j, j + BATCH_SIZE);
|
||||
const epRows = db
|
||||
.select({ stillPath: episodes.stillPath })
|
||||
.from(episodes)
|
||||
.where(inArray(episodes.seasonId, sBatch))
|
||||
.all();
|
||||
for (const r of epRows) {
|
||||
if (r.stillPath) images.push({ category: "stills", path: r.stillPath });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return images;
|
||||
}
|
||||
|
||||
/** Collect profile paths for persons that will be orphaned (no remaining titleCast). */
|
||||
function collectOrphanedPersonImages(): OrphanedImage[] {
|
||||
const orphaned = db
|
||||
.select({ profilePath: persons.profilePath })
|
||||
.from(persons)
|
||||
.where(
|
||||
notInArray(persons.id, db.selectDistinct({ personId: titleCast.personId }).from(titleCast)),
|
||||
)
|
||||
.all();
|
||||
|
||||
return orphaned
|
||||
.filter((p): p is { profilePath: string } => p.profilePath != null)
|
||||
.map((p) => ({ category: "profiles" as const, path: p.profilePath }));
|
||||
}
|
||||
|
||||
export function purgeShellTitlesTransaction(): PurgeResult {
|
||||
return db.transaction(() => {
|
||||
const shellTitles = db
|
||||
.select({ id: titles.id })
|
||||
@@ -14,7 +92,8 @@ export function purgeShellTitlesTransaction(): { deletedTitles: number; deletedP
|
||||
.all();
|
||||
|
||||
if (shellTitles.length === 0) {
|
||||
return { deletedTitles: 0, deletedPersons: purgeOrphanedPersons() };
|
||||
const orphanedImages = collectOrphanedPersonImages();
|
||||
return { deletedTitles: 0, deletedPersons: purgeOrphanedPersons(), orphanedImages };
|
||||
}
|
||||
|
||||
const libraryTitleIds = new Set(
|
||||
@@ -28,15 +107,21 @@ export function purgeShellTitlesTransaction(): { deletedTitles: number; deletedP
|
||||
const toDelete = shellTitles.map((t) => t.id).filter((id) => !libraryTitleIds.has(id));
|
||||
|
||||
if (toDelete.length === 0) {
|
||||
return { deletedTitles: 0, deletedPersons: purgeOrphanedPersons() };
|
||||
const orphanedImages = collectOrphanedPersonImages();
|
||||
return { deletedTitles: 0, deletedPersons: purgeOrphanedPersons(), orphanedImages };
|
||||
}
|
||||
|
||||
// Collect image paths BEFORE cascade-deleting the title rows
|
||||
const orphanedImages = collectTitleImagePaths(toDelete);
|
||||
|
||||
for (let i = 0; i < toDelete.length; i += BATCH_SIZE) {
|
||||
const batch = toDelete.slice(i, i + BATCH_SIZE);
|
||||
db.delete(titles).where(inArray(titles.id, batch)).run();
|
||||
}
|
||||
|
||||
return { deletedTitles: toDelete.length, deletedPersons: purgeOrphanedPersons() };
|
||||
// After title cascade deletes, collect orphaned person images then delete persons
|
||||
orphanedImages.push(...collectOrphanedPersonImages());
|
||||
return { deletedTitles: toDelete.length, deletedPersons: purgeOrphanedPersons(), orphanedImages };
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
import { and, eq, inArray, isNotNull, lt, or, sql } from "drizzle-orm";
|
||||
import { and, eq, inArray, isNotNull, lt, or } from "drizzle-orm";
|
||||
|
||||
import { db } from "../client";
|
||||
import {
|
||||
availabilityOffers,
|
||||
cronRuns,
|
||||
episodes,
|
||||
persons,
|
||||
seasons,
|
||||
titleCast,
|
||||
titles,
|
||||
@@ -43,52 +41,6 @@ export function getLibraryTitleIds(): string[] {
|
||||
.map((r) => r.titleId);
|
||||
}
|
||||
|
||||
export function getTitlesWithMissingThumbhashes() {
|
||||
return db
|
||||
.select({ id: titles.id })
|
||||
.from(titles)
|
||||
.where(
|
||||
or(
|
||||
and(isNotNull(titles.posterPath), sql`${titles.posterThumbHash} IS NULL`),
|
||||
and(isNotNull(titles.backdropPath), sql`${titles.backdropThumbHash} IS NULL`),
|
||||
),
|
||||
)
|
||||
.all()
|
||||
.map((row) => row.id);
|
||||
}
|
||||
|
||||
export function getTitleIdsWithMissingSeasonThumbhashes() {
|
||||
return db
|
||||
.select({ titleId: seasons.titleId })
|
||||
.from(seasons)
|
||||
.where(and(isNotNull(seasons.posterPath), sql`${seasons.posterThumbHash} IS NULL`))
|
||||
.groupBy(seasons.titleId)
|
||||
.all()
|
||||
.map((row) => row.titleId);
|
||||
}
|
||||
|
||||
export function getTitleIdsWithMissingEpisodeThumbhashes() {
|
||||
return db
|
||||
.select({ titleId: seasons.titleId })
|
||||
.from(episodes)
|
||||
.innerJoin(seasons, eq(episodes.seasonId, seasons.id))
|
||||
.where(and(isNotNull(episodes.stillPath), sql`${episodes.stillThumbHash} IS NULL`))
|
||||
.groupBy(seasons.titleId)
|
||||
.all()
|
||||
.map((row) => row.titleId);
|
||||
}
|
||||
|
||||
export function getTitleIdsWithMissingProfileThumbhashes() {
|
||||
return db
|
||||
.select({ titleId: titleCast.titleId })
|
||||
.from(titleCast)
|
||||
.innerJoin(persons, eq(titleCast.personId, persons.id))
|
||||
.where(and(isNotNull(persons.profilePath), sql`${persons.profileThumbHash} IS NULL`))
|
||||
.groupBy(titleCast.titleId)
|
||||
.all()
|
||||
.map((row) => row.titleId);
|
||||
}
|
||||
|
||||
export function getStaleTitles(titleIds: string[], staleDate: Date) {
|
||||
if (titleIds.length === 0) return [];
|
||||
return db
|
||||
|
||||
Reference in New Issue
Block a user