fix(imports): cap uncompressed size and entry count of uploaded ZIPs

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
jakeandClaude Opus 5.5 committed 2026-10-04 23:06:34 -04:00
1 parent 69b252e99b
commit 810bbecf78
2 files changed
+116 -4

No files matched your search

+63 -4
View File
@@ -10,6 +10,49 @@ import { createLogger } from "@sofa/logger";
const log = createLogger("imports");
// Real exports are a few MB; these bound memory if an archive is a decompression bomb.
// adm-zip caps each entry's inflate output at its declared size, so checking declared
// sizes before getData() is sufficient.
export const ZIP_LIMITS = {
maxEntries: 2_000,
maxEntryBytes: 128 * 1024 * 1024,
maxTotalBytes: 256 * 1024 * 1024,
};
export class ZipTooLargeError extends Error {}
interface ZipEntryLike {
entryName: string;
isDirectory: boolean;
header: { size: number };
}
/**
* Throws ZipTooLargeError if the archive has too many entries, or if the entries
* that `willRead` selects declare more uncompressed bytes than allowed.
*/
export function assertZipWithinLimits(
entries: readonly ZipEntryLike[],
willRead: (entry: ZipEntryLike) => boolean,
limits: typeof ZIP_LIMITS = ZIP_LIMITS,
): void {
if (entries.length > limits.maxEntries) {
throw new ZipTooLargeError(`ZIP has ${entries.length} entries (max ${limits.maxEntries})`);
}
let total = 0;
for (const entry of entries) {
if (entry.isDirectory || !willRead(entry)) continue;
const size = entry.header.size;
if (size > limits.maxEntryBytes) {
throw new ZipTooLargeError(`ZIP entry ${entry.entryName} is too large`);
}
total += size;
if (total > limits.maxTotalBytes) {
throw new ZipTooLargeError("ZIP contents are too large");
}
}
}
// ─── Types ──────────────────────────────────────────────────────────
export interface ImportMovie {
@@ -454,10 +497,11 @@ export async function parseTraktExport(file: Blob): Promise<ParseResult> {
} catch {
throw new Error("Invalid export file");
}
const isTraktJson = (entry: { entryName: string }) =>
(entry.entryName.split("/").pop() ?? entry.entryName).toLowerCase().endsWith(".json");
assertZipWithinLimits(entries, isTraktJson);
for (const entry of entries) {
if (entry.isDirectory) continue;
const name = entry.entryName.split("/").pop() ?? entry.entryName;
if (!name.toLowerCase().endsWith(".json")) continue;
if (entry.isDirectory || !isTraktJson(entry)) continue;
try {
addTraktDocument(agg, parseJsonText(entry.getData().toString("utf-8")));
} catch {
@@ -752,6 +796,21 @@ export async function parseLetterboxdExport(zipFile: Blob): Promise<ParseResult>
}
const entries = zip.getEntries();
const isExpectedLetterboxdCsv = (entry: { entryName: string }) =>
LETTERBOXD_EXPECTED_FILES.includes(
(entry.entryName.split("/").pop() ??
entry.entryName) as (typeof LETTERBOXD_EXPECTED_FILES)[number],
);
try {
assertZipWithinLimits(entries, isExpectedLetterboxdCsv);
} catch (err) {
if (!(err instanceof ZipTooLargeError)) throw err;
warnings.push("ZIP file is too large to import. Ensure it is an unmodified Letterboxd export.");
return {
data: { source: "letterboxd", movies, episodes: [], watchlist, ratings },
warnings,
};
}
const entryMap = new Map<string, string>();
const allFilenames: string[] = [];
@@ -760,7 +819,7 @@ export async function parseLetterboxdExport(zipFile: Blob): Promise<ParseResult>
// Letterboxd exports may nest files in a subdirectory — use the basename
const name = entry.entryName.split("/").pop() ?? entry.entryName;
allFilenames.push(name);
if (LETTERBOXD_EXPECTED_FILES.includes(name as (typeof LETTERBOXD_EXPECTED_FILES)[number])) {
if (isExpectedLetterboxdCsv(entry)) {
entryMap.set(name, entry.getData().toString("utf-8"));
}
}
+53
View File
@@ -4,11 +4,14 @@ import { describe, expect, test } from "vitest";
import { NormalizedImportSchema } from "@sofa/api/schemas";
import {
assertZipWithinLimits,
type ParseResult,
parseLetterboxdExport,
parseSimklPayload,
parseTraktExport,
parseTraktPayload,
ZIP_LIMITS,
ZipTooLargeError,
} from "../src/imports/parsers";
// ─── Helpers ─────────────────────────────────────────────────────────
@@ -1393,3 +1396,53 @@ describe("parseTraktExport", () => {
expect(result.data.episodes).toHaveLength(1);
});
});
describe("ZIP limits", () => {
const limits = { maxEntries: 10, maxEntryBytes: 10, maxTotalBytes: 100 };
const entry = (entryName: string, size: number) => ({
entryName,
isDirectory: false,
header: { size },
});
const isJson = (e: { entryName: string }) => e.entryName.endsWith(".json");
function createOversizedZip(): Blob {
const zip = new AdmZip();
for (let i = 0; i < ZIP_LIMITS.maxEntries + 1; i++) {
zip.addFile(`f${i}.txt`, Buffer.from("x"));
}
return new Blob([zip.toBuffer()], { type: "application/zip" });
}
test("rejects a selected entry that declares more than maxEntryBytes", () => {
expect(() => assertZipWithinLimits([entry("a.json", 11)], isJson, limits)).toThrow(
ZipTooLargeError,
);
});
test("rejects when the selected entries' total exceeds maxTotalBytes", () => {
const entries = [entry("a.json", 40), entry("b.json", 40), entry("c.json", 40)];
expect(() => assertZipWithinLimits(entries, isJson, { ...limits, maxEntryBytes: 50 })).toThrow(
ZipTooLargeError,
);
});
test("ignores oversized entries the predicate does not select", () => {
expect(() => assertZipWithinLimits([entry("reviews.csv", 1000)], isJson, limits)).not.toThrow();
});
test("rejects archives with more than maxEntries entries", () => {
const entries = Array.from({ length: 11 }, (_, i) => entry(`f${i}.json`, 1));
expect(() => assertZipWithinLimits(entries, isJson, limits)).toThrow(ZipTooLargeError);
});
test("parseTraktExport rejects an archive with too many entries", async () => {
await expect(parseTraktExport(createOversizedZip())).rejects.toBeInstanceOf(ZipTooLargeError);
});
test("parseLetterboxdExport returns a warning for an archive with too many entries", async () => {
const result = await parseLetterboxdExport(createOversizedZip());
expect(result.data.movies).toHaveLength(0);
expect(result.warnings.some((w) => w.includes("too large"))).toBe(true);
});
});