mirror of
https://github.com/jakejarvis/sofa.git
synced 2026-10-16 05:25:42 -04:00
fix(imports): cap uncompressed size and entry count of uploaded ZIPs
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2 files changed
+116
-4
No files matched your search
@@ -10,6 +10,49 @@ import { createLogger } from "@sofa/logger";
|
||||
|
||||
const log = createLogger("imports");
|
||||
|
||||
// Real exports are a few MB; these bound memory if an archive is a decompression bomb.
|
||||
// adm-zip caps each entry's inflate output at its declared size, so checking declared
|
||||
// sizes before getData() is sufficient.
|
||||
export const ZIP_LIMITS = {
|
||||
maxEntries: 2_000,
|
||||
maxEntryBytes: 128 * 1024 * 1024,
|
||||
maxTotalBytes: 256 * 1024 * 1024,
|
||||
};
|
||||
|
||||
export class ZipTooLargeError extends Error {}
|
||||
|
||||
interface ZipEntryLike {
|
||||
entryName: string;
|
||||
isDirectory: boolean;
|
||||
header: { size: number };
|
||||
}
|
||||
|
||||
/**
|
||||
* Throws ZipTooLargeError if the archive has too many entries, or if the entries
|
||||
* that `willRead` selects declare more uncompressed bytes than allowed.
|
||||
*/
|
||||
export function assertZipWithinLimits(
|
||||
entries: readonly ZipEntryLike[],
|
||||
willRead: (entry: ZipEntryLike) => boolean,
|
||||
limits: typeof ZIP_LIMITS = ZIP_LIMITS,
|
||||
): void {
|
||||
if (entries.length > limits.maxEntries) {
|
||||
throw new ZipTooLargeError(`ZIP has ${entries.length} entries (max ${limits.maxEntries})`);
|
||||
}
|
||||
let total = 0;
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory || !willRead(entry)) continue;
|
||||
const size = entry.header.size;
|
||||
if (size > limits.maxEntryBytes) {
|
||||
throw new ZipTooLargeError(`ZIP entry ${entry.entryName} is too large`);
|
||||
}
|
||||
total += size;
|
||||
if (total > limits.maxTotalBytes) {
|
||||
throw new ZipTooLargeError("ZIP contents are too large");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Types ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface ImportMovie {
|
||||
@@ -454,10 +497,11 @@ export async function parseTraktExport(file: Blob): Promise<ParseResult> {
|
||||
} catch {
|
||||
throw new Error("Invalid export file");
|
||||
}
|
||||
const isTraktJson = (entry: { entryName: string }) =>
|
||||
(entry.entryName.split("/").pop() ?? entry.entryName).toLowerCase().endsWith(".json");
|
||||
assertZipWithinLimits(entries, isTraktJson);
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory) continue;
|
||||
const name = entry.entryName.split("/").pop() ?? entry.entryName;
|
||||
if (!name.toLowerCase().endsWith(".json")) continue;
|
||||
if (entry.isDirectory || !isTraktJson(entry)) continue;
|
||||
try {
|
||||
addTraktDocument(agg, parseJsonText(entry.getData().toString("utf-8")));
|
||||
} catch {
|
||||
@@ -752,6 +796,21 @@ export async function parseLetterboxdExport(zipFile: Blob): Promise<ParseResult>
|
||||
}
|
||||
|
||||
const entries = zip.getEntries();
|
||||
const isExpectedLetterboxdCsv = (entry: { entryName: string }) =>
|
||||
LETTERBOXD_EXPECTED_FILES.includes(
|
||||
(entry.entryName.split("/").pop() ??
|
||||
entry.entryName) as (typeof LETTERBOXD_EXPECTED_FILES)[number],
|
||||
);
|
||||
try {
|
||||
assertZipWithinLimits(entries, isExpectedLetterboxdCsv);
|
||||
} catch (err) {
|
||||
if (!(err instanceof ZipTooLargeError)) throw err;
|
||||
warnings.push("ZIP file is too large to import. Ensure it is an unmodified Letterboxd export.");
|
||||
return {
|
||||
data: { source: "letterboxd", movies, episodes: [], watchlist, ratings },
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
const entryMap = new Map<string, string>();
|
||||
const allFilenames: string[] = [];
|
||||
|
||||
@@ -760,7 +819,7 @@ export async function parseLetterboxdExport(zipFile: Blob): Promise<ParseResult>
|
||||
// Letterboxd exports may nest files in a subdirectory — use the basename
|
||||
const name = entry.entryName.split("/").pop() ?? entry.entryName;
|
||||
allFilenames.push(name);
|
||||
if (LETTERBOXD_EXPECTED_FILES.includes(name as (typeof LETTERBOXD_EXPECTED_FILES)[number])) {
|
||||
if (isExpectedLetterboxdCsv(entry)) {
|
||||
entryMap.set(name, entry.getData().toString("utf-8"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,11 +4,14 @@ import { describe, expect, test } from "vitest";
|
||||
import { NormalizedImportSchema } from "@sofa/api/schemas";
|
||||
|
||||
import {
|
||||
assertZipWithinLimits,
|
||||
type ParseResult,
|
||||
parseLetterboxdExport,
|
||||
parseSimklPayload,
|
||||
parseTraktExport,
|
||||
parseTraktPayload,
|
||||
ZIP_LIMITS,
|
||||
ZipTooLargeError,
|
||||
} from "../src/imports/parsers";
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────────────────
|
||||
@@ -1393,3 +1396,53 @@ describe("parseTraktExport", () => {
|
||||
expect(result.data.episodes).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ZIP limits", () => {
|
||||
const limits = { maxEntries: 10, maxEntryBytes: 10, maxTotalBytes: 100 };
|
||||
const entry = (entryName: string, size: number) => ({
|
||||
entryName,
|
||||
isDirectory: false,
|
||||
header: { size },
|
||||
});
|
||||
const isJson = (e: { entryName: string }) => e.entryName.endsWith(".json");
|
||||
|
||||
function createOversizedZip(): Blob {
|
||||
const zip = new AdmZip();
|
||||
for (let i = 0; i < ZIP_LIMITS.maxEntries + 1; i++) {
|
||||
zip.addFile(`f${i}.txt`, Buffer.from("x"));
|
||||
}
|
||||
return new Blob([zip.toBuffer()], { type: "application/zip" });
|
||||
}
|
||||
|
||||
test("rejects a selected entry that declares more than maxEntryBytes", () => {
|
||||
expect(() => assertZipWithinLimits([entry("a.json", 11)], isJson, limits)).toThrow(
|
||||
ZipTooLargeError,
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects when the selected entries' total exceeds maxTotalBytes", () => {
|
||||
const entries = [entry("a.json", 40), entry("b.json", 40), entry("c.json", 40)];
|
||||
expect(() => assertZipWithinLimits(entries, isJson, { ...limits, maxEntryBytes: 50 })).toThrow(
|
||||
ZipTooLargeError,
|
||||
);
|
||||
});
|
||||
|
||||
test("ignores oversized entries the predicate does not select", () => {
|
||||
expect(() => assertZipWithinLimits([entry("reviews.csv", 1000)], isJson, limits)).not.toThrow();
|
||||
});
|
||||
|
||||
test("rejects archives with more than maxEntries entries", () => {
|
||||
const entries = Array.from({ length: 11 }, (_, i) => entry(`f${i}.json`, 1));
|
||||
expect(() => assertZipWithinLimits(entries, isJson, limits)).toThrow(ZipTooLargeError);
|
||||
});
|
||||
|
||||
test("parseTraktExport rejects an archive with too many entries", async () => {
|
||||
await expect(parseTraktExport(createOversizedZip())).rejects.toBeInstanceOf(ZipTooLargeError);
|
||||
});
|
||||
|
||||
test("parseLetterboxdExport returns a warning for an archive with too many entries", async () => {
|
||||
const result = await parseLetterboxdExport(createOversizedZip());
|
||||
expect(result.data.movies).toHaveLength(0);
|
||||
expect(result.warnings.some((w) => w.includes("too large"))).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user