From 3023c5dd1bf619ae77df5eafae46a27366eabb56 Mon Sep 17 00:00:00 2001 From: Jake Jarvis Date: Sat, 19 Sep 2026 12:14:19 -0400 Subject: [PATCH] feat: add SSRF host validation, rate_limited/unparseable error codes, and referral chain warnings - Add `isSafeWhoisReferralHost` to reject private/loopback/link-local IPs and malformed hostnames before following WHOIS referrals (SSRF guard) - Detect WHOIS throttle replies in the referral chain: drop the registrar response, keep the registry record, and surface a warning instead of returning bad data - Add `looksEmptyWhois` guard in the main lookup path: a "registered" record with no parseable fields now fails with `errorCode: "unparseable"` rather than resolving silently - Map RDAP 429 responses to a structured `rate_limited` error (including `Retry-After` header) so callers can distinguish throttling from generic HTTP errors and the fallback to WHOIS is recorded in `attempts` - Add `rate_limited` and `unparseable` to `LookupErrorCode` - Change `collectWhoisReferralChain` to return `{ results, warnings }` instead of a bare array; warnings are merged onto the final `DomainRecord` - Remove the `followWhoisReferrals` fallback path from `index.ts` (dead code after the chain API stabilised) --- src/index.attempts.test.ts | 41 +++++++++++++++++++++++++++++ src/index.test.ts | 29 ++++++++++---------- src/index.ts | 34 ++++++++++++++---------- src/rdap/client.ts | 7 +++++ src/types.ts | 4 +++ src/whois/host.test.ts | 42 +++++++++++++++++++++++++++++ src/whois/host.ts | 54 ++++++++++++++++++++++++++++++++++++++ src/whois/merge.test.ts | 2 +- src/whois/referral.test.ts | 36 ++++++++++++++++++++++++- src/whois/referral.ts | 39 +++++++++++++++++++++++---- src/whois/throttle.test.ts | 36 +++++++++++++++++++++++++ src/whois/throttle.ts | 40 ++++++++++++++++++++++++++++ 12 files changed, 328 insertions(+), 36 deletions(-) create mode 100644 src/whois/host.test.ts create mode 100644 src/whois/host.ts create mode 100644 src/whois/throttle.test.ts create mode 100644 src/whois/throttle.ts diff --git a/src/index.attempts.test.ts b/src/index.attempts.test.ts index eb861dc..4444f84 100644 --- a/src/index.attempts.test.ts +++ b/src/index.attempts.test.ts @@ -40,6 +40,47 @@ afterEach(() => { vi.useRealTimers(); }); +describe("throttle and empty-response guards", () => { + const ianaThen = (text: string) => async (server: string) => ({ + serverQueried: server, + text: server === "whois.iana.org" ? "whois: whois.verisign-grs.com\n" : text, + }); + + it("fails with rate_limited when the registry throttles", async () => { + vi.mocked(whoisQuery).mockImplementation(ianaThen("WHOIS LIMIT EXCEEDED")); + const res = await lookup("example.com", { whoisOnly: true }); + expect(res.ok).toBe(false); + expect(res.errorCode).toBe("rate_limited"); + expect(res.errorPhase).toBe("whois"); + expect(res.errorServer).toBe("whois.verisign-grs.com"); + }); + + it("fails with unparseable when a long reply has no fields and no availability phrase", async () => { + vi.mocked(whoisQuery).mockImplementation(ianaThen(`${"lorem ipsum ".repeat(300)}\n`)); + const res = await lookup("example.com", { whoisOnly: true }); + expect(res.ok).toBe(false); + expect(res.errorCode).toBe("unparseable"); + }); + + it("still reports availability for a genuine not-found reply", async () => { + vi.mocked(whoisQuery).mockImplementation(ianaThen("No match for EXAMPLE.COM")); + const res = await lookup("example.com", { whoisOnly: true }); + expect(res.ok, res.error).toBe(true); + expect(res.record?.isRegistered).toBe(false); + }); + + it("classifies RDAP 429 as rate_limited and falls through to WHOIS", async () => { + const customFetch: FetchLike = vi.fn( + async () => new Response("slow down", { status: 429, headers: { "retry-after": "30" } }), + ); + vi.mocked(whoisQuery).mockImplementation(ianaThen(whoisText)); + const res = await lookup("example.com", { customBootstrapData: bootstrap, customFetch }); + expect(res.ok, res.error).toBe(true); + expect(res.attempts[0]).toMatchObject({ errorCode: "rate_limited" }); + expect(res.attempts[0]?.error).toContain("Retry-After: 30"); + }); +}); + describe("attempts trace", () => { it("records a failed RDAP base followed by a WHOIS success, in order", async () => { const customFetch: FetchLike = vi.fn(async () => new Response("nope", { status: 503 })); diff --git a/src/index.test.ts b/src/index.test.ts index 54a1b7c..07165cd 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -21,7 +21,7 @@ vi.mock("./rdap/merge.js", () => ({ vi.mock("./whois/client.js", () => ({ whoisQuery: vi.fn(async () => ({ - text: "Domain Name: EXAMPLE.COM", + text: "Domain Name: EXAMPLE.COM\nRegistrar: Test Registrar", serverQueried: "whois.verisign-grs.com", })), })); @@ -29,14 +29,11 @@ vi.mock("./whois/client.js", () => ({ vi.mock("./whois/referral.js", async () => { const client = await import("./whois/client.js"); return { - followWhoisReferrals: vi.fn( - async (server: string, domain: string, opts?: import("./types").LookupOptions) => - client.whoisQuery(server, domain, opts), - ), collectWhoisReferralChain: vi.fn( - async (server: string, domain: string, opts?: import("./types").LookupOptions) => [ - await client.whoisQuery(server, domain, opts), - ], + async (server: string, domain: string, opts?: import("./types").LookupOptions) => ({ + results: [await client.whoisQuery(server, domain, opts)], + warnings: [], + }), ), }; }); @@ -60,7 +57,6 @@ vi.mock("./lib/domain.js", async () => { import { lookup } from "."; import * as rdapClient from "./rdap/client"; -import type { WhoisQueryResult } from "./whois/client"; import * as whoisClient from "./whois/client"; import * as discovery from "./whois/discovery"; import * as whoisReferral from "./whois/referral"; @@ -167,12 +163,15 @@ describe("WHOIS referral & includeRaw", () => { }); it("includes rawWhois when includeRaw is true", async () => { - vi.mocked(whoisReferral.followWhoisReferrals).mockImplementation( - async (_server: string, _domain: string): Promise => ({ - text: "Domain Name: EXAMPLE.COM\nRegistrar: Registrar LLC", - serverQueried: "whois.registrar.test", - }), - ); + vi.mocked(whoisReferral.collectWhoisReferralChain).mockResolvedValueOnce({ + results: [ + { + text: "Domain Name: EXAMPLE.COM\nRegistrar: Registrar LLC", + serverQueried: "whois.registrar.test", + }, + ], + warnings: [], + }); const res = await lookup("example.com", { timeoutMs: 200, diff --git a/src/index.ts b/src/index.ts index 9fe4a8a..23b81e8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,7 +16,8 @@ import type { import { discoverWhoisServer, parseIanaRegistrationInfoUrl } from "./whois/discovery"; import { mergeWhoisRecords } from "./whois/merge"; import { normalizeWhois } from "./whois/normalize"; -import { collectWhoisReferralChain, followWhoisReferrals } from "./whois/referral"; +import { looksEmptyWhois } from "./whois/throttle"; +import { collectWhoisReferralChain } from "./whois/referral"; function failure( ctx: LookupContext, @@ -172,19 +173,12 @@ async function runLookup( // Query the TLD server first; optionally follow registrar referrals (multi-hop) // Collect the chain and coalesce so we don't lose details when a registrar returns minimal/empty data. - const chain = await collectWhoisReferralChain(whoisServer, domain, opts, ctx); - if (chain.length === 0) { - // Fallback to previous behavior as a safety net - const res = await followWhoisReferrals(whoisServer, domain, opts, ctx); - const record: DomainRecord = normalizeWhois( - domain, - tld, - res.text, - res.serverQueried, - !!opts?.includeRaw, - ); - return { ok: true, record, attempts: ctx.attempts }; - } + const { results: chain, warnings } = await collectWhoisReferralChain( + whoisServer, + domain, + opts, + ctx, + ); // Normalize all WHOIS texts in the chain and merge conservatively const normalizedRecords = chain.map((r) => @@ -194,7 +188,19 @@ async function runLookup( if (!first) { return failure(ctx, "no_data", "No WHOIS data retrieved"); } + // A "registered" answer with no fields at all is an error page or throttle notice, not a record + if (first.isRegistered && looksEmptyWhois(first)) { + return failure( + ctx, + "unparseable", + `WHOIS response from ${whoisServer} contained no recognizable domain data`, + { phase: "whois", server: whoisServer }, + ); + } const mergedRecord = rest.length ? mergeWhoisRecords(first, rest) : first; + if (warnings.length) { + mergedRecord.warnings = [...(mergedRecord.warnings ?? []), ...warnings]; + } return { ok: true, record: mergedRecord, attempts: ctx.attempts }; } diff --git a/src/rdap/client.ts b/src/rdap/client.ts index 408d5ee..5408dc6 100644 --- a/src/rdap/client.ts +++ b/src/rdap/client.ts @@ -44,6 +44,13 @@ export async function fetchRdapDomain( if (res.status === 404) { return { url, json: null, notFound: true }; } + if (res.status === 429) { + const retryAfter = res.headers.get("retry-after"); + throw new RdapperError( + "rate_limited", + `RDAP 429 rate limited${retryAfter ? ` (Retry-After: ${retryAfter})` : ""}`, + ); + } if (!res.ok) { const bodyText = await res.text().catch(() => ""); throw new RdapperError("http_error", `RDAP ${res.status}: ${bodyText.slice(0, 500)}`); diff --git a/src/types.ts b/src/types.ts index fb1b4d0..4c20f4f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -392,6 +392,8 @@ export interface LookupResult { * - `http_error`: RDAP responded with a non-2xx status other than 404 * - `rdap_unavailable`: `rdapOnly` was set and no RDAP server worked * - `no_server`: IANA answered but no WHOIS server exists for the TLD + * - `rate_limited`: the server throttled the query (RDAP 429, or a WHOIS throttle notice) + * - `unparseable`: WHOIS replied with text that is neither an availability notice nor a record * - `unsupported_runtime`: WHOIS needs `node:net`, which this runtime lacks */ export type LookupErrorCode = @@ -404,6 +406,8 @@ export type LookupErrorCode = | "rdap_unavailable" | "no_server" | "no_data" + | "rate_limited" + | "unparseable" | "unsupported_runtime" | "unknown"; diff --git a/src/whois/host.test.ts b/src/whois/host.test.ts new file mode 100644 index 0000000..64909f1 --- /dev/null +++ b/src/whois/host.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { isSafeWhoisReferralHost } from "./host"; + +describe("isSafeWhoisReferralHost", () => { + it.each([ + "whois.markmonitor.com", + "whois.1api.net", + "WHOIS.GODADDY.COM", + "whois.nic.xn--p1ai", + "8.8.8.8", + "whois.example.com.", + ])("accepts %s", (h) => expect(isSafeWhoisReferralHost(h)).toBe(true)); + + it.each([ + "", + "localhost", + "foo.localhost", + "printer.local", + "db.internal", + "intranet", + "127.0.0.1", + "10.0.0.5", + "172.16.0.1", + "192.168.1.1", + "169.254.169.254", + "100.64.0.1", + "0.0.0.0", + "::1", + "fe80::1", + "fd00::1", + "::ffff:127.0.0.1", + "2130706433", + "0x7f.1", + "whois.example.com:43", + "user@whois.example.com", + "whois.example.com/path", + "who is.example.com", + "-bad.example.com", + `${"a".repeat(64)}.example.com`, + `${"a.".repeat(130)}com`, + ])("rejects %j", (h) => expect(isSafeWhoisReferralHost(h)).toBe(false)); +}); diff --git a/src/whois/host.ts b/src/whois/host.ts new file mode 100644 index 0000000..10342cd --- /dev/null +++ b/src/whois/host.ts @@ -0,0 +1,54 @@ +import { isIP } from "node:net"; + +const LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i; +const BLOCKED_SUFFIXES = [".local", ".localhost", ".internal", ".localdomain", ".lan", ".home"]; + +function isPrivateIpv4(ip: string): boolean { + const [a = 0, b = 0] = ip.split(".").map(Number); + return ( + a === 0 || + a === 10 || + a === 127 || + (a === 100 && b >= 64 && b <= 127) || // CGNAT + (a === 169 && b === 254) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 168) || + (a === 192 && b === 0) || + (a === 198 && (b === 18 || b === 19)) || + a >= 224 // multicast + reserved + ); +} + +function isPrivateIpv6(ip: string): boolean { + const lower = ip.toLowerCase(); + const mapped = lower.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/); + if (mapped?.[1]) return isPrivateIpv4(mapped[1]); + return ( + lower === "::" || + lower === "::1" || + /^f[cd]/.test(lower) || // unique local + /^fe[89ab]/.test(lower) || // link-local + lower.startsWith("ff") // multicast + ); +} + +/** + * Whether a WHOIS referral host taken from upstream response text is safe to connect to: + * a well-formed public hostname or public IP literal, with no port, path or userinfo. + */ +export function isSafeWhoisReferralHost(host: string): boolean { + const value = host.trim().replace(/\.$/, ""); + if (!value || value.length > 253) return false; + + const ipVersion = isIP(value); + if (ipVersion === 4) return !isPrivateIpv4(value); + if (ipVersion === 6) return !isPrivateIpv6(value); + + const lower = value.toLowerCase(); + if (lower === "localhost" || BLOCKED_SUFFIXES.some((s) => lower.endsWith(s))) return false; + const labels = lower.split("."); + if (labels.length < 2) return false; + if (!labels.every((l) => LABEL.test(l))) return false; + // All-numeric last label means a malformed/obfuscated IP (e.g. 0x7f.1, 2130706433) + return !/^\d+$/.test(labels[labels.length - 1] as string); +} diff --git a/src/whois/merge.test.ts b/src/whois/merge.test.ts index 01f767f..6834dcd 100644 --- a/src/whois/merge.test.ts +++ b/src/whois/merge.test.ts @@ -22,7 +22,7 @@ import { collectWhoisReferralChain } from "./referral"; describe("WHOIS coalescing", () => { it("retains TLD data when registrar provides no details", async () => { - const chain = await collectWhoisReferralChain("whois.nic.io", "gitpod.io", { + const { results: chain } = await collectWhoisReferralChain("whois.nic.io", "gitpod.io", { followWhoisReferral: true, maxWhoisReferralHops: 2, }); diff --git a/src/whois/referral.test.ts b/src/whois/referral.test.ts index b78c46c..50f8685 100644 --- a/src/whois/referral.test.ts +++ b/src/whois/referral.test.ts @@ -32,7 +32,7 @@ describe("WHOIS referral contradiction handling", () => { }); it("collects chain and does not append contradictory registrar", async () => { - const chain = await collectWhoisReferralChain("whois.nic.io", "raindrop.io", { + const { results: chain } = await collectWhoisReferralChain("whois.nic.io", "raindrop.io", { followWhoisReferral: true, maxWhoisReferralHops: 2, }); @@ -42,3 +42,37 @@ describe("WHOIS referral contradiction handling", () => { expect(chain[0]?.serverQueried).toBe("whois.nic.io"); }); }); + +describe("WHOIS referral safety", () => { + it("does not query an unsafe referral host and reports a warning", async () => { + const { whoisQuery } = await import("./client.js"); + const mocked = vi.mocked(whoisQuery); + mocked.mockClear(); + mocked.mockImplementation(async (server: string) => ({ + serverQueried: server, + text: "Domain Name: EVIL.COM\nCreation Date: 2013-08-20T20:30:16Z\nRegistrar WHOIS Server: 169.254.169.254\n", + })); + const { results, warnings } = await collectWhoisReferralChain("whois.nic.io", "evil.com", { + followWhoisReferral: true, + }); + expect(results).toHaveLength(1); + expect(mocked).toHaveBeenCalledTimes(1); + expect(warnings[0]).toMatch(/unsafe host/); + }); + + it("keeps the registry record when the registrar throttles", async () => { + const { whoisQuery } = await import("./client.js"); + vi.mocked(whoisQuery).mockImplementation(async (server: string) => ({ + serverQueried: server, + text: + server === "whois.nic.io" + ? "Domain Name: X.IO\nCreation Date: 2013-08-20T20:30:16Z\nRegistrar WHOIS Server: whois.1api.net\n" + : "WHOIS LIMIT EXCEEDED", + })); + const { results, warnings } = await collectWhoisReferralChain("whois.nic.io", "x.io", { + followWhoisReferral: true, + }); + expect(results).toHaveLength(1); + expect(warnings[0]).toMatch(/rate limited/); + }); +}); diff --git a/src/whois/referral.ts b/src/whois/referral.ts index 3e09191..a326d79 100644 --- a/src/whois/referral.ts +++ b/src/whois/referral.ts @@ -1,10 +1,13 @@ import { throwIfAborted } from "../lib/async"; +import { classifyError, RdapperError } from "../lib/errors"; import { type LookupContext, traced } from "../lib/trace"; import type { LookupOptions } from "../types"; import type { WhoisQueryResult } from "./client"; import { whoisQuery } from "./client"; import { extractWhoisReferral } from "./discovery"; -import { isAvailableByWhois } from "./normalize"; +import { isSafeWhoisReferralHost } from "./host"; +import { isAvailableByWhois, normalizeWhois } from "./normalize"; +import { detectWhoisThrottle, looksEmptyWhois } from "./throttle"; /** * Follow registrar WHOIS referrals up to a configured hop limit. @@ -28,6 +31,7 @@ export async function followWhoisReferrals( throwIfAborted(opts?.signal); const next = extractWhoisReferral(current.text); if (!next) break; + if (!isSafeWhoisReferralHost(normalize(next))) break; const normalized = normalize(next); if (visited.has(normalized)) break; // cycle protection / same as current visited.add(normalized); @@ -61,12 +65,13 @@ export async function collectWhoisReferralChain( domain: string, opts?: LookupOptions, ctx?: LookupContext, -): Promise { +): Promise<{ results: WhoisQueryResult[]; warnings: string[] }> { const results: WhoisQueryResult[] = []; + const warnings: string[] = []; const maxHops = Math.max(0, opts?.maxWhoisReferralHops ?? 2); const first = await tracedWhoisQuery(initialServer, domain, opts, ctx); results.push(first); - if (opts?.followWhoisReferral === false || maxHops === 0) return results; + if (opts?.followWhoisReferral === false || maxHops === 0) return { results, warnings }; const visited = new Set([normalize(first.serverQueried)]); let current = first; @@ -76,6 +81,10 @@ export async function collectWhoisReferralChain( const next = extractWhoisReferral(current.text); if (!next) break; const normalized = normalize(next); + if (!isSafeWhoisReferralHost(normalized)) { + warnings.push(`Skipped WHOIS referral to unsafe host "${next.slice(0, 100)}"`); + break; + } if (visited.has(normalized)) break; visited.add(normalized); try { @@ -87,15 +96,28 @@ export async function collectWhoisReferralChain( // Do not adopt or append contradictory registrar; keep authoritative TLD only. break; } + if ( + registeredAfter && + looksEmptyWhois(normalizeWhois(domain, "", res.text, res.serverQueried)) + ) { + warnings.push(`WHOIS referral ${normalized} returned no usable data`); + break; + } results.push(res); current = res; - } catch { + } catch (err) { throwIfAborted(opts?.signal); + const { code, error } = classifyError(err); + warnings.push( + code === "rate_limited" + ? `WHOIS referral ${normalized} rate limited the query` + : `WHOIS referral ${normalized} failed (${error})`, + ); break; } hops += 1; } - return results; + return { results, warnings }; } function normalize(server: string): string { @@ -115,6 +137,13 @@ function tracedWhoisQuery( async (notes) => { const res = await whoisQuery(server, domain, opts); if (res.partial) notes.partial = true; + if (detectWhoisThrottle(res.text)) { + throw new RdapperError( + "rate_limited", + `WHOIS server ${res.serverQueried} rate limited the query`, + { stage: "read" }, + ); + } return res; }, ); diff --git a/src/whois/throttle.test.ts b/src/whois/throttle.test.ts new file mode 100644 index 0000000..3aeb454 --- /dev/null +++ b/src/whois/throttle.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; +import { detectWhoisThrottle, looksEmptyWhois } from "./throttle"; + +describe("detectWhoisThrottle", () => { + it.each([ + "WHOIS LIMIT EXCEEDED - SEE WWW.PIR.ORG/WHOIS FOR DETAILS", + "Quota exceeded", + "You have exceeded the allowed number of queries. Try again later.", + "Too many requests", + "Service Unavailable", + "Your IP has been blocked", + ])("flags %j", (text) => { + expect(detectWhoisThrottle(text)).toBe(true); + }); + + it("does not flag a long real record that mentions rate limits", () => { + const record = `Domain Name: EXAMPLE.COM\nRegistrar: Test\n${"Name Server: NS.EXAMPLE.COM\n".repeat(100)}\nNote: queries are subject to rate limiting\n`; + expect(detectWhoisThrottle(record)).toBe(false); + }); + + it("does not flag empty or ordinary availability text", () => { + expect(detectWhoisThrottle(undefined)).toBe(false); + expect(detectWhoisThrottle("No match for EXAMPLE.COM")).toBe(false); + }); +}); + +describe("looksEmptyWhois", () => { + const base = { domain: "example.com", tld: "com", isRegistered: true, source: "whois" as const }; + it("is true when nothing useful was parsed", () => { + expect(looksEmptyWhois(base)).toBe(true); + }); + it("is false with any real field", () => { + expect(looksEmptyWhois({ ...base, registrar: { name: "X" } })).toBe(false); + expect(looksEmptyWhois({ ...base, nameservers: [{ host: "ns.example.com" }] })).toBe(false); + }); +}); diff --git a/src/whois/throttle.ts b/src/whois/throttle.ts new file mode 100644 index 0000000..db6d91c --- /dev/null +++ b/src/whois/throttle.ts @@ -0,0 +1,40 @@ +import type { DomainRecord } from "../types"; + +// Real WHOIS records are much longer; throttle notices and error pages are short. +const MAX_THROTTLE_TEXT_LENGTH = 2048; + +const THROTTLE_PATTERNS: RegExp[] = [ + /rate[\s-]?limit/i, + /limit\s+exceeded/i, + /quota\s+exceeded/i, + /exceeded\s+.{0,40}(quer|limit|request|connection)/i, + /too\s+many\s+(quer|request|connection)/i, + /try\s+again\s+(later|in)/i, + /access\s+(denied|limit)/i, + /\b(blocked|blacklisted|banned)\b/i, + /^\s*<(!doctype|html)\b/i, +]; + +/** + * Heuristic: does this WHOIS response look like a throttle notice or error page rather than + * a domain record? Only short responses qualify, so a real record that mentions "rate limit" + * in a remark is not rejected. + */ +export function detectWhoisThrottle(text: string | undefined): boolean { + if (!text) return false; + if (text.length > MAX_THROTTLE_TEXT_LENGTH) return false; + return THROTTLE_PATTERNS.some((re) => re.test(text)); +} + +/** True when a normalized WHOIS record carries none of the fields a real registration would. */ +export function looksEmptyWhois(record: DomainRecord): boolean { + return !( + record.registrar || + record.creationDate || + record.updatedDate || + record.expirationDate || + record.nameservers?.length || + record.statuses?.length || + record.contacts?.length + ); +}