feat: surface retryAfterMs, add blocked error code, and harden private-address guard

- Parse `Retry-After` headers (delay-seconds or HTTP-date) into `retryAfterMs` via a new `parseRetryAfterMs` helper and expose the value on both `LookupAttempt` and `LookupResult` so callers can honour server-requested back-off without parsing the error string
- Propagate `retryAfterMs` through `classifyError`, `traced`, `failure`, and the `rdap_unavailable` path so the value surfaces on the top-level result
- Add `blocked` to `LookupErrorCode` for WHOIS servers that permanently refuse a client (distinct from `rate_limited`, which is a temporary throttle that may succeed on retry)
- Add `blockPrivateAddresses` to `WhoisTransportOptions`; when set, a custom `dns.lookup` shim rejects the connection before it opens if any resolved address is non-public, covering DNS rebinding and hostnames that resolve to private ranges
- Expand `isPrivateIp` / `isSafeWhoisReferralHost` to reject IPv4-mapped (`::ffff:7f00:1`), NAT64 (`64:ff9b::`), 6to4 (`2002:7f00::`), Teredo (`2001:0:`), deprecated site-local (`fec0::`), and bracketed IPv6 literals
This commit is contained in:
2026-09-19 12:25:30 -04:00
parent 208fd6c69f
commit 35959b208b
14 changed files with 269 additions and 112 deletions
+24
View File
@@ -55,6 +55,14 @@ describe("throttle and empty-response guards", () => {
expect(res.errorServer).toBe("whois.verisign-grs.com"); expect(res.errorServer).toBe("whois.verisign-grs.com");
}); });
it("fails with blocked (not rate_limited) for a permanent refusal", async () => {
vi.mocked(whoisQuery).mockImplementation(
ianaThen("Requests of this client are not permitted. Please use https://www.nic.ch/whois/"),
);
const res = await lookup("example.com", { whoisOnly: true });
expect(res.errorCode).toBe("blocked");
});
it("fails with unparseable when a long reply has no fields and no availability phrase", async () => { it("fails with unparseable when a long reply has no fields and no availability phrase", async () => {
vi.mocked(whoisQuery).mockImplementation(ianaThen(`${"lorem ipsum ".repeat(300)}\n`)); vi.mocked(whoisQuery).mockImplementation(ianaThen(`${"lorem ipsum ".repeat(300)}\n`));
const res = await lookup("example.com", { whoisOnly: true }); const res = await lookup("example.com", { whoisOnly: true });
@@ -78,6 +86,22 @@ describe("throttle and empty-response guards", () => {
expect(res.ok, res.error).toBe(true); expect(res.ok, res.error).toBe(true);
expect(res.attempts[0]).toMatchObject({ errorCode: "rate_limited" }); expect(res.attempts[0]).toMatchObject({ errorCode: "rate_limited" });
expect(res.attempts[0]?.error).toContain("Retry-After: 30"); expect(res.attempts[0]?.error).toContain("Retry-After: 30");
expect(res.attempts[0]?.retryAfterMs).toBe(30_000);
});
});
describe("rdapOnly rate limiting", () => {
it("surfaces retryAfterMs on the result", async () => {
const customFetch: FetchLike = vi.fn(
async () => new Response("", { status: 429, headers: { "retry-after": "12" } }),
);
const res = await lookup("example.com", {
customBootstrapData: bootstrap,
customFetch,
rdapOnly: true,
});
expect(res.ok).toBe(false);
expect(res.retryAfterMs).toBe(12_000);
}); });
}); });
+9 -4
View File
@@ -23,7 +23,7 @@ function failure(
ctx: LookupContext, ctx: LookupContext,
errorCode: LookupErrorCode, errorCode: LookupErrorCode,
error: string, error: string,
where?: { phase?: LookupAttempt["phase"]; server?: string }, where?: { phase?: LookupAttempt["phase"]; server?: string; retryAfterMs?: number },
): LookupResult { ): LookupResult {
return { return {
ok: false, ok: false,
@@ -31,6 +31,7 @@ function failure(
errorCode, errorCode,
...(where?.phase ? { errorPhase: where.phase } : {}), ...(where?.phase ? { errorPhase: where.phase } : {}),
...(where?.server ? { errorServer: where.server } : {}), ...(where?.server ? { errorServer: where.server } : {}),
...(where?.retryAfterMs !== undefined ? { retryAfterMs: where.retryAfterMs } : {}),
attempts: ctx.attempts, attempts: ctx.attempts,
}; };
} }
@@ -70,9 +71,13 @@ export async function lookup(domain: string, opts?: LookupOptions): Promise<Look
try { try {
return await runLookup(domain, signalOpts, ctx); return await runLookup(domain, signalOpts, ctx);
} catch (err: unknown) { } catch (err: unknown) {
const { code, error } = classifyError(err); const { code, error, retryAfterMs } = classifyError(err);
const source = attemptForError(err); const source = attemptForError(err);
return failure(ctx, code, error, { phase: source?.phase, server: source?.server }); return failure(ctx, code, error, {
phase: source?.phase,
server: source?.server,
retryAfterMs,
});
} finally { } finally {
if (deadlineTimer !== undefined) clearTimeout(deadlineTimer); if (deadlineTimer !== undefined) clearTimeout(deadlineTimer);
link?.dispose(); link?.dispose();
@@ -142,7 +147,7 @@ async function runLookup(
ctx, ctx,
"rdap_unavailable", "rdap_unavailable",
`RDAP not available or failed for TLD '${tld}'${detail}. Many TLDs do not publish RDAP; try WHOIS fallback (omit rdapOnly).`, `RDAP not available or failed for TLD '${tld}'${detail}. Many TLDs do not publish RDAP; try WHOIS fallback (omit rdapOnly).`,
{ phase: last?.phase, server: last?.server }, { phase: last?.phase, server: last?.server, retryAfterMs: last?.retryAfterMs },
); );
} }
} }
+15 -2
View File
@@ -9,6 +9,7 @@ export class RdapperError extends Error {
readonly phase?: LookupAttempt["phase"]; readonly phase?: LookupAttempt["phase"];
readonly server?: string; readonly server?: string;
readonly stage?: "connect" | "read"; readonly stage?: "connect" | "read";
readonly retryAfterMs?: number;
constructor( constructor(
code: LookupErrorCode, code: LookupErrorCode,
@@ -17,6 +18,7 @@ export class RdapperError extends Error {
phase?: LookupAttempt["phase"]; phase?: LookupAttempt["phase"];
server?: string; server?: string;
stage?: "connect" | "read"; stage?: "connect" | "read";
retryAfterMs?: number;
cause?: unknown; cause?: unknown;
}, },
) { ) {
@@ -26,6 +28,7 @@ export class RdapperError extends Error {
this.phase = extra?.phase; this.phase = extra?.phase;
this.server = extra?.server; this.server = extra?.server;
this.stage = extra?.stage; this.stage = extra?.stage;
this.retryAfterMs = extra?.retryAfterMs;
} }
} }
@@ -74,8 +77,18 @@ function describeError(err: unknown, errno: string | undefined): string {
} }
/** Map any thrown value to a stable error code plus a human-readable message. */ /** Map any thrown value to a stable error code plus a human-readable message. */
export function classifyError(err: unknown): { code: LookupErrorCode; error: string } { export function classifyError(err: unknown): {
if (err instanceof RdapperError) return { code: err.code, error: err.message }; code: LookupErrorCode;
error: string;
retryAfterMs?: number;
} {
if (err instanceof RdapperError) {
return {
code: err.code,
error: err.message,
...(err.retryAfterMs !== undefined ? { retryAfterMs: err.retryAfterMs } : {}),
};
}
const name = err instanceof Error ? err.name : ""; const name = err instanceof Error ? err.name : "";
const cause = err instanceof Error ? (err as { cause?: unknown }).cause : undefined; const cause = err instanceof Error ? (err as { cause?: unknown }).cause : undefined;
+2 -1
View File
@@ -33,13 +33,14 @@ export async function traced<T>(
ctx.attempts.push({ ...meta, ok: true, durationMs: Date.now() - start, ...notes }); ctx.attempts.push({ ...meta, ok: true, durationMs: Date.now() - start, ...notes });
return result; return result;
} catch (err) { } catch (err) {
const { code, error } = classifyError(err); const { code, error, retryAfterMs } = classifyError(err);
const attempt: LookupAttempt = { const attempt: LookupAttempt = {
...meta, ...meta,
ok: false, ok: false,
durationMs: Date.now() - start, durationMs: Date.now() - start,
errorCode: code, errorCode: code,
error, error,
...(retryAfterMs !== undefined ? { retryAfterMs } : {}),
...(err instanceof RdapperError && err.stage ? { stage: err.stage } : {}), ...(err instanceof RdapperError && err.stage ? { stage: err.stage } : {}),
...notes, ...notes,
}; };
+11
View File
@@ -15,6 +15,15 @@ export interface RdapFetchResult {
notFound?: boolean; notFound?: boolean;
} }
/** Parse a `Retry-After` header (delay-seconds or HTTP date) into milliseconds. */
export function parseRetryAfterMs(value: string | null | undefined): number | undefined {
const v = value?.trim();
if (!v) return undefined;
if (/^\d+$/.test(v)) return Number(v) * 1000;
const at = Date.parse(v);
return Number.isNaN(at) ? undefined : Math.max(0, at - Date.now());
}
/** /**
* Fetch RDAP JSON for a domain from a specific RDAP base URL. * Fetch RDAP JSON for a domain from a specific RDAP base URL.
* Returns `{ notFound: true }` for HTTP 404 (domain not registered). * Returns `{ notFound: true }` for HTTP 404 (domain not registered).
@@ -46,9 +55,11 @@ export async function fetchRdapDomain(
} }
if (res.status === 429) { if (res.status === 429) {
const retryAfter = res.headers.get("retry-after"); const retryAfter = res.headers.get("retry-after");
const retryAfterMs = parseRetryAfterMs(retryAfter);
throw new RdapperError( throw new RdapperError(
"rate_limited", "rate_limited",
`RDAP 429 rate limited${retryAfter ? ` (Retry-After: ${retryAfter})` : ""}`, `RDAP 429 rate limited${retryAfter ? ` (Retry-After: ${retryAfter})` : ""}`,
retryAfterMs !== undefined ? { retryAfterMs } : undefined,
); );
} }
if (!res.ok) { if (!res.ok) {
+6
View File
@@ -379,6 +379,8 @@ export interface LookupResult {
errorPhase?: LookupAttempt["phase"]; errorPhase?: LookupAttempt["phase"];
/** Server (RDAP URL or WHOIS host) involved in the failure, when known */ /** Server (RDAP URL or WHOIS host) involved in the failure, when known */
errorServer?: string; errorServer?: string;
/** Suggested wait before retrying, from an RDAP `Retry-After` header on a `rate_limited` failure */
retryAfterMs?: number;
/** Every network attempt made during the lookup, in order (successes and failures) */ /** Every network attempt made during the lookup, in order (successes and failures) */
attempts: LookupAttempt[]; attempts: LookupAttempt[];
} }
@@ -393,6 +395,7 @@ export interface LookupResult {
* - `rdap_unavailable`: `rdapOnly` was set and no RDAP server worked * - `rdap_unavailable`: `rdapOnly` was set and no RDAP server worked
* - `no_server`: IANA answered but no WHOIS server exists for the TLD * - `no_server`: IANA answered but no WHOIS server exists for the TLD
* - `rate_limited`: the server throttled the query (RDAP 429, or a WHOIS throttle notice) * - `rate_limited`: the server throttled the query (RDAP 429, or a WHOIS throttle notice)
* - `blocked`: the WHOIS server refuses this client outright (retrying will not help)
* - `unparseable`: WHOIS replied with text that is neither an availability notice nor a record * - `unparseable`: WHOIS replied with text that is neither an availability notice nor a record
* - `unsupported_runtime`: WHOIS needs `node:net`, which this runtime lacks * - `unsupported_runtime`: WHOIS needs `node:net`, which this runtime lacks
*/ */
@@ -407,6 +410,7 @@ export type LookupErrorCode =
| "no_server" | "no_server"
| "no_data" | "no_data"
| "rate_limited" | "rate_limited"
| "blocked"
| "unparseable" | "unparseable"
| "unsupported_runtime" | "unsupported_runtime"
| "unknown"; | "unknown";
@@ -420,6 +424,8 @@ export interface LookupAttempt {
durationMs: number; durationMs: number;
errorCode?: LookupErrorCode; errorCode?: LookupErrorCode;
error?: string; error?: string;
/** `rate_limited` RDAP failures only: the server's `Retry-After`, in milliseconds */
retryAfterMs?: number;
/** WHOIS failures only: `connect` if the socket never connected; `read` if it connected but sent nothing (timeout, or closed without a response, which is `no_data`) */ /** WHOIS failures only: `connect` if the socket never connected; `read` if it connected but sent nothing (timeout, or closed without a response, which is `no_data`) */
stage?: "connect" | "read"; stage?: "connect" | "read";
/** WHOIS only: the read timed out after some data arrived, so the text is partial */ /** WHOIS only: the read timed out after some data arrived, so the text is partial */
+13
View File
@@ -0,0 +1,13 @@
import { describe, expect, it } from "vitest";
import { whoisQuery } from "./client";
describe("blockPrivateAddresses", () => {
it("refuses to connect when the host resolves to loopback", async () => {
await expect(
whoisQuery("localhost", "example.com", { timeoutMs: 2000 }, { blockPrivateAddresses: true }),
).rejects.toMatchObject({
code: "connect_failed",
message: expect.stringContaining("non-public"),
});
});
});
+46 -2
View File
@@ -1,6 +1,7 @@
import { resolveTimeoutMs, throwIfAborted } from "../lib/async"; import { resolveTimeoutMs, throwIfAborted } from "../lib/async";
import { abortError, RdapperError } from "../lib/errors"; import { abortError, RdapperError } from "../lib/errors";
import type { LookupOptions } from "../types"; import type { LookupOptions } from "../types";
import { isPrivateIp } from "./host";
export interface WhoisQueryResult { export interface WhoisQueryResult {
serverQueried: string; serverQueried: string;
@@ -20,6 +21,15 @@ const WHOIS_QUERY_TRANSFORMERS: Record<string, (query: string) => string> = {
"whois.jprs.jp": (query) => `${query}/e`, // Append /e for English-only response "whois.jprs.jp": (query) => `${query}/e`, // Append /e for English-only response
}; };
export interface WhoisTransportOptions {
/**
* Reject any resolved address that is not public unicast, checked at connect time so it covers
* DNS rebinding and hostnames that resolve to private ranges. Used for referral hosts, which
* come from upstream response text.
*/
blockPrivateAddresses?: boolean;
}
/** /**
* Perform a WHOIS query against an RFC 3912 server over TCP 43. * Perform a WHOIS query against an RFC 3912 server over TCP 43.
* Returns the raw text and the server used. * Returns the raw text and the server used.
@@ -28,6 +38,7 @@ export async function whoisQuery(
server: string, server: string,
query: string, query: string,
options?: LookupOptions, options?: LookupOptions,
transport?: WhoisTransportOptions,
): Promise<WhoisQueryResult> { ): Promise<WhoisQueryResult> {
const port = 43; const port = 43;
const host = server.replace(/^whois:\/\//i, ""); const host = server.replace(/^whois:\/\//i, "");
@@ -36,7 +47,7 @@ export async function whoisQuery(
const transformer = WHOIS_QUERY_TRANSFORMERS[host]; const transformer = WHOIS_QUERY_TRANSFORMERS[host];
const transformedQuery = transformer ? transformer(query) : query; const transformedQuery = transformer ? transformer(query) : query;
const { text, partial } = await queryTcp(host, port, transformedQuery, options); const { text, partial } = await queryTcp(host, port, transformedQuery, options, transport);
return { serverQueried: server, text, ...(partial ? { partial } : {}) }; return { serverQueried: server, text, ...(partial ? { partial } : {}) };
} }
@@ -48,6 +59,7 @@ async function queryTcp(
port: number, port: number,
query: string, query: string,
options?: LookupOptions, options?: LookupOptions,
transport?: WhoisTransportOptions,
): Promise<{ text: string; partial?: boolean }> { ): Promise<{ text: string; partial?: boolean }> {
let net: typeof import("node:net") | null; let net: typeof import("node:net") | null;
try { try {
@@ -67,9 +79,16 @@ async function queryTcp(
throwIfAborted(signal); throwIfAborted(signal);
const timeoutMs = resolveTimeoutMs(options); const timeoutMs = resolveTimeoutMs(options);
const createConnection = net.createConnection; const createConnection = net.createConnection;
const guardedLookup = transport?.blockPrivateAddresses
? await privateBlockingLookup()
: undefined;
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const socket = createConnection({ host, port }); const socket = createConnection({
host,
port,
...(guardedLookup ? { lookup: guardedLookup } : {}),
});
const chunks: Buffer[] = []; const chunks: Buffer[] = [];
let received = 0; let received = 0;
let connected = false; let connected = false;
@@ -165,3 +184,28 @@ async function queryTcp(
}); });
}); });
} }
/** A `dns.lookup` replacement that fails the connection when any answer is a non-public address. */
async function privateBlockingLookup(): Promise<
NonNullable<import("node:net").TcpNetConnectOpts["lookup"]>
> {
const dns = await import("node:dns");
return (hostname, options, callback) => {
dns.lookup(hostname, options, (err, address, family) => {
if (err) return callback(err, address as never, family as never);
const answers = Array.isArray(address) ? address.map((a) => a.address) : [address];
const bad = answers.find((a) => isPrivateIp(a));
if (bad) {
return callback(
new RdapperError(
"connect_failed",
`WHOIS host ${hostname} resolves to a non-public address`,
),
"" as never,
4 as never,
);
}
callback(err, address as never, family as never);
});
};
}
+11
View File
@@ -8,6 +8,8 @@ describe("isSafeWhoisReferralHost", () => {
"WHOIS.GODADDY.COM", "WHOIS.GODADDY.COM",
"whois.nic.xn--p1ai", "whois.nic.xn--p1ai",
"8.8.8.8", "8.8.8.8",
"2606:4700:4700::1111",
"::ffff:808:808",
"whois.example.com.", "whois.example.com.",
])("accepts %s", (h) => expect(isSafeWhoisReferralHost(h)).toBe(true)); ])("accepts %s", (h) => expect(isSafeWhoisReferralHost(h)).toBe(true));
@@ -29,6 +31,15 @@ describe("isSafeWhoisReferralHost", () => {
"fe80::1", "fe80::1",
"fd00::1", "fd00::1",
"::ffff:127.0.0.1", "::ffff:127.0.0.1",
"::ffff:7f00:1",
"::ffff:a00:1",
"64:ff9b::7f00:1",
"::127.0.0.1",
"2002:7f00:1::",
"2001:0:4136:e378:8000:63bf:3fff:fdd2",
"fec0::1",
"[::1]",
"1:2:3:4:5:6:7:8:9",
"2130706433", "2130706433",
"0x7f.1", "0x7f.1",
"whois.example.com:43", "whois.example.com:43",
+74 -15
View File
@@ -1,10 +1,52 @@
import { isIP } from "node:net"; // No `node:` imports here: this module is loaded in every runtime, WHOIS transport is not.
const LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i; const LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i;
const BLOCKED_SUFFIXES = [".local", ".localhost", ".internal", ".localdomain", ".lan", ".home"]; const BLOCKED_SUFFIXES = [".local", ".localhost", ".internal", ".localdomain", ".lan", ".home"];
function isPrivateIpv4(ip: string): boolean { function parseIpv4(ip: string): number[] | null {
const [a = 0, b = 0] = ip.split(".").map(Number); const parts = ip.split(".");
if (parts.length !== 4) return null;
const out: number[] = [];
for (const p of parts) {
if (!/^(0|[1-9]\d{0,2})$/.test(p)) return null;
const n = Number(p);
if (n > 255) return null;
out.push(n);
}
return out;
}
/** Parse an IPv6 literal into eight 16-bit groups (no zone ids, no brackets). */
function parseIpv6(ip: string): number[] | null {
if (!ip.includes(":") || ip.includes("%")) return null;
let text = ip;
// Embedded dotted IPv4 tail, e.g. ::ffff:127.0.0.1
const tail = text.match(/:(\d+\.\d+\.\d+\.\d+)$/);
if (tail?.[1]) {
const v4 = parseIpv4(tail[1]);
if (!v4) return null;
const hex = (a: number, b: number) => ((a << 8) | b).toString(16);
text = `${text.slice(0, -tail[1].length)}${hex(v4[0] as number, v4[1] as number)}:${hex(v4[2] as number, v4[3] as number)}`;
}
const halves = text.split("::");
if (halves.length > 2) return null;
const toGroups = (s: string) => (s === "" ? [] : s.split(":"));
const head = toGroups(halves[0] as string);
const rest = halves.length === 2 ? toGroups(halves[1] as string) : [];
if (halves.length === 1 && head.length !== 8) return null;
if (halves.length === 2 && head.length + rest.length > 7) return null;
const all = [
...head,
...Array(halves.length === 2 ? 8 - head.length - rest.length : 0).fill("0"),
...rest,
];
if (all.length !== 8) return null;
const nums = all.map((g) => (/^[0-9a-f]{1,4}$/i.test(g) ? Number.parseInt(g, 16) : Number.NaN));
return nums.some(Number.isNaN) ? null : nums;
}
function isPrivateIpv4(o: number[]): boolean {
const [a = 0, b = 0] = o;
return ( return (
a === 0 || a === 0 ||
a === 10 || a === 10 ||
@@ -19,30 +61,47 @@ function isPrivateIpv4(ip: string): boolean {
); );
} }
function isPrivateIpv6(ip: string): boolean { const v4From = (hi: number, lo: number) => [hi >> 8, hi & 0xff, lo >> 8, lo & 0xff];
const lower = ip.toLowerCase();
const mapped = lower.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/); function isPrivateIpv6(g: number[]): boolean {
if (mapped?.[1]) return isPrivateIpv4(mapped[1]); const [g0 = 0, g1 = 0, g2 = 0, g3 = 0, g4 = 0, g5 = 0, g6 = 0, g7 = 0] = g;
const embedded = () => isPrivateIpv4(v4From(g6, g7));
const first96Zero = g0 === 0 && g1 === 0 && g2 === 0 && g3 === 0 && g4 === 0;
if (first96Zero && g5 === 0) return true; // ::, ::1, ::a.b.c.d (IPv4-compatible)
if (first96Zero && g5 === 0xffff) return embedded(); // ::ffff:a.b.c.d (IPv4-mapped)
if (g0 === 0x64 && g1 === 0xff9b && g2 === 0 && g3 === 0 && g4 === 0 && g5 === 0) {
return embedded(); // NAT64 64:ff9b::/96
}
if (g0 === 0x2002) return isPrivateIpv4(v4From(g1, g2)); // 6to4
if (g0 === 0x2001 && g1 === 0) return true; // Teredo: embeds arbitrary addresses
if (g0 === 0x2001 && g1 === 0xdb8) return true; // documentation
return ( return (
lower === "::" || (g0 & 0xfe00) === 0xfc00 || // unique local fc00::/7
lower === "::1" || (g0 & 0xffc0) === 0xfe80 || // link-local
/^f[cd]/.test(lower) || // unique local (g0 & 0xffc0) === 0xfec0 || // site-local
/^fe[89ab]/.test(lower) || // link-local (g0 & 0xff00) === 0xff00 || // multicast
lower.startsWith("ff") // multicast (g0 === 0x100 && g1 === 0 && g2 === 0 && g3 === 0) // discard-only
); );
} }
/** True for an IP literal that is not a public unicast address (or that cannot be parsed). */
export function isPrivateIp(ip: string): boolean {
const v4 = parseIpv4(ip);
if (v4) return isPrivateIpv4(v4);
const v6 = parseIpv6(ip);
return v6 ? isPrivateIpv6(v6) : true;
}
/** /**
* Whether a WHOIS referral host taken from upstream response text is safe to connect to: * Whether a WHOIS referral host taken from upstream response text is safe to connect to:
* a well-formed public hostname or public IP literal, with no port, path or userinfo. * a well-formed public hostname or public IP literal, with no port, path or userinfo.
* This is a literal check only; the resolved address is checked again at connect time.
*/ */
export function isSafeWhoisReferralHost(host: string): boolean { export function isSafeWhoisReferralHost(host: string): boolean {
const value = host.trim().replace(/\.$/, ""); const value = host.trim().replace(/\.$/, "");
if (!value || value.length > 253) return false; if (!value || value.length > 253) return false;
const ipVersion = isIP(value); if (parseIpv4(value) || value.includes(":")) return !isPrivateIp(value);
if (ipVersion === 4) return !isPrivateIpv4(value);
if (ipVersion === 6) return !isPrivateIpv6(value);
const lower = value.toLowerCase(); const lower = value.toLowerCase();
if (lower === "localhost" || BLOCKED_SUFFIXES.some((s) => lower.endsWith(s))) return false; if (lower === "localhost" || BLOCKED_SUFFIXES.some((s) => lower.endsWith(s))) return false;
+1 -12
View File
@@ -17,20 +17,9 @@ vi.mock("./client.js", () => ({
}), }),
})); }));
import { collectWhoisReferralChain, followWhoisReferrals } from "./referral"; import { collectWhoisReferralChain } from "./referral";
describe("WHOIS referral contradiction handling", () => { describe("WHOIS referral contradiction handling", () => {
it("keeps TLD WHOIS when registrar claims availability", async () => {
const res = await followWhoisReferrals("whois.nic.io", "raindrop.io", {
followWhoisReferral: true,
maxWhoisReferralHops: 2,
});
expect(res.serverQueried).toBe("whois.nic.io");
// ensure we didn't adopt the registrar response
expect(res.text.toLowerCase().includes("creation date")).toBe(true);
expect(res.text.toLowerCase().includes("no match")).toBe(false);
});
it("collects chain and does not append contradictory registrar", async () => { it("collects chain and does not append contradictory registrar", async () => {
const { results: chain } = await collectWhoisReferralChain("whois.nic.io", "raindrop.io", { const { results: chain } = await collectWhoisReferralChain("whois.nic.io", "raindrop.io", {
followWhoisReferral: true, followWhoisReferral: true,
+12 -54
View File
@@ -7,53 +7,7 @@ import { whoisQuery } from "./client";
import { extractWhoisReferral } from "./discovery"; import { extractWhoisReferral } from "./discovery";
import { isSafeWhoisReferralHost } from "./host"; import { isSafeWhoisReferralHost } from "./host";
import { isAvailableByWhois, normalizeWhois } from "./normalize"; import { isAvailableByWhois, normalizeWhois } from "./normalize";
import { detectWhoisThrottle, looksEmptyWhois } from "./throttle"; import { detectWhoisRefusal, looksEmptyWhois } from "./throttle";
/**
* Follow registrar WHOIS referrals up to a configured hop limit.
* Returns the last successful WHOIS response (best-effort; keeps original on failures).
*/
export async function followWhoisReferrals(
initialServer: string,
domain: string,
opts?: LookupOptions,
ctx?: LookupContext,
): Promise<WhoisQueryResult> {
const maxHops = Math.max(0, opts?.maxWhoisReferralHops ?? 2);
// First query against the provided server
let current = await tracedWhoisQuery(initialServer, domain, opts, ctx);
if (opts?.followWhoisReferral === false || maxHops === 0) return current;
const visited = new Set<string>([normalize(current.serverQueried)]);
let hops = 0;
// Iterate while we see a new referral and are under hop limit
while (hops < maxHops) {
throwIfAborted(opts?.signal);
const next = extractWhoisReferral(current.text);
if (!next) break;
if (!isSafeWhoisReferralHost(normalize(next))) break;
const normalized = normalize(next);
if (visited.has(normalized)) break; // cycle protection / same as current
visited.add(normalized);
try {
const res = await tracedWhoisQuery(next, domain, opts, ctx);
// Prefer authoritative TLD response when registrar contradicts availability
const registeredBefore = !isAvailableByWhois(current.text);
const registeredAfter = !isAvailableByWhois(res.text);
if (registeredBefore && !registeredAfter) {
// Registrar claims availability but TLD shows registered: keep TLD
break;
}
current = res; // adopt registrar when it does not downgrade registration
} catch {
throwIfAborted(opts?.signal);
// If referral server fails, stop following and keep the last good response
break;
}
hops += 1;
}
return current;
}
/** /**
* Collect the WHOIS referral chain starting from the TLD server. * Collect the WHOIS referral chain starting from the TLD server.
@@ -88,7 +42,7 @@ export async function collectWhoisReferralChain(
if (visited.has(normalized)) break; if (visited.has(normalized)) break;
visited.add(normalized); visited.add(normalized);
try { try {
const res = await tracedWhoisQuery(next, domain, opts, ctx); const res = await tracedWhoisQuery(next, domain, opts, ctx, true);
// If registrar claims availability while TLD indicated registered, stop. // If registrar claims availability while TLD indicated registered, stop.
const registeredBefore = !isAvailableByWhois(current.text); const registeredBefore = !isAvailableByWhois(current.text);
const registeredAfter = !isAvailableByWhois(res.text); const registeredAfter = !isAvailableByWhois(res.text);
@@ -109,8 +63,8 @@ export async function collectWhoisReferralChain(
throwIfAborted(opts?.signal); throwIfAborted(opts?.signal);
const { code, error } = classifyError(err); const { code, error } = classifyError(err);
warnings.push( warnings.push(
code === "rate_limited" code === "rate_limited" || code === "blocked"
? `WHOIS referral ${normalized} rate limited the query` ? `WHOIS referral ${normalized} ${code === "blocked" ? "blocked" : "rate limited"} the query`
: `WHOIS referral ${normalized} failed (${error})`, : `WHOIS referral ${normalized} failed (${error})`,
); );
break; break;
@@ -130,17 +84,21 @@ function tracedWhoisQuery(
domain: string, domain: string,
opts?: LookupOptions, opts?: LookupOptions,
ctx?: LookupContext, ctx?: LookupContext,
referral = false,
): Promise<WhoisQueryResult> { ): Promise<WhoisQueryResult> {
return traced( return traced(
ctx, ctx,
{ phase: "whois", server: server.replace(/^whois:\/\//i, "") }, { phase: "whois", server: server.replace(/^whois:\/\//i, "") },
async (notes) => { async (notes) => {
const res = await whoisQuery(server, domain, opts); const res = await whoisQuery(server, domain, opts, { blockPrivateAddresses: referral });
if (res.partial) notes.partial = true; if (res.partial) notes.partial = true;
if (detectWhoisThrottle(res.text)) { const refusal = detectWhoisRefusal(res.text);
if (refusal) {
throw new RdapperError( throw new RdapperError(
"rate_limited", refusal,
`WHOIS server ${res.serverQueried} rate limited the query`, refusal === "blocked"
? `WHOIS server ${res.serverQueried} refuses requests from this client`
: `WHOIS server ${res.serverQueried} rate limited the query`,
{ stage: "read" }, { stage: "read" },
); );
} }
+24 -10
View File
@@ -1,26 +1,40 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { detectWhoisThrottle, looksEmptyWhois } from "./throttle"; import { detectWhoisRefusal, looksEmptyWhois } from "./throttle";
describe("detectWhoisThrottle", () => { describe("detectWhoisRefusal", () => {
it.each([ it.each([
"WHOIS LIMIT EXCEEDED - SEE WWW.PIR.ORG/WHOIS FOR DETAILS", "WHOIS LIMIT EXCEEDED - SEE WWW.PIR.ORG/WHOIS FOR DETAILS",
"Quota exceeded", "Quota exceeded",
"You have exceeded the allowed number of queries. Try again later.", "You have exceeded the allowed number of queries.",
"Too many requests", "Too many requests",
"<!DOCTYPE html><html><body>Service Unavailable</body></html>", "<!DOCTYPE html><html><body>Service Unavailable</body></html>",
"Your IP has been blocked", ])("classifies %j as rate_limited", (text) => {
])("flags %j", (text) => { expect(detectWhoisRefusal(text)).toBe("rate_limited");
expect(detectWhoisThrottle(text)).toBe(true); });
it.each([
"Requests of this client are not permitted. Please use https://www.nic.ch/whois/ for queries.",
"Your IP address has been blocked",
])("classifies %j as blocked", (text) => {
expect(detectWhoisRefusal(text)).toBe("blocked");
});
it.each([
"This domain name is blocked. Try again later.",
"Access denied for reserved name. No match for EXAMPLE.COM",
"No match for EXAMPLE.COM",
"Domain is banned by policy",
])("leaves %j alone", (text) => {
expect(detectWhoisRefusal(text)).toBeUndefined();
}); });
it("does not flag a long real record that mentions rate limits", () => { it("does not flag a long real record that mentions rate limits", () => {
const record = `Domain Name: EXAMPLE.COM\nRegistrar: Test\n${"Name Server: NS.EXAMPLE.COM\n".repeat(100)}\nNote: queries are subject to rate limiting\n`; const record = `Domain Name: EXAMPLE.COM\nRegistrar: Test\n${"Name Server: NS.EXAMPLE.COM\n".repeat(100)}\nNote: queries are subject to rate limiting\n`;
expect(detectWhoisThrottle(record)).toBe(false); expect(detectWhoisRefusal(record)).toBeUndefined();
}); });
it("does not flag empty or ordinary availability text", () => { it("returns undefined for empty input", () => {
expect(detectWhoisThrottle(undefined)).toBe(false); expect(detectWhoisRefusal(undefined)).toBeUndefined();
expect(detectWhoisThrottle("No match for EXAMPLE.COM")).toBe(false);
}); });
}); });
+21 -12
View File
@@ -1,29 +1,38 @@
import type { DomainRecord } from "../types"; import type { DomainRecord } from "../types";
import { isAvailableByWhois } from "./normalize";
// Real WHOIS records are much longer; throttle notices and error pages are short. // Real WHOIS records are much longer; refusal notices and error pages are short.
const MAX_THROTTLE_TEXT_LENGTH = 2048; const MAX_REFUSAL_TEXT_LENGTH = 2048;
// Transient: the same query may succeed later.
const THROTTLE_PATTERNS: RegExp[] = [ const THROTTLE_PATTERNS: RegExp[] = [
/rate[\s-]?limit/i, /rate[\s-]?limit/i,
/limit\s+exceeded/i, /limit\s+exceeded/i,
/quota\s+exceeded/i, /quota\s+exceeded/i,
/exceeded\s+.{0,40}(quer|limit|request|connection)/i, /exceeded\s+.{0,40}(quer|limit|request|connection)/i,
/too\s+many\s+(quer|request|connection)/i, /too\s+many\s+(quer|request|connection)/i,
/try\s+again\s+(later|in)/i,
/access\s+(denied|limit)/i,
/\b(blocked|blacklisted|banned)\b/i,
/^\s*<(!doctype|html)\b/i, /^\s*<(!doctype|html)\b/i,
]; ];
// Permanent: this client is refused outright, so retrying will not help.
const BLOCK_PATTERNS: RegExp[] = [
/requests\s+of\s+this\s+client\s+are\s+not\s+permitted/i, // .ch/.li
/\b(your|this)\s+(ip|address|client)\b.{0,60}\b(blocked|banned|blacklisted|not\s+permitted)\b/i,
];
/** /**
* Heuristic: does this WHOIS response look like a throttle notice or error page rather than * Classify a short WHOIS reply that is a refusal rather than a record: `rate_limited` for a
* a domain record? Only short responses qualify, so a real record that mentions "rate limit" * transient throttle, `blocked` for a permanent block. Availability notices win, and long
* in a remark is not rejected. * responses are never refusals, so a real record that mentions "rate limit" is not rejected.
*/ */
export function detectWhoisThrottle(text: string | undefined): boolean { export function detectWhoisRefusal(
if (!text) return false; text: string | undefined,
if (text.length > MAX_THROTTLE_TEXT_LENGTH) return false; ): "rate_limited" | "blocked" | undefined {
return THROTTLE_PATTERNS.some((re) => re.test(text)); if (!text || text.length > MAX_REFUSAL_TEXT_LENGTH) return undefined;
if (isAvailableByWhois(text)) return undefined;
if (BLOCK_PATTERNS.some((re) => re.test(text))) return "blocked";
if (THROTTLE_PATTERNS.some((re) => re.test(text))) return "rate_limited";
return undefined;
} }
/** True when a normalized WHOIS record carries none of the fields a real registration would. */ /** True when a normalized WHOIS record carries none of the fields a real registration would. */