mirror of
https://github.com/jakejarvis/rdapper.git
synced 2026-09-22 23:05:31 -04:00
feat: surface retryAfterMs, add blocked error code, and harden private-address guard
- Parse `Retry-After` headers (delay-seconds or HTTP-date) into `retryAfterMs` via a new `parseRetryAfterMs` helper and expose the value on both `LookupAttempt` and `LookupResult` so callers can honour server-requested back-off without parsing the error string - Propagate `retryAfterMs` through `classifyError`, `traced`, `failure`, and the `rdap_unavailable` path so the value surfaces on the top-level result - Add `blocked` to `LookupErrorCode` for WHOIS servers that permanently refuse a client (distinct from `rate_limited`, which is a temporary throttle that may succeed on retry) - Add `blockPrivateAddresses` to `WhoisTransportOptions`; when set, a custom `dns.lookup` shim rejects the connection before it opens if any resolved address is non-public, covering DNS rebinding and hostnames that resolve to private ranges - Expand `isPrivateIp` / `isSafeWhoisReferralHost` to reject IPv4-mapped (`::ffff:7f00:1`), NAT64 (`64:ff9b::`), 6to4 (`2002:7f00::`), Teredo (`2001:0:`), deprecated site-local (`fec0::`), and bracketed IPv6 literals
This commit is contained in:
@@ -55,6 +55,14 @@ describe("throttle and empty-response guards", () => {
|
||||
expect(res.errorServer).toBe("whois.verisign-grs.com");
|
||||
});
|
||||
|
||||
it("fails with blocked (not rate_limited) for a permanent refusal", async () => {
|
||||
vi.mocked(whoisQuery).mockImplementation(
|
||||
ianaThen("Requests of this client are not permitted. Please use https://www.nic.ch/whois/"),
|
||||
);
|
||||
const res = await lookup("example.com", { whoisOnly: true });
|
||||
expect(res.errorCode).toBe("blocked");
|
||||
});
|
||||
|
||||
it("fails with unparseable when a long reply has no fields and no availability phrase", async () => {
|
||||
vi.mocked(whoisQuery).mockImplementation(ianaThen(`${"lorem ipsum ".repeat(300)}\n`));
|
||||
const res = await lookup("example.com", { whoisOnly: true });
|
||||
@@ -78,6 +86,22 @@ describe("throttle and empty-response guards", () => {
|
||||
expect(res.ok, res.error).toBe(true);
|
||||
expect(res.attempts[0]).toMatchObject({ errorCode: "rate_limited" });
|
||||
expect(res.attempts[0]?.error).toContain("Retry-After: 30");
|
||||
expect(res.attempts[0]?.retryAfterMs).toBe(30_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe("rdapOnly rate limiting", () => {
|
||||
it("surfaces retryAfterMs on the result", async () => {
|
||||
const customFetch: FetchLike = vi.fn(
|
||||
async () => new Response("", { status: 429, headers: { "retry-after": "12" } }),
|
||||
);
|
||||
const res = await lookup("example.com", {
|
||||
customBootstrapData: bootstrap,
|
||||
customFetch,
|
||||
rdapOnly: true,
|
||||
});
|
||||
expect(res.ok).toBe(false);
|
||||
expect(res.retryAfterMs).toBe(12_000);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+9
-4
@@ -23,7 +23,7 @@ function failure(
|
||||
ctx: LookupContext,
|
||||
errorCode: LookupErrorCode,
|
||||
error: string,
|
||||
where?: { phase?: LookupAttempt["phase"]; server?: string },
|
||||
where?: { phase?: LookupAttempt["phase"]; server?: string; retryAfterMs?: number },
|
||||
): LookupResult {
|
||||
return {
|
||||
ok: false,
|
||||
@@ -31,6 +31,7 @@ function failure(
|
||||
errorCode,
|
||||
...(where?.phase ? { errorPhase: where.phase } : {}),
|
||||
...(where?.server ? { errorServer: where.server } : {}),
|
||||
...(where?.retryAfterMs !== undefined ? { retryAfterMs: where.retryAfterMs } : {}),
|
||||
attempts: ctx.attempts,
|
||||
};
|
||||
}
|
||||
@@ -70,9 +71,13 @@ export async function lookup(domain: string, opts?: LookupOptions): Promise<Look
|
||||
try {
|
||||
return await runLookup(domain, signalOpts, ctx);
|
||||
} catch (err: unknown) {
|
||||
const { code, error } = classifyError(err);
|
||||
const { code, error, retryAfterMs } = classifyError(err);
|
||||
const source = attemptForError(err);
|
||||
return failure(ctx, code, error, { phase: source?.phase, server: source?.server });
|
||||
return failure(ctx, code, error, {
|
||||
phase: source?.phase,
|
||||
server: source?.server,
|
||||
retryAfterMs,
|
||||
});
|
||||
} finally {
|
||||
if (deadlineTimer !== undefined) clearTimeout(deadlineTimer);
|
||||
link?.dispose();
|
||||
@@ -142,7 +147,7 @@ async function runLookup(
|
||||
ctx,
|
||||
"rdap_unavailable",
|
||||
`RDAP not available or failed for TLD '${tld}'${detail}. Many TLDs do not publish RDAP; try WHOIS fallback (omit rdapOnly).`,
|
||||
{ phase: last?.phase, server: last?.server },
|
||||
{ phase: last?.phase, server: last?.server, retryAfterMs: last?.retryAfterMs },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+15
-2
@@ -9,6 +9,7 @@ export class RdapperError extends Error {
|
||||
readonly phase?: LookupAttempt["phase"];
|
||||
readonly server?: string;
|
||||
readonly stage?: "connect" | "read";
|
||||
readonly retryAfterMs?: number;
|
||||
|
||||
constructor(
|
||||
code: LookupErrorCode,
|
||||
@@ -17,6 +18,7 @@ export class RdapperError extends Error {
|
||||
phase?: LookupAttempt["phase"];
|
||||
server?: string;
|
||||
stage?: "connect" | "read";
|
||||
retryAfterMs?: number;
|
||||
cause?: unknown;
|
||||
},
|
||||
) {
|
||||
@@ -26,6 +28,7 @@ export class RdapperError extends Error {
|
||||
this.phase = extra?.phase;
|
||||
this.server = extra?.server;
|
||||
this.stage = extra?.stage;
|
||||
this.retryAfterMs = extra?.retryAfterMs;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,8 +77,18 @@ function describeError(err: unknown, errno: string | undefined): string {
|
||||
}
|
||||
|
||||
/** Map any thrown value to a stable error code plus a human-readable message. */
|
||||
export function classifyError(err: unknown): { code: LookupErrorCode; error: string } {
|
||||
if (err instanceof RdapperError) return { code: err.code, error: err.message };
|
||||
export function classifyError(err: unknown): {
|
||||
code: LookupErrorCode;
|
||||
error: string;
|
||||
retryAfterMs?: number;
|
||||
} {
|
||||
if (err instanceof RdapperError) {
|
||||
return {
|
||||
code: err.code,
|
||||
error: err.message,
|
||||
...(err.retryAfterMs !== undefined ? { retryAfterMs: err.retryAfterMs } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
const name = err instanceof Error ? err.name : "";
|
||||
const cause = err instanceof Error ? (err as { cause?: unknown }).cause : undefined;
|
||||
|
||||
+2
-1
@@ -33,13 +33,14 @@ export async function traced<T>(
|
||||
ctx.attempts.push({ ...meta, ok: true, durationMs: Date.now() - start, ...notes });
|
||||
return result;
|
||||
} catch (err) {
|
||||
const { code, error } = classifyError(err);
|
||||
const { code, error, retryAfterMs } = classifyError(err);
|
||||
const attempt: LookupAttempt = {
|
||||
...meta,
|
||||
ok: false,
|
||||
durationMs: Date.now() - start,
|
||||
errorCode: code,
|
||||
error,
|
||||
...(retryAfterMs !== undefined ? { retryAfterMs } : {}),
|
||||
...(err instanceof RdapperError && err.stage ? { stage: err.stage } : {}),
|
||||
...notes,
|
||||
};
|
||||
|
||||
@@ -15,6 +15,15 @@ export interface RdapFetchResult {
|
||||
notFound?: boolean;
|
||||
}
|
||||
|
||||
/** Parse a `Retry-After` header (delay-seconds or HTTP date) into milliseconds. */
|
||||
export function parseRetryAfterMs(value: string | null | undefined): number | undefined {
|
||||
const v = value?.trim();
|
||||
if (!v) return undefined;
|
||||
if (/^\d+$/.test(v)) return Number(v) * 1000;
|
||||
const at = Date.parse(v);
|
||||
return Number.isNaN(at) ? undefined : Math.max(0, at - Date.now());
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch RDAP JSON for a domain from a specific RDAP base URL.
|
||||
* Returns `{ notFound: true }` for HTTP 404 (domain not registered).
|
||||
@@ -46,9 +55,11 @@ export async function fetchRdapDomain(
|
||||
}
|
||||
if (res.status === 429) {
|
||||
const retryAfter = res.headers.get("retry-after");
|
||||
const retryAfterMs = parseRetryAfterMs(retryAfter);
|
||||
throw new RdapperError(
|
||||
"rate_limited",
|
||||
`RDAP 429 rate limited${retryAfter ? ` (Retry-After: ${retryAfter})` : ""}`,
|
||||
retryAfterMs !== undefined ? { retryAfterMs } : undefined,
|
||||
);
|
||||
}
|
||||
if (!res.ok) {
|
||||
|
||||
@@ -379,6 +379,8 @@ export interface LookupResult {
|
||||
errorPhase?: LookupAttempt["phase"];
|
||||
/** Server (RDAP URL or WHOIS host) involved in the failure, when known */
|
||||
errorServer?: string;
|
||||
/** Suggested wait before retrying, from an RDAP `Retry-After` header on a `rate_limited` failure */
|
||||
retryAfterMs?: number;
|
||||
/** Every network attempt made during the lookup, in order (successes and failures) */
|
||||
attempts: LookupAttempt[];
|
||||
}
|
||||
@@ -393,6 +395,7 @@ export interface LookupResult {
|
||||
* - `rdap_unavailable`: `rdapOnly` was set and no RDAP server worked
|
||||
* - `no_server`: IANA answered but no WHOIS server exists for the TLD
|
||||
* - `rate_limited`: the server throttled the query (RDAP 429, or a WHOIS throttle notice)
|
||||
* - `blocked`: the WHOIS server refuses this client outright (retrying will not help)
|
||||
* - `unparseable`: WHOIS replied with text that is neither an availability notice nor a record
|
||||
* - `unsupported_runtime`: WHOIS needs `node:net`, which this runtime lacks
|
||||
*/
|
||||
@@ -407,6 +410,7 @@ export type LookupErrorCode =
|
||||
| "no_server"
|
||||
| "no_data"
|
||||
| "rate_limited"
|
||||
| "blocked"
|
||||
| "unparseable"
|
||||
| "unsupported_runtime"
|
||||
| "unknown";
|
||||
@@ -420,6 +424,8 @@ export interface LookupAttempt {
|
||||
durationMs: number;
|
||||
errorCode?: LookupErrorCode;
|
||||
error?: string;
|
||||
/** `rate_limited` RDAP failures only: the server's `Retry-After`, in milliseconds */
|
||||
retryAfterMs?: number;
|
||||
/** WHOIS failures only: `connect` if the socket never connected; `read` if it connected but sent nothing (timeout, or closed without a response, which is `no_data`) */
|
||||
stage?: "connect" | "read";
|
||||
/** WHOIS only: the read timed out after some data arrived, so the text is partial */
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { whoisQuery } from "./client";
|
||||
|
||||
describe("blockPrivateAddresses", () => {
|
||||
it("refuses to connect when the host resolves to loopback", async () => {
|
||||
await expect(
|
||||
whoisQuery("localhost", "example.com", { timeoutMs: 2000 }, { blockPrivateAddresses: true }),
|
||||
).rejects.toMatchObject({
|
||||
code: "connect_failed",
|
||||
message: expect.stringContaining("non-public"),
|
||||
});
|
||||
});
|
||||
});
|
||||
+46
-2
@@ -1,6 +1,7 @@
|
||||
import { resolveTimeoutMs, throwIfAborted } from "../lib/async";
|
||||
import { abortError, RdapperError } from "../lib/errors";
|
||||
import type { LookupOptions } from "../types";
|
||||
import { isPrivateIp } from "./host";
|
||||
|
||||
export interface WhoisQueryResult {
|
||||
serverQueried: string;
|
||||
@@ -20,6 +21,15 @@ const WHOIS_QUERY_TRANSFORMERS: Record<string, (query: string) => string> = {
|
||||
"whois.jprs.jp": (query) => `${query}/e`, // Append /e for English-only response
|
||||
};
|
||||
|
||||
export interface WhoisTransportOptions {
|
||||
/**
|
||||
* Reject any resolved address that is not public unicast, checked at connect time so it covers
|
||||
* DNS rebinding and hostnames that resolve to private ranges. Used for referral hosts, which
|
||||
* come from upstream response text.
|
||||
*/
|
||||
blockPrivateAddresses?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform a WHOIS query against an RFC 3912 server over TCP 43.
|
||||
* Returns the raw text and the server used.
|
||||
@@ -28,6 +38,7 @@ export async function whoisQuery(
|
||||
server: string,
|
||||
query: string,
|
||||
options?: LookupOptions,
|
||||
transport?: WhoisTransportOptions,
|
||||
): Promise<WhoisQueryResult> {
|
||||
const port = 43;
|
||||
const host = server.replace(/^whois:\/\//i, "");
|
||||
@@ -36,7 +47,7 @@ export async function whoisQuery(
|
||||
const transformer = WHOIS_QUERY_TRANSFORMERS[host];
|
||||
const transformedQuery = transformer ? transformer(query) : query;
|
||||
|
||||
const { text, partial } = await queryTcp(host, port, transformedQuery, options);
|
||||
const { text, partial } = await queryTcp(host, port, transformedQuery, options, transport);
|
||||
return { serverQueried: server, text, ...(partial ? { partial } : {}) };
|
||||
}
|
||||
|
||||
@@ -48,6 +59,7 @@ async function queryTcp(
|
||||
port: number,
|
||||
query: string,
|
||||
options?: LookupOptions,
|
||||
transport?: WhoisTransportOptions,
|
||||
): Promise<{ text: string; partial?: boolean }> {
|
||||
let net: typeof import("node:net") | null;
|
||||
try {
|
||||
@@ -67,9 +79,16 @@ async function queryTcp(
|
||||
throwIfAborted(signal);
|
||||
const timeoutMs = resolveTimeoutMs(options);
|
||||
const createConnection = net.createConnection;
|
||||
const guardedLookup = transport?.blockPrivateAddresses
|
||||
? await privateBlockingLookup()
|
||||
: undefined;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const socket = createConnection({ host, port });
|
||||
const socket = createConnection({
|
||||
host,
|
||||
port,
|
||||
...(guardedLookup ? { lookup: guardedLookup } : {}),
|
||||
});
|
||||
const chunks: Buffer[] = [];
|
||||
let received = 0;
|
||||
let connected = false;
|
||||
@@ -165,3 +184,28 @@ async function queryTcp(
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** A `dns.lookup` replacement that fails the connection when any answer is a non-public address. */
|
||||
async function privateBlockingLookup(): Promise<
|
||||
NonNullable<import("node:net").TcpNetConnectOpts["lookup"]>
|
||||
> {
|
||||
const dns = await import("node:dns");
|
||||
return (hostname, options, callback) => {
|
||||
dns.lookup(hostname, options, (err, address, family) => {
|
||||
if (err) return callback(err, address as never, family as never);
|
||||
const answers = Array.isArray(address) ? address.map((a) => a.address) : [address];
|
||||
const bad = answers.find((a) => isPrivateIp(a));
|
||||
if (bad) {
|
||||
return callback(
|
||||
new RdapperError(
|
||||
"connect_failed",
|
||||
`WHOIS host ${hostname} resolves to a non-public address`,
|
||||
),
|
||||
"" as never,
|
||||
4 as never,
|
||||
);
|
||||
}
|
||||
callback(err, address as never, family as never);
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
@@ -8,6 +8,8 @@ describe("isSafeWhoisReferralHost", () => {
|
||||
"WHOIS.GODADDY.COM",
|
||||
"whois.nic.xn--p1ai",
|
||||
"8.8.8.8",
|
||||
"2606:4700:4700::1111",
|
||||
"::ffff:808:808",
|
||||
"whois.example.com.",
|
||||
])("accepts %s", (h) => expect(isSafeWhoisReferralHost(h)).toBe(true));
|
||||
|
||||
@@ -29,6 +31,15 @@ describe("isSafeWhoisReferralHost", () => {
|
||||
"fe80::1",
|
||||
"fd00::1",
|
||||
"::ffff:127.0.0.1",
|
||||
"::ffff:7f00:1",
|
||||
"::ffff:a00:1",
|
||||
"64:ff9b::7f00:1",
|
||||
"::127.0.0.1",
|
||||
"2002:7f00:1::",
|
||||
"2001:0:4136:e378:8000:63bf:3fff:fdd2",
|
||||
"fec0::1",
|
||||
"[::1]",
|
||||
"1:2:3:4:5:6:7:8:9",
|
||||
"2130706433",
|
||||
"0x7f.1",
|
||||
"whois.example.com:43",
|
||||
|
||||
+74
-15
@@ -1,10 +1,52 @@
|
||||
import { isIP } from "node:net";
|
||||
// No `node:` imports here: this module is loaded in every runtime, WHOIS transport is not.
|
||||
|
||||
const LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i;
|
||||
const BLOCKED_SUFFIXES = [".local", ".localhost", ".internal", ".localdomain", ".lan", ".home"];
|
||||
|
||||
function isPrivateIpv4(ip: string): boolean {
|
||||
const [a = 0, b = 0] = ip.split(".").map(Number);
|
||||
function parseIpv4(ip: string): number[] | null {
|
||||
const parts = ip.split(".");
|
||||
if (parts.length !== 4) return null;
|
||||
const out: number[] = [];
|
||||
for (const p of parts) {
|
||||
if (!/^(0|[1-9]\d{0,2})$/.test(p)) return null;
|
||||
const n = Number(p);
|
||||
if (n > 255) return null;
|
||||
out.push(n);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Parse an IPv6 literal into eight 16-bit groups (no zone ids, no brackets). */
|
||||
function parseIpv6(ip: string): number[] | null {
|
||||
if (!ip.includes(":") || ip.includes("%")) return null;
|
||||
let text = ip;
|
||||
// Embedded dotted IPv4 tail, e.g. ::ffff:127.0.0.1
|
||||
const tail = text.match(/:(\d+\.\d+\.\d+\.\d+)$/);
|
||||
if (tail?.[1]) {
|
||||
const v4 = parseIpv4(tail[1]);
|
||||
if (!v4) return null;
|
||||
const hex = (a: number, b: number) => ((a << 8) | b).toString(16);
|
||||
text = `${text.slice(0, -tail[1].length)}${hex(v4[0] as number, v4[1] as number)}:${hex(v4[2] as number, v4[3] as number)}`;
|
||||
}
|
||||
const halves = text.split("::");
|
||||
if (halves.length > 2) return null;
|
||||
const toGroups = (s: string) => (s === "" ? [] : s.split(":"));
|
||||
const head = toGroups(halves[0] as string);
|
||||
const rest = halves.length === 2 ? toGroups(halves[1] as string) : [];
|
||||
if (halves.length === 1 && head.length !== 8) return null;
|
||||
if (halves.length === 2 && head.length + rest.length > 7) return null;
|
||||
const all = [
|
||||
...head,
|
||||
...Array(halves.length === 2 ? 8 - head.length - rest.length : 0).fill("0"),
|
||||
...rest,
|
||||
];
|
||||
if (all.length !== 8) return null;
|
||||
const nums = all.map((g) => (/^[0-9a-f]{1,4}$/i.test(g) ? Number.parseInt(g, 16) : Number.NaN));
|
||||
return nums.some(Number.isNaN) ? null : nums;
|
||||
}
|
||||
|
||||
function isPrivateIpv4(o: number[]): boolean {
|
||||
const [a = 0, b = 0] = o;
|
||||
return (
|
||||
a === 0 ||
|
||||
a === 10 ||
|
||||
@@ -19,30 +61,47 @@ function isPrivateIpv4(ip: string): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
function isPrivateIpv6(ip: string): boolean {
|
||||
const lower = ip.toLowerCase();
|
||||
const mapped = lower.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/);
|
||||
if (mapped?.[1]) return isPrivateIpv4(mapped[1]);
|
||||
const v4From = (hi: number, lo: number) => [hi >> 8, hi & 0xff, lo >> 8, lo & 0xff];
|
||||
|
||||
function isPrivateIpv6(g: number[]): boolean {
|
||||
const [g0 = 0, g1 = 0, g2 = 0, g3 = 0, g4 = 0, g5 = 0, g6 = 0, g7 = 0] = g;
|
||||
const embedded = () => isPrivateIpv4(v4From(g6, g7));
|
||||
const first96Zero = g0 === 0 && g1 === 0 && g2 === 0 && g3 === 0 && g4 === 0;
|
||||
if (first96Zero && g5 === 0) return true; // ::, ::1, ::a.b.c.d (IPv4-compatible)
|
||||
if (first96Zero && g5 === 0xffff) return embedded(); // ::ffff:a.b.c.d (IPv4-mapped)
|
||||
if (g0 === 0x64 && g1 === 0xff9b && g2 === 0 && g3 === 0 && g4 === 0 && g5 === 0) {
|
||||
return embedded(); // NAT64 64:ff9b::/96
|
||||
}
|
||||
if (g0 === 0x2002) return isPrivateIpv4(v4From(g1, g2)); // 6to4
|
||||
if (g0 === 0x2001 && g1 === 0) return true; // Teredo: embeds arbitrary addresses
|
||||
if (g0 === 0x2001 && g1 === 0xdb8) return true; // documentation
|
||||
return (
|
||||
lower === "::" ||
|
||||
lower === "::1" ||
|
||||
/^f[cd]/.test(lower) || // unique local
|
||||
/^fe[89ab]/.test(lower) || // link-local
|
||||
lower.startsWith("ff") // multicast
|
||||
(g0 & 0xfe00) === 0xfc00 || // unique local fc00::/7
|
||||
(g0 & 0xffc0) === 0xfe80 || // link-local
|
||||
(g0 & 0xffc0) === 0xfec0 || // site-local
|
||||
(g0 & 0xff00) === 0xff00 || // multicast
|
||||
(g0 === 0x100 && g1 === 0 && g2 === 0 && g3 === 0) // discard-only
|
||||
);
|
||||
}
|
||||
|
||||
/** True for an IP literal that is not a public unicast address (or that cannot be parsed). */
|
||||
export function isPrivateIp(ip: string): boolean {
|
||||
const v4 = parseIpv4(ip);
|
||||
if (v4) return isPrivateIpv4(v4);
|
||||
const v6 = parseIpv6(ip);
|
||||
return v6 ? isPrivateIpv6(v6) : true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a WHOIS referral host taken from upstream response text is safe to connect to:
|
||||
* a well-formed public hostname or public IP literal, with no port, path or userinfo.
|
||||
* This is a literal check only; the resolved address is checked again at connect time.
|
||||
*/
|
||||
export function isSafeWhoisReferralHost(host: string): boolean {
|
||||
const value = host.trim().replace(/\.$/, "");
|
||||
if (!value || value.length > 253) return false;
|
||||
|
||||
const ipVersion = isIP(value);
|
||||
if (ipVersion === 4) return !isPrivateIpv4(value);
|
||||
if (ipVersion === 6) return !isPrivateIpv6(value);
|
||||
if (parseIpv4(value) || value.includes(":")) return !isPrivateIp(value);
|
||||
|
||||
const lower = value.toLowerCase();
|
||||
if (lower === "localhost" || BLOCKED_SUFFIXES.some((s) => lower.endsWith(s))) return false;
|
||||
|
||||
@@ -17,20 +17,9 @@ vi.mock("./client.js", () => ({
|
||||
}),
|
||||
}));
|
||||
|
||||
import { collectWhoisReferralChain, followWhoisReferrals } from "./referral";
|
||||
import { collectWhoisReferralChain } from "./referral";
|
||||
|
||||
describe("WHOIS referral contradiction handling", () => {
|
||||
it("keeps TLD WHOIS when registrar claims availability", async () => {
|
||||
const res = await followWhoisReferrals("whois.nic.io", "raindrop.io", {
|
||||
followWhoisReferral: true,
|
||||
maxWhoisReferralHops: 2,
|
||||
});
|
||||
expect(res.serverQueried).toBe("whois.nic.io");
|
||||
// ensure we didn't adopt the registrar response
|
||||
expect(res.text.toLowerCase().includes("creation date")).toBe(true);
|
||||
expect(res.text.toLowerCase().includes("no match")).toBe(false);
|
||||
});
|
||||
|
||||
it("collects chain and does not append contradictory registrar", async () => {
|
||||
const { results: chain } = await collectWhoisReferralChain("whois.nic.io", "raindrop.io", {
|
||||
followWhoisReferral: true,
|
||||
|
||||
+12
-54
@@ -7,53 +7,7 @@ import { whoisQuery } from "./client";
|
||||
import { extractWhoisReferral } from "./discovery";
|
||||
import { isSafeWhoisReferralHost } from "./host";
|
||||
import { isAvailableByWhois, normalizeWhois } from "./normalize";
|
||||
import { detectWhoisThrottle, looksEmptyWhois } from "./throttle";
|
||||
|
||||
/**
|
||||
* Follow registrar WHOIS referrals up to a configured hop limit.
|
||||
* Returns the last successful WHOIS response (best-effort; keeps original on failures).
|
||||
*/
|
||||
export async function followWhoisReferrals(
|
||||
initialServer: string,
|
||||
domain: string,
|
||||
opts?: LookupOptions,
|
||||
ctx?: LookupContext,
|
||||
): Promise<WhoisQueryResult> {
|
||||
const maxHops = Math.max(0, opts?.maxWhoisReferralHops ?? 2);
|
||||
// First query against the provided server
|
||||
let current = await tracedWhoisQuery(initialServer, domain, opts, ctx);
|
||||
if (opts?.followWhoisReferral === false || maxHops === 0) return current;
|
||||
|
||||
const visited = new Set<string>([normalize(current.serverQueried)]);
|
||||
let hops = 0;
|
||||
// Iterate while we see a new referral and are under hop limit
|
||||
while (hops < maxHops) {
|
||||
throwIfAborted(opts?.signal);
|
||||
const next = extractWhoisReferral(current.text);
|
||||
if (!next) break;
|
||||
if (!isSafeWhoisReferralHost(normalize(next))) break;
|
||||
const normalized = normalize(next);
|
||||
if (visited.has(normalized)) break; // cycle protection / same as current
|
||||
visited.add(normalized);
|
||||
try {
|
||||
const res = await tracedWhoisQuery(next, domain, opts, ctx);
|
||||
// Prefer authoritative TLD response when registrar contradicts availability
|
||||
const registeredBefore = !isAvailableByWhois(current.text);
|
||||
const registeredAfter = !isAvailableByWhois(res.text);
|
||||
if (registeredBefore && !registeredAfter) {
|
||||
// Registrar claims availability but TLD shows registered: keep TLD
|
||||
break;
|
||||
}
|
||||
current = res; // adopt registrar when it does not downgrade registration
|
||||
} catch {
|
||||
throwIfAborted(opts?.signal);
|
||||
// If referral server fails, stop following and keep the last good response
|
||||
break;
|
||||
}
|
||||
hops += 1;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
import { detectWhoisRefusal, looksEmptyWhois } from "./throttle";
|
||||
|
||||
/**
|
||||
* Collect the WHOIS referral chain starting from the TLD server.
|
||||
@@ -88,7 +42,7 @@ export async function collectWhoisReferralChain(
|
||||
if (visited.has(normalized)) break;
|
||||
visited.add(normalized);
|
||||
try {
|
||||
const res = await tracedWhoisQuery(next, domain, opts, ctx);
|
||||
const res = await tracedWhoisQuery(next, domain, opts, ctx, true);
|
||||
// If registrar claims availability while TLD indicated registered, stop.
|
||||
const registeredBefore = !isAvailableByWhois(current.text);
|
||||
const registeredAfter = !isAvailableByWhois(res.text);
|
||||
@@ -109,8 +63,8 @@ export async function collectWhoisReferralChain(
|
||||
throwIfAborted(opts?.signal);
|
||||
const { code, error } = classifyError(err);
|
||||
warnings.push(
|
||||
code === "rate_limited"
|
||||
? `WHOIS referral ${normalized} rate limited the query`
|
||||
code === "rate_limited" || code === "blocked"
|
||||
? `WHOIS referral ${normalized} ${code === "blocked" ? "blocked" : "rate limited"} the query`
|
||||
: `WHOIS referral ${normalized} failed (${error})`,
|
||||
);
|
||||
break;
|
||||
@@ -130,17 +84,21 @@ function tracedWhoisQuery(
|
||||
domain: string,
|
||||
opts?: LookupOptions,
|
||||
ctx?: LookupContext,
|
||||
referral = false,
|
||||
): Promise<WhoisQueryResult> {
|
||||
return traced(
|
||||
ctx,
|
||||
{ phase: "whois", server: server.replace(/^whois:\/\//i, "") },
|
||||
async (notes) => {
|
||||
const res = await whoisQuery(server, domain, opts);
|
||||
const res = await whoisQuery(server, domain, opts, { blockPrivateAddresses: referral });
|
||||
if (res.partial) notes.partial = true;
|
||||
if (detectWhoisThrottle(res.text)) {
|
||||
const refusal = detectWhoisRefusal(res.text);
|
||||
if (refusal) {
|
||||
throw new RdapperError(
|
||||
"rate_limited",
|
||||
`WHOIS server ${res.serverQueried} rate limited the query`,
|
||||
refusal,
|
||||
refusal === "blocked"
|
||||
? `WHOIS server ${res.serverQueried} refuses requests from this client`
|
||||
: `WHOIS server ${res.serverQueried} rate limited the query`,
|
||||
{ stage: "read" },
|
||||
);
|
||||
}
|
||||
|
||||
+24
-10
@@ -1,26 +1,40 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { detectWhoisThrottle, looksEmptyWhois } from "./throttle";
|
||||
import { detectWhoisRefusal, looksEmptyWhois } from "./throttle";
|
||||
|
||||
describe("detectWhoisThrottle", () => {
|
||||
describe("detectWhoisRefusal", () => {
|
||||
it.each([
|
||||
"WHOIS LIMIT EXCEEDED - SEE WWW.PIR.ORG/WHOIS FOR DETAILS",
|
||||
"Quota exceeded",
|
||||
"You have exceeded the allowed number of queries. Try again later.",
|
||||
"You have exceeded the allowed number of queries.",
|
||||
"Too many requests",
|
||||
"<!DOCTYPE html><html><body>Service Unavailable</body></html>",
|
||||
"Your IP has been blocked",
|
||||
])("flags %j", (text) => {
|
||||
expect(detectWhoisThrottle(text)).toBe(true);
|
||||
])("classifies %j as rate_limited", (text) => {
|
||||
expect(detectWhoisRefusal(text)).toBe("rate_limited");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"Requests of this client are not permitted. Please use https://www.nic.ch/whois/ for queries.",
|
||||
"Your IP address has been blocked",
|
||||
])("classifies %j as blocked", (text) => {
|
||||
expect(detectWhoisRefusal(text)).toBe("blocked");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"This domain name is blocked. Try again later.",
|
||||
"Access denied for reserved name. No match for EXAMPLE.COM",
|
||||
"No match for EXAMPLE.COM",
|
||||
"Domain is banned by policy",
|
||||
])("leaves %j alone", (text) => {
|
||||
expect(detectWhoisRefusal(text)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not flag a long real record that mentions rate limits", () => {
|
||||
const record = `Domain Name: EXAMPLE.COM\nRegistrar: Test\n${"Name Server: NS.EXAMPLE.COM\n".repeat(100)}\nNote: queries are subject to rate limiting\n`;
|
||||
expect(detectWhoisThrottle(record)).toBe(false);
|
||||
expect(detectWhoisRefusal(record)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not flag empty or ordinary availability text", () => {
|
||||
expect(detectWhoisThrottle(undefined)).toBe(false);
|
||||
expect(detectWhoisThrottle("No match for EXAMPLE.COM")).toBe(false);
|
||||
it("returns undefined for empty input", () => {
|
||||
expect(detectWhoisRefusal(undefined)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+21
-12
@@ -1,29 +1,38 @@
|
||||
import type { DomainRecord } from "../types";
|
||||
import { isAvailableByWhois } from "./normalize";
|
||||
|
||||
// Real WHOIS records are much longer; throttle notices and error pages are short.
|
||||
const MAX_THROTTLE_TEXT_LENGTH = 2048;
|
||||
// Real WHOIS records are much longer; refusal notices and error pages are short.
|
||||
const MAX_REFUSAL_TEXT_LENGTH = 2048;
|
||||
|
||||
// Transient: the same query may succeed later.
|
||||
const THROTTLE_PATTERNS: RegExp[] = [
|
||||
/rate[\s-]?limit/i,
|
||||
/limit\s+exceeded/i,
|
||||
/quota\s+exceeded/i,
|
||||
/exceeded\s+.{0,40}(quer|limit|request|connection)/i,
|
||||
/too\s+many\s+(quer|request|connection)/i,
|
||||
/try\s+again\s+(later|in)/i,
|
||||
/access\s+(denied|limit)/i,
|
||||
/\b(blocked|blacklisted|banned)\b/i,
|
||||
/^\s*<(!doctype|html)\b/i,
|
||||
];
|
||||
|
||||
// Permanent: this client is refused outright, so retrying will not help.
|
||||
const BLOCK_PATTERNS: RegExp[] = [
|
||||
/requests\s+of\s+this\s+client\s+are\s+not\s+permitted/i, // .ch/.li
|
||||
/\b(your|this)\s+(ip|address|client)\b.{0,60}\b(blocked|banned|blacklisted|not\s+permitted)\b/i,
|
||||
];
|
||||
|
||||
/**
|
||||
* Heuristic: does this WHOIS response look like a throttle notice or error page rather than
|
||||
* a domain record? Only short responses qualify, so a real record that mentions "rate limit"
|
||||
* in a remark is not rejected.
|
||||
* Classify a short WHOIS reply that is a refusal rather than a record: `rate_limited` for a
|
||||
* transient throttle, `blocked` for a permanent block. Availability notices win, and long
|
||||
* responses are never refusals, so a real record that mentions "rate limit" is not rejected.
|
||||
*/
|
||||
export function detectWhoisThrottle(text: string | undefined): boolean {
|
||||
if (!text) return false;
|
||||
if (text.length > MAX_THROTTLE_TEXT_LENGTH) return false;
|
||||
return THROTTLE_PATTERNS.some((re) => re.test(text));
|
||||
export function detectWhoisRefusal(
|
||||
text: string | undefined,
|
||||
): "rate_limited" | "blocked" | undefined {
|
||||
if (!text || text.length > MAX_REFUSAL_TEXT_LENGTH) return undefined;
|
||||
if (isAvailableByWhois(text)) return undefined;
|
||||
if (BLOCK_PATTERNS.some((re) => re.test(text))) return "blocked";
|
||||
if (THROTTLE_PATTERNS.some((re) => re.test(text))) return "rate_limited";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** True when a normalized WHOIS record carries none of the fields a real registration would. */
|
||||
|
||||
Reference in New Issue
Block a user