mirror of
https://github.com/jakejarvis/rdapper.git
synced 2026-09-23 00:15:31 -04:00
feat: surface retryAfterMs, add blocked error code, and harden private-address guard
- Parse `Retry-After` headers (delay-seconds or HTTP-date) into `retryAfterMs` via a new `parseRetryAfterMs` helper and expose the value on both `LookupAttempt` and `LookupResult` so callers can honour server-requested back-off without parsing the error string - Propagate `retryAfterMs` through `classifyError`, `traced`, `failure`, and the `rdap_unavailable` path so the value surfaces on the top-level result - Add `blocked` to `LookupErrorCode` for WHOIS servers that permanently refuse a client (distinct from `rate_limited`, which is a temporary throttle that may succeed on retry) - Add `blockPrivateAddresses` to `WhoisTransportOptions`; when set, a custom `dns.lookup` shim rejects the connection before it opens if any resolved address is non-public, covering DNS rebinding and hostnames that resolve to private ranges - Expand `isPrivateIp` / `isSafeWhoisReferralHost` to reject IPv4-mapped (`::ffff:7f00:1`), NAT64 (`64:ff9b::`), 6to4 (`2002:7f00::`), Teredo (`2001:0:`), deprecated site-local (`fec0::`), and bracketed IPv6 literals
This commit is contained in:
@@ -55,6 +55,14 @@ describe("throttle and empty-response guards", () => {
|
|||||||
expect(res.errorServer).toBe("whois.verisign-grs.com");
|
expect(res.errorServer).toBe("whois.verisign-grs.com");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("fails with blocked (not rate_limited) for a permanent refusal", async () => {
|
||||||
|
vi.mocked(whoisQuery).mockImplementation(
|
||||||
|
ianaThen("Requests of this client are not permitted. Please use https://www.nic.ch/whois/"),
|
||||||
|
);
|
||||||
|
const res = await lookup("example.com", { whoisOnly: true });
|
||||||
|
expect(res.errorCode).toBe("blocked");
|
||||||
|
});
|
||||||
|
|
||||||
it("fails with unparseable when a long reply has no fields and no availability phrase", async () => {
|
it("fails with unparseable when a long reply has no fields and no availability phrase", async () => {
|
||||||
vi.mocked(whoisQuery).mockImplementation(ianaThen(`${"lorem ipsum ".repeat(300)}\n`));
|
vi.mocked(whoisQuery).mockImplementation(ianaThen(`${"lorem ipsum ".repeat(300)}\n`));
|
||||||
const res = await lookup("example.com", { whoisOnly: true });
|
const res = await lookup("example.com", { whoisOnly: true });
|
||||||
@@ -78,6 +86,22 @@ describe("throttle and empty-response guards", () => {
|
|||||||
expect(res.ok, res.error).toBe(true);
|
expect(res.ok, res.error).toBe(true);
|
||||||
expect(res.attempts[0]).toMatchObject({ errorCode: "rate_limited" });
|
expect(res.attempts[0]).toMatchObject({ errorCode: "rate_limited" });
|
||||||
expect(res.attempts[0]?.error).toContain("Retry-After: 30");
|
expect(res.attempts[0]?.error).toContain("Retry-After: 30");
|
||||||
|
expect(res.attempts[0]?.retryAfterMs).toBe(30_000);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("rdapOnly rate limiting", () => {
|
||||||
|
it("surfaces retryAfterMs on the result", async () => {
|
||||||
|
const customFetch: FetchLike = vi.fn(
|
||||||
|
async () => new Response("", { status: 429, headers: { "retry-after": "12" } }),
|
||||||
|
);
|
||||||
|
const res = await lookup("example.com", {
|
||||||
|
customBootstrapData: bootstrap,
|
||||||
|
customFetch,
|
||||||
|
rdapOnly: true,
|
||||||
|
});
|
||||||
|
expect(res.ok).toBe(false);
|
||||||
|
expect(res.retryAfterMs).toBe(12_000);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
+9
-4
@@ -23,7 +23,7 @@ function failure(
|
|||||||
ctx: LookupContext,
|
ctx: LookupContext,
|
||||||
errorCode: LookupErrorCode,
|
errorCode: LookupErrorCode,
|
||||||
error: string,
|
error: string,
|
||||||
where?: { phase?: LookupAttempt["phase"]; server?: string },
|
where?: { phase?: LookupAttempt["phase"]; server?: string; retryAfterMs?: number },
|
||||||
): LookupResult {
|
): LookupResult {
|
||||||
return {
|
return {
|
||||||
ok: false,
|
ok: false,
|
||||||
@@ -31,6 +31,7 @@ function failure(
|
|||||||
errorCode,
|
errorCode,
|
||||||
...(where?.phase ? { errorPhase: where.phase } : {}),
|
...(where?.phase ? { errorPhase: where.phase } : {}),
|
||||||
...(where?.server ? { errorServer: where.server } : {}),
|
...(where?.server ? { errorServer: where.server } : {}),
|
||||||
|
...(where?.retryAfterMs !== undefined ? { retryAfterMs: where.retryAfterMs } : {}),
|
||||||
attempts: ctx.attempts,
|
attempts: ctx.attempts,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -70,9 +71,13 @@ export async function lookup(domain: string, opts?: LookupOptions): Promise<Look
|
|||||||
try {
|
try {
|
||||||
return await runLookup(domain, signalOpts, ctx);
|
return await runLookup(domain, signalOpts, ctx);
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
const { code, error } = classifyError(err);
|
const { code, error, retryAfterMs } = classifyError(err);
|
||||||
const source = attemptForError(err);
|
const source = attemptForError(err);
|
||||||
return failure(ctx, code, error, { phase: source?.phase, server: source?.server });
|
return failure(ctx, code, error, {
|
||||||
|
phase: source?.phase,
|
||||||
|
server: source?.server,
|
||||||
|
retryAfterMs,
|
||||||
|
});
|
||||||
} finally {
|
} finally {
|
||||||
if (deadlineTimer !== undefined) clearTimeout(deadlineTimer);
|
if (deadlineTimer !== undefined) clearTimeout(deadlineTimer);
|
||||||
link?.dispose();
|
link?.dispose();
|
||||||
@@ -142,7 +147,7 @@ async function runLookup(
|
|||||||
ctx,
|
ctx,
|
||||||
"rdap_unavailable",
|
"rdap_unavailable",
|
||||||
`RDAP not available or failed for TLD '${tld}'${detail}. Many TLDs do not publish RDAP; try WHOIS fallback (omit rdapOnly).`,
|
`RDAP not available or failed for TLD '${tld}'${detail}. Many TLDs do not publish RDAP; try WHOIS fallback (omit rdapOnly).`,
|
||||||
{ phase: last?.phase, server: last?.server },
|
{ phase: last?.phase, server: last?.server, retryAfterMs: last?.retryAfterMs },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+15
-2
@@ -9,6 +9,7 @@ export class RdapperError extends Error {
|
|||||||
readonly phase?: LookupAttempt["phase"];
|
readonly phase?: LookupAttempt["phase"];
|
||||||
readonly server?: string;
|
readonly server?: string;
|
||||||
readonly stage?: "connect" | "read";
|
readonly stage?: "connect" | "read";
|
||||||
|
readonly retryAfterMs?: number;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
code: LookupErrorCode,
|
code: LookupErrorCode,
|
||||||
@@ -17,6 +18,7 @@ export class RdapperError extends Error {
|
|||||||
phase?: LookupAttempt["phase"];
|
phase?: LookupAttempt["phase"];
|
||||||
server?: string;
|
server?: string;
|
||||||
stage?: "connect" | "read";
|
stage?: "connect" | "read";
|
||||||
|
retryAfterMs?: number;
|
||||||
cause?: unknown;
|
cause?: unknown;
|
||||||
},
|
},
|
||||||
) {
|
) {
|
||||||
@@ -26,6 +28,7 @@ export class RdapperError extends Error {
|
|||||||
this.phase = extra?.phase;
|
this.phase = extra?.phase;
|
||||||
this.server = extra?.server;
|
this.server = extra?.server;
|
||||||
this.stage = extra?.stage;
|
this.stage = extra?.stage;
|
||||||
|
this.retryAfterMs = extra?.retryAfterMs;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,8 +77,18 @@ function describeError(err: unknown, errno: string | undefined): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Map any thrown value to a stable error code plus a human-readable message. */
|
/** Map any thrown value to a stable error code plus a human-readable message. */
|
||||||
export function classifyError(err: unknown): { code: LookupErrorCode; error: string } {
|
export function classifyError(err: unknown): {
|
||||||
if (err instanceof RdapperError) return { code: err.code, error: err.message };
|
code: LookupErrorCode;
|
||||||
|
error: string;
|
||||||
|
retryAfterMs?: number;
|
||||||
|
} {
|
||||||
|
if (err instanceof RdapperError) {
|
||||||
|
return {
|
||||||
|
code: err.code,
|
||||||
|
error: err.message,
|
||||||
|
...(err.retryAfterMs !== undefined ? { retryAfterMs: err.retryAfterMs } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
const name = err instanceof Error ? err.name : "";
|
const name = err instanceof Error ? err.name : "";
|
||||||
const cause = err instanceof Error ? (err as { cause?: unknown }).cause : undefined;
|
const cause = err instanceof Error ? (err as { cause?: unknown }).cause : undefined;
|
||||||
|
|||||||
+2
-1
@@ -33,13 +33,14 @@ export async function traced<T>(
|
|||||||
ctx.attempts.push({ ...meta, ok: true, durationMs: Date.now() - start, ...notes });
|
ctx.attempts.push({ ...meta, ok: true, durationMs: Date.now() - start, ...notes });
|
||||||
return result;
|
return result;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const { code, error } = classifyError(err);
|
const { code, error, retryAfterMs } = classifyError(err);
|
||||||
const attempt: LookupAttempt = {
|
const attempt: LookupAttempt = {
|
||||||
...meta,
|
...meta,
|
||||||
ok: false,
|
ok: false,
|
||||||
durationMs: Date.now() - start,
|
durationMs: Date.now() - start,
|
||||||
errorCode: code,
|
errorCode: code,
|
||||||
error,
|
error,
|
||||||
|
...(retryAfterMs !== undefined ? { retryAfterMs } : {}),
|
||||||
...(err instanceof RdapperError && err.stage ? { stage: err.stage } : {}),
|
...(err instanceof RdapperError && err.stage ? { stage: err.stage } : {}),
|
||||||
...notes,
|
...notes,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -15,6 +15,15 @@ export interface RdapFetchResult {
|
|||||||
notFound?: boolean;
|
notFound?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Parse a `Retry-After` header (delay-seconds or HTTP date) into milliseconds. */
|
||||||
|
export function parseRetryAfterMs(value: string | null | undefined): number | undefined {
|
||||||
|
const v = value?.trim();
|
||||||
|
if (!v) return undefined;
|
||||||
|
if (/^\d+$/.test(v)) return Number(v) * 1000;
|
||||||
|
const at = Date.parse(v);
|
||||||
|
return Number.isNaN(at) ? undefined : Math.max(0, at - Date.now());
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fetch RDAP JSON for a domain from a specific RDAP base URL.
|
* Fetch RDAP JSON for a domain from a specific RDAP base URL.
|
||||||
* Returns `{ notFound: true }` for HTTP 404 (domain not registered).
|
* Returns `{ notFound: true }` for HTTP 404 (domain not registered).
|
||||||
@@ -46,9 +55,11 @@ export async function fetchRdapDomain(
|
|||||||
}
|
}
|
||||||
if (res.status === 429) {
|
if (res.status === 429) {
|
||||||
const retryAfter = res.headers.get("retry-after");
|
const retryAfter = res.headers.get("retry-after");
|
||||||
|
const retryAfterMs = parseRetryAfterMs(retryAfter);
|
||||||
throw new RdapperError(
|
throw new RdapperError(
|
||||||
"rate_limited",
|
"rate_limited",
|
||||||
`RDAP 429 rate limited${retryAfter ? ` (Retry-After: ${retryAfter})` : ""}`,
|
`RDAP 429 rate limited${retryAfter ? ` (Retry-After: ${retryAfter})` : ""}`,
|
||||||
|
retryAfterMs !== undefined ? { retryAfterMs } : undefined,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
|
|||||||
@@ -379,6 +379,8 @@ export interface LookupResult {
|
|||||||
errorPhase?: LookupAttempt["phase"];
|
errorPhase?: LookupAttempt["phase"];
|
||||||
/** Server (RDAP URL or WHOIS host) involved in the failure, when known */
|
/** Server (RDAP URL or WHOIS host) involved in the failure, when known */
|
||||||
errorServer?: string;
|
errorServer?: string;
|
||||||
|
/** Suggested wait before retrying, from an RDAP `Retry-After` header on a `rate_limited` failure */
|
||||||
|
retryAfterMs?: number;
|
||||||
/** Every network attempt made during the lookup, in order (successes and failures) */
|
/** Every network attempt made during the lookup, in order (successes and failures) */
|
||||||
attempts: LookupAttempt[];
|
attempts: LookupAttempt[];
|
||||||
}
|
}
|
||||||
@@ -393,6 +395,7 @@ export interface LookupResult {
|
|||||||
* - `rdap_unavailable`: `rdapOnly` was set and no RDAP server worked
|
* - `rdap_unavailable`: `rdapOnly` was set and no RDAP server worked
|
||||||
* - `no_server`: IANA answered but no WHOIS server exists for the TLD
|
* - `no_server`: IANA answered but no WHOIS server exists for the TLD
|
||||||
* - `rate_limited`: the server throttled the query (RDAP 429, or a WHOIS throttle notice)
|
* - `rate_limited`: the server throttled the query (RDAP 429, or a WHOIS throttle notice)
|
||||||
|
* - `blocked`: the WHOIS server refuses this client outright (retrying will not help)
|
||||||
* - `unparseable`: WHOIS replied with text that is neither an availability notice nor a record
|
* - `unparseable`: WHOIS replied with text that is neither an availability notice nor a record
|
||||||
* - `unsupported_runtime`: WHOIS needs `node:net`, which this runtime lacks
|
* - `unsupported_runtime`: WHOIS needs `node:net`, which this runtime lacks
|
||||||
*/
|
*/
|
||||||
@@ -407,6 +410,7 @@ export type LookupErrorCode =
|
|||||||
| "no_server"
|
| "no_server"
|
||||||
| "no_data"
|
| "no_data"
|
||||||
| "rate_limited"
|
| "rate_limited"
|
||||||
|
| "blocked"
|
||||||
| "unparseable"
|
| "unparseable"
|
||||||
| "unsupported_runtime"
|
| "unsupported_runtime"
|
||||||
| "unknown";
|
| "unknown";
|
||||||
@@ -420,6 +424,8 @@ export interface LookupAttempt {
|
|||||||
durationMs: number;
|
durationMs: number;
|
||||||
errorCode?: LookupErrorCode;
|
errorCode?: LookupErrorCode;
|
||||||
error?: string;
|
error?: string;
|
||||||
|
/** `rate_limited` RDAP failures only: the server's `Retry-After`, in milliseconds */
|
||||||
|
retryAfterMs?: number;
|
||||||
/** WHOIS failures only: `connect` if the socket never connected; `read` if it connected but sent nothing (timeout, or closed without a response, which is `no_data`) */
|
/** WHOIS failures only: `connect` if the socket never connected; `read` if it connected but sent nothing (timeout, or closed without a response, which is `no_data`) */
|
||||||
stage?: "connect" | "read";
|
stage?: "connect" | "read";
|
||||||
/** WHOIS only: the read timed out after some data arrived, so the text is partial */
|
/** WHOIS only: the read timed out after some data arrived, so the text is partial */
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { whoisQuery } from "./client";
|
||||||
|
|
||||||
|
describe("blockPrivateAddresses", () => {
|
||||||
|
it("refuses to connect when the host resolves to loopback", async () => {
|
||||||
|
await expect(
|
||||||
|
whoisQuery("localhost", "example.com", { timeoutMs: 2000 }, { blockPrivateAddresses: true }),
|
||||||
|
).rejects.toMatchObject({
|
||||||
|
code: "connect_failed",
|
||||||
|
message: expect.stringContaining("non-public"),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
+46
-2
@@ -1,6 +1,7 @@
|
|||||||
import { resolveTimeoutMs, throwIfAborted } from "../lib/async";
|
import { resolveTimeoutMs, throwIfAborted } from "../lib/async";
|
||||||
import { abortError, RdapperError } from "../lib/errors";
|
import { abortError, RdapperError } from "../lib/errors";
|
||||||
import type { LookupOptions } from "../types";
|
import type { LookupOptions } from "../types";
|
||||||
|
import { isPrivateIp } from "./host";
|
||||||
|
|
||||||
export interface WhoisQueryResult {
|
export interface WhoisQueryResult {
|
||||||
serverQueried: string;
|
serverQueried: string;
|
||||||
@@ -20,6 +21,15 @@ const WHOIS_QUERY_TRANSFORMERS: Record<string, (query: string) => string> = {
|
|||||||
"whois.jprs.jp": (query) => `${query}/e`, // Append /e for English-only response
|
"whois.jprs.jp": (query) => `${query}/e`, // Append /e for English-only response
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export interface WhoisTransportOptions {
|
||||||
|
/**
|
||||||
|
* Reject any resolved address that is not public unicast, checked at connect time so it covers
|
||||||
|
* DNS rebinding and hostnames that resolve to private ranges. Used for referral hosts, which
|
||||||
|
* come from upstream response text.
|
||||||
|
*/
|
||||||
|
blockPrivateAddresses?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Perform a WHOIS query against an RFC 3912 server over TCP 43.
|
* Perform a WHOIS query against an RFC 3912 server over TCP 43.
|
||||||
* Returns the raw text and the server used.
|
* Returns the raw text and the server used.
|
||||||
@@ -28,6 +38,7 @@ export async function whoisQuery(
|
|||||||
server: string,
|
server: string,
|
||||||
query: string,
|
query: string,
|
||||||
options?: LookupOptions,
|
options?: LookupOptions,
|
||||||
|
transport?: WhoisTransportOptions,
|
||||||
): Promise<WhoisQueryResult> {
|
): Promise<WhoisQueryResult> {
|
||||||
const port = 43;
|
const port = 43;
|
||||||
const host = server.replace(/^whois:\/\//i, "");
|
const host = server.replace(/^whois:\/\//i, "");
|
||||||
@@ -36,7 +47,7 @@ export async function whoisQuery(
|
|||||||
const transformer = WHOIS_QUERY_TRANSFORMERS[host];
|
const transformer = WHOIS_QUERY_TRANSFORMERS[host];
|
||||||
const transformedQuery = transformer ? transformer(query) : query;
|
const transformedQuery = transformer ? transformer(query) : query;
|
||||||
|
|
||||||
const { text, partial } = await queryTcp(host, port, transformedQuery, options);
|
const { text, partial } = await queryTcp(host, port, transformedQuery, options, transport);
|
||||||
return { serverQueried: server, text, ...(partial ? { partial } : {}) };
|
return { serverQueried: server, text, ...(partial ? { partial } : {}) };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -48,6 +59,7 @@ async function queryTcp(
|
|||||||
port: number,
|
port: number,
|
||||||
query: string,
|
query: string,
|
||||||
options?: LookupOptions,
|
options?: LookupOptions,
|
||||||
|
transport?: WhoisTransportOptions,
|
||||||
): Promise<{ text: string; partial?: boolean }> {
|
): Promise<{ text: string; partial?: boolean }> {
|
||||||
let net: typeof import("node:net") | null;
|
let net: typeof import("node:net") | null;
|
||||||
try {
|
try {
|
||||||
@@ -67,9 +79,16 @@ async function queryTcp(
|
|||||||
throwIfAborted(signal);
|
throwIfAborted(signal);
|
||||||
const timeoutMs = resolveTimeoutMs(options);
|
const timeoutMs = resolveTimeoutMs(options);
|
||||||
const createConnection = net.createConnection;
|
const createConnection = net.createConnection;
|
||||||
|
const guardedLookup = transport?.blockPrivateAddresses
|
||||||
|
? await privateBlockingLookup()
|
||||||
|
: undefined;
|
||||||
|
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
const socket = createConnection({ host, port });
|
const socket = createConnection({
|
||||||
|
host,
|
||||||
|
port,
|
||||||
|
...(guardedLookup ? { lookup: guardedLookup } : {}),
|
||||||
|
});
|
||||||
const chunks: Buffer[] = [];
|
const chunks: Buffer[] = [];
|
||||||
let received = 0;
|
let received = 0;
|
||||||
let connected = false;
|
let connected = false;
|
||||||
@@ -165,3 +184,28 @@ async function queryTcp(
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** A `dns.lookup` replacement that fails the connection when any answer is a non-public address. */
|
||||||
|
async function privateBlockingLookup(): Promise<
|
||||||
|
NonNullable<import("node:net").TcpNetConnectOpts["lookup"]>
|
||||||
|
> {
|
||||||
|
const dns = await import("node:dns");
|
||||||
|
return (hostname, options, callback) => {
|
||||||
|
dns.lookup(hostname, options, (err, address, family) => {
|
||||||
|
if (err) return callback(err, address as never, family as never);
|
||||||
|
const answers = Array.isArray(address) ? address.map((a) => a.address) : [address];
|
||||||
|
const bad = answers.find((a) => isPrivateIp(a));
|
||||||
|
if (bad) {
|
||||||
|
return callback(
|
||||||
|
new RdapperError(
|
||||||
|
"connect_failed",
|
||||||
|
`WHOIS host ${hostname} resolves to a non-public address`,
|
||||||
|
),
|
||||||
|
"" as never,
|
||||||
|
4 as never,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
callback(err, address as never, family as never);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ describe("isSafeWhoisReferralHost", () => {
|
|||||||
"WHOIS.GODADDY.COM",
|
"WHOIS.GODADDY.COM",
|
||||||
"whois.nic.xn--p1ai",
|
"whois.nic.xn--p1ai",
|
||||||
"8.8.8.8",
|
"8.8.8.8",
|
||||||
|
"2606:4700:4700::1111",
|
||||||
|
"::ffff:808:808",
|
||||||
"whois.example.com.",
|
"whois.example.com.",
|
||||||
])("accepts %s", (h) => expect(isSafeWhoisReferralHost(h)).toBe(true));
|
])("accepts %s", (h) => expect(isSafeWhoisReferralHost(h)).toBe(true));
|
||||||
|
|
||||||
@@ -29,6 +31,15 @@ describe("isSafeWhoisReferralHost", () => {
|
|||||||
"fe80::1",
|
"fe80::1",
|
||||||
"fd00::1",
|
"fd00::1",
|
||||||
"::ffff:127.0.0.1",
|
"::ffff:127.0.0.1",
|
||||||
|
"::ffff:7f00:1",
|
||||||
|
"::ffff:a00:1",
|
||||||
|
"64:ff9b::7f00:1",
|
||||||
|
"::127.0.0.1",
|
||||||
|
"2002:7f00:1::",
|
||||||
|
"2001:0:4136:e378:8000:63bf:3fff:fdd2",
|
||||||
|
"fec0::1",
|
||||||
|
"[::1]",
|
||||||
|
"1:2:3:4:5:6:7:8:9",
|
||||||
"2130706433",
|
"2130706433",
|
||||||
"0x7f.1",
|
"0x7f.1",
|
||||||
"whois.example.com:43",
|
"whois.example.com:43",
|
||||||
|
|||||||
+74
-15
@@ -1,10 +1,52 @@
|
|||||||
import { isIP } from "node:net";
|
// No `node:` imports here: this module is loaded in every runtime, WHOIS transport is not.
|
||||||
|
|
||||||
const LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i;
|
const LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i;
|
||||||
const BLOCKED_SUFFIXES = [".local", ".localhost", ".internal", ".localdomain", ".lan", ".home"];
|
const BLOCKED_SUFFIXES = [".local", ".localhost", ".internal", ".localdomain", ".lan", ".home"];
|
||||||
|
|
||||||
function isPrivateIpv4(ip: string): boolean {
|
function parseIpv4(ip: string): number[] | null {
|
||||||
const [a = 0, b = 0] = ip.split(".").map(Number);
|
const parts = ip.split(".");
|
||||||
|
if (parts.length !== 4) return null;
|
||||||
|
const out: number[] = [];
|
||||||
|
for (const p of parts) {
|
||||||
|
if (!/^(0|[1-9]\d{0,2})$/.test(p)) return null;
|
||||||
|
const n = Number(p);
|
||||||
|
if (n > 255) return null;
|
||||||
|
out.push(n);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Parse an IPv6 literal into eight 16-bit groups (no zone ids, no brackets). */
|
||||||
|
function parseIpv6(ip: string): number[] | null {
|
||||||
|
if (!ip.includes(":") || ip.includes("%")) return null;
|
||||||
|
let text = ip;
|
||||||
|
// Embedded dotted IPv4 tail, e.g. ::ffff:127.0.0.1
|
||||||
|
const tail = text.match(/:(\d+\.\d+\.\d+\.\d+)$/);
|
||||||
|
if (tail?.[1]) {
|
||||||
|
const v4 = parseIpv4(tail[1]);
|
||||||
|
if (!v4) return null;
|
||||||
|
const hex = (a: number, b: number) => ((a << 8) | b).toString(16);
|
||||||
|
text = `${text.slice(0, -tail[1].length)}${hex(v4[0] as number, v4[1] as number)}:${hex(v4[2] as number, v4[3] as number)}`;
|
||||||
|
}
|
||||||
|
const halves = text.split("::");
|
||||||
|
if (halves.length > 2) return null;
|
||||||
|
const toGroups = (s: string) => (s === "" ? [] : s.split(":"));
|
||||||
|
const head = toGroups(halves[0] as string);
|
||||||
|
const rest = halves.length === 2 ? toGroups(halves[1] as string) : [];
|
||||||
|
if (halves.length === 1 && head.length !== 8) return null;
|
||||||
|
if (halves.length === 2 && head.length + rest.length > 7) return null;
|
||||||
|
const all = [
|
||||||
|
...head,
|
||||||
|
...Array(halves.length === 2 ? 8 - head.length - rest.length : 0).fill("0"),
|
||||||
|
...rest,
|
||||||
|
];
|
||||||
|
if (all.length !== 8) return null;
|
||||||
|
const nums = all.map((g) => (/^[0-9a-f]{1,4}$/i.test(g) ? Number.parseInt(g, 16) : Number.NaN));
|
||||||
|
return nums.some(Number.isNaN) ? null : nums;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPrivateIpv4(o: number[]): boolean {
|
||||||
|
const [a = 0, b = 0] = o;
|
||||||
return (
|
return (
|
||||||
a === 0 ||
|
a === 0 ||
|
||||||
a === 10 ||
|
a === 10 ||
|
||||||
@@ -19,30 +61,47 @@ function isPrivateIpv4(ip: string): boolean {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function isPrivateIpv6(ip: string): boolean {
|
const v4From = (hi: number, lo: number) => [hi >> 8, hi & 0xff, lo >> 8, lo & 0xff];
|
||||||
const lower = ip.toLowerCase();
|
|
||||||
const mapped = lower.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/);
|
function isPrivateIpv6(g: number[]): boolean {
|
||||||
if (mapped?.[1]) return isPrivateIpv4(mapped[1]);
|
const [g0 = 0, g1 = 0, g2 = 0, g3 = 0, g4 = 0, g5 = 0, g6 = 0, g7 = 0] = g;
|
||||||
|
const embedded = () => isPrivateIpv4(v4From(g6, g7));
|
||||||
|
const first96Zero = g0 === 0 && g1 === 0 && g2 === 0 && g3 === 0 && g4 === 0;
|
||||||
|
if (first96Zero && g5 === 0) return true; // ::, ::1, ::a.b.c.d (IPv4-compatible)
|
||||||
|
if (first96Zero && g5 === 0xffff) return embedded(); // ::ffff:a.b.c.d (IPv4-mapped)
|
||||||
|
if (g0 === 0x64 && g1 === 0xff9b && g2 === 0 && g3 === 0 && g4 === 0 && g5 === 0) {
|
||||||
|
return embedded(); // NAT64 64:ff9b::/96
|
||||||
|
}
|
||||||
|
if (g0 === 0x2002) return isPrivateIpv4(v4From(g1, g2)); // 6to4
|
||||||
|
if (g0 === 0x2001 && g1 === 0) return true; // Teredo: embeds arbitrary addresses
|
||||||
|
if (g0 === 0x2001 && g1 === 0xdb8) return true; // documentation
|
||||||
return (
|
return (
|
||||||
lower === "::" ||
|
(g0 & 0xfe00) === 0xfc00 || // unique local fc00::/7
|
||||||
lower === "::1" ||
|
(g0 & 0xffc0) === 0xfe80 || // link-local
|
||||||
/^f[cd]/.test(lower) || // unique local
|
(g0 & 0xffc0) === 0xfec0 || // site-local
|
||||||
/^fe[89ab]/.test(lower) || // link-local
|
(g0 & 0xff00) === 0xff00 || // multicast
|
||||||
lower.startsWith("ff") // multicast
|
(g0 === 0x100 && g1 === 0 && g2 === 0 && g3 === 0) // discard-only
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** True for an IP literal that is not a public unicast address (or that cannot be parsed). */
|
||||||
|
export function isPrivateIp(ip: string): boolean {
|
||||||
|
const v4 = parseIpv4(ip);
|
||||||
|
if (v4) return isPrivateIpv4(v4);
|
||||||
|
const v6 = parseIpv6(ip);
|
||||||
|
return v6 ? isPrivateIpv6(v6) : true;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Whether a WHOIS referral host taken from upstream response text is safe to connect to:
|
* Whether a WHOIS referral host taken from upstream response text is safe to connect to:
|
||||||
* a well-formed public hostname or public IP literal, with no port, path or userinfo.
|
* a well-formed public hostname or public IP literal, with no port, path or userinfo.
|
||||||
|
* This is a literal check only; the resolved address is checked again at connect time.
|
||||||
*/
|
*/
|
||||||
export function isSafeWhoisReferralHost(host: string): boolean {
|
export function isSafeWhoisReferralHost(host: string): boolean {
|
||||||
const value = host.trim().replace(/\.$/, "");
|
const value = host.trim().replace(/\.$/, "");
|
||||||
if (!value || value.length > 253) return false;
|
if (!value || value.length > 253) return false;
|
||||||
|
|
||||||
const ipVersion = isIP(value);
|
if (parseIpv4(value) || value.includes(":")) return !isPrivateIp(value);
|
||||||
if (ipVersion === 4) return !isPrivateIpv4(value);
|
|
||||||
if (ipVersion === 6) return !isPrivateIpv6(value);
|
|
||||||
|
|
||||||
const lower = value.toLowerCase();
|
const lower = value.toLowerCase();
|
||||||
if (lower === "localhost" || BLOCKED_SUFFIXES.some((s) => lower.endsWith(s))) return false;
|
if (lower === "localhost" || BLOCKED_SUFFIXES.some((s) => lower.endsWith(s))) return false;
|
||||||
|
|||||||
@@ -17,20 +17,9 @@ vi.mock("./client.js", () => ({
|
|||||||
}),
|
}),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
import { collectWhoisReferralChain, followWhoisReferrals } from "./referral";
|
import { collectWhoisReferralChain } from "./referral";
|
||||||
|
|
||||||
describe("WHOIS referral contradiction handling", () => {
|
describe("WHOIS referral contradiction handling", () => {
|
||||||
it("keeps TLD WHOIS when registrar claims availability", async () => {
|
|
||||||
const res = await followWhoisReferrals("whois.nic.io", "raindrop.io", {
|
|
||||||
followWhoisReferral: true,
|
|
||||||
maxWhoisReferralHops: 2,
|
|
||||||
});
|
|
||||||
expect(res.serverQueried).toBe("whois.nic.io");
|
|
||||||
// ensure we didn't adopt the registrar response
|
|
||||||
expect(res.text.toLowerCase().includes("creation date")).toBe(true);
|
|
||||||
expect(res.text.toLowerCase().includes("no match")).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("collects chain and does not append contradictory registrar", async () => {
|
it("collects chain and does not append contradictory registrar", async () => {
|
||||||
const { results: chain } = await collectWhoisReferralChain("whois.nic.io", "raindrop.io", {
|
const { results: chain } = await collectWhoisReferralChain("whois.nic.io", "raindrop.io", {
|
||||||
followWhoisReferral: true,
|
followWhoisReferral: true,
|
||||||
|
|||||||
+12
-54
@@ -7,53 +7,7 @@ import { whoisQuery } from "./client";
|
|||||||
import { extractWhoisReferral } from "./discovery";
|
import { extractWhoisReferral } from "./discovery";
|
||||||
import { isSafeWhoisReferralHost } from "./host";
|
import { isSafeWhoisReferralHost } from "./host";
|
||||||
import { isAvailableByWhois, normalizeWhois } from "./normalize";
|
import { isAvailableByWhois, normalizeWhois } from "./normalize";
|
||||||
import { detectWhoisThrottle, looksEmptyWhois } from "./throttle";
|
import { detectWhoisRefusal, looksEmptyWhois } from "./throttle";
|
||||||
|
|
||||||
/**
|
|
||||||
* Follow registrar WHOIS referrals up to a configured hop limit.
|
|
||||||
* Returns the last successful WHOIS response (best-effort; keeps original on failures).
|
|
||||||
*/
|
|
||||||
export async function followWhoisReferrals(
|
|
||||||
initialServer: string,
|
|
||||||
domain: string,
|
|
||||||
opts?: LookupOptions,
|
|
||||||
ctx?: LookupContext,
|
|
||||||
): Promise<WhoisQueryResult> {
|
|
||||||
const maxHops = Math.max(0, opts?.maxWhoisReferralHops ?? 2);
|
|
||||||
// First query against the provided server
|
|
||||||
let current = await tracedWhoisQuery(initialServer, domain, opts, ctx);
|
|
||||||
if (opts?.followWhoisReferral === false || maxHops === 0) return current;
|
|
||||||
|
|
||||||
const visited = new Set<string>([normalize(current.serverQueried)]);
|
|
||||||
let hops = 0;
|
|
||||||
// Iterate while we see a new referral and are under hop limit
|
|
||||||
while (hops < maxHops) {
|
|
||||||
throwIfAborted(opts?.signal);
|
|
||||||
const next = extractWhoisReferral(current.text);
|
|
||||||
if (!next) break;
|
|
||||||
if (!isSafeWhoisReferralHost(normalize(next))) break;
|
|
||||||
const normalized = normalize(next);
|
|
||||||
if (visited.has(normalized)) break; // cycle protection / same as current
|
|
||||||
visited.add(normalized);
|
|
||||||
try {
|
|
||||||
const res = await tracedWhoisQuery(next, domain, opts, ctx);
|
|
||||||
// Prefer authoritative TLD response when registrar contradicts availability
|
|
||||||
const registeredBefore = !isAvailableByWhois(current.text);
|
|
||||||
const registeredAfter = !isAvailableByWhois(res.text);
|
|
||||||
if (registeredBefore && !registeredAfter) {
|
|
||||||
// Registrar claims availability but TLD shows registered: keep TLD
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
current = res; // adopt registrar when it does not downgrade registration
|
|
||||||
} catch {
|
|
||||||
throwIfAborted(opts?.signal);
|
|
||||||
// If referral server fails, stop following and keep the last good response
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
hops += 1;
|
|
||||||
}
|
|
||||||
return current;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Collect the WHOIS referral chain starting from the TLD server.
|
* Collect the WHOIS referral chain starting from the TLD server.
|
||||||
@@ -88,7 +42,7 @@ export async function collectWhoisReferralChain(
|
|||||||
if (visited.has(normalized)) break;
|
if (visited.has(normalized)) break;
|
||||||
visited.add(normalized);
|
visited.add(normalized);
|
||||||
try {
|
try {
|
||||||
const res = await tracedWhoisQuery(next, domain, opts, ctx);
|
const res = await tracedWhoisQuery(next, domain, opts, ctx, true);
|
||||||
// If registrar claims availability while TLD indicated registered, stop.
|
// If registrar claims availability while TLD indicated registered, stop.
|
||||||
const registeredBefore = !isAvailableByWhois(current.text);
|
const registeredBefore = !isAvailableByWhois(current.text);
|
||||||
const registeredAfter = !isAvailableByWhois(res.text);
|
const registeredAfter = !isAvailableByWhois(res.text);
|
||||||
@@ -109,8 +63,8 @@ export async function collectWhoisReferralChain(
|
|||||||
throwIfAborted(opts?.signal);
|
throwIfAborted(opts?.signal);
|
||||||
const { code, error } = classifyError(err);
|
const { code, error } = classifyError(err);
|
||||||
warnings.push(
|
warnings.push(
|
||||||
code === "rate_limited"
|
code === "rate_limited" || code === "blocked"
|
||||||
? `WHOIS referral ${normalized} rate limited the query`
|
? `WHOIS referral ${normalized} ${code === "blocked" ? "blocked" : "rate limited"} the query`
|
||||||
: `WHOIS referral ${normalized} failed (${error})`,
|
: `WHOIS referral ${normalized} failed (${error})`,
|
||||||
);
|
);
|
||||||
break;
|
break;
|
||||||
@@ -130,17 +84,21 @@ function tracedWhoisQuery(
|
|||||||
domain: string,
|
domain: string,
|
||||||
opts?: LookupOptions,
|
opts?: LookupOptions,
|
||||||
ctx?: LookupContext,
|
ctx?: LookupContext,
|
||||||
|
referral = false,
|
||||||
): Promise<WhoisQueryResult> {
|
): Promise<WhoisQueryResult> {
|
||||||
return traced(
|
return traced(
|
||||||
ctx,
|
ctx,
|
||||||
{ phase: "whois", server: server.replace(/^whois:\/\//i, "") },
|
{ phase: "whois", server: server.replace(/^whois:\/\//i, "") },
|
||||||
async (notes) => {
|
async (notes) => {
|
||||||
const res = await whoisQuery(server, domain, opts);
|
const res = await whoisQuery(server, domain, opts, { blockPrivateAddresses: referral });
|
||||||
if (res.partial) notes.partial = true;
|
if (res.partial) notes.partial = true;
|
||||||
if (detectWhoisThrottle(res.text)) {
|
const refusal = detectWhoisRefusal(res.text);
|
||||||
|
if (refusal) {
|
||||||
throw new RdapperError(
|
throw new RdapperError(
|
||||||
"rate_limited",
|
refusal,
|
||||||
`WHOIS server ${res.serverQueried} rate limited the query`,
|
refusal === "blocked"
|
||||||
|
? `WHOIS server ${res.serverQueried} refuses requests from this client`
|
||||||
|
: `WHOIS server ${res.serverQueried} rate limited the query`,
|
||||||
{ stage: "read" },
|
{ stage: "read" },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
+24
-10
@@ -1,26 +1,40 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { detectWhoisThrottle, looksEmptyWhois } from "./throttle";
|
import { detectWhoisRefusal, looksEmptyWhois } from "./throttle";
|
||||||
|
|
||||||
describe("detectWhoisThrottle", () => {
|
describe("detectWhoisRefusal", () => {
|
||||||
it.each([
|
it.each([
|
||||||
"WHOIS LIMIT EXCEEDED - SEE WWW.PIR.ORG/WHOIS FOR DETAILS",
|
"WHOIS LIMIT EXCEEDED - SEE WWW.PIR.ORG/WHOIS FOR DETAILS",
|
||||||
"Quota exceeded",
|
"Quota exceeded",
|
||||||
"You have exceeded the allowed number of queries. Try again later.",
|
"You have exceeded the allowed number of queries.",
|
||||||
"Too many requests",
|
"Too many requests",
|
||||||
"<!DOCTYPE html><html><body>Service Unavailable</body></html>",
|
"<!DOCTYPE html><html><body>Service Unavailable</body></html>",
|
||||||
"Your IP has been blocked",
|
])("classifies %j as rate_limited", (text) => {
|
||||||
])("flags %j", (text) => {
|
expect(detectWhoisRefusal(text)).toBe("rate_limited");
|
||||||
expect(detectWhoisThrottle(text)).toBe(true);
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
"Requests of this client are not permitted. Please use https://www.nic.ch/whois/ for queries.",
|
||||||
|
"Your IP address has been blocked",
|
||||||
|
])("classifies %j as blocked", (text) => {
|
||||||
|
expect(detectWhoisRefusal(text)).toBe("blocked");
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
"This domain name is blocked. Try again later.",
|
||||||
|
"Access denied for reserved name. No match for EXAMPLE.COM",
|
||||||
|
"No match for EXAMPLE.COM",
|
||||||
|
"Domain is banned by policy",
|
||||||
|
])("leaves %j alone", (text) => {
|
||||||
|
expect(detectWhoisRefusal(text)).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not flag a long real record that mentions rate limits", () => {
|
it("does not flag a long real record that mentions rate limits", () => {
|
||||||
const record = `Domain Name: EXAMPLE.COM\nRegistrar: Test\n${"Name Server: NS.EXAMPLE.COM\n".repeat(100)}\nNote: queries are subject to rate limiting\n`;
|
const record = `Domain Name: EXAMPLE.COM\nRegistrar: Test\n${"Name Server: NS.EXAMPLE.COM\n".repeat(100)}\nNote: queries are subject to rate limiting\n`;
|
||||||
expect(detectWhoisThrottle(record)).toBe(false);
|
expect(detectWhoisRefusal(record)).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not flag empty or ordinary availability text", () => {
|
it("returns undefined for empty input", () => {
|
||||||
expect(detectWhoisThrottle(undefined)).toBe(false);
|
expect(detectWhoisRefusal(undefined)).toBeUndefined();
|
||||||
expect(detectWhoisThrottle("No match for EXAMPLE.COM")).toBe(false);
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
+21
-12
@@ -1,29 +1,38 @@
|
|||||||
import type { DomainRecord } from "../types";
|
import type { DomainRecord } from "../types";
|
||||||
|
import { isAvailableByWhois } from "./normalize";
|
||||||
|
|
||||||
// Real WHOIS records are much longer; throttle notices and error pages are short.
|
// Real WHOIS records are much longer; refusal notices and error pages are short.
|
||||||
const MAX_THROTTLE_TEXT_LENGTH = 2048;
|
const MAX_REFUSAL_TEXT_LENGTH = 2048;
|
||||||
|
|
||||||
|
// Transient: the same query may succeed later.
|
||||||
const THROTTLE_PATTERNS: RegExp[] = [
|
const THROTTLE_PATTERNS: RegExp[] = [
|
||||||
/rate[\s-]?limit/i,
|
/rate[\s-]?limit/i,
|
||||||
/limit\s+exceeded/i,
|
/limit\s+exceeded/i,
|
||||||
/quota\s+exceeded/i,
|
/quota\s+exceeded/i,
|
||||||
/exceeded\s+.{0,40}(quer|limit|request|connection)/i,
|
/exceeded\s+.{0,40}(quer|limit|request|connection)/i,
|
||||||
/too\s+many\s+(quer|request|connection)/i,
|
/too\s+many\s+(quer|request|connection)/i,
|
||||||
/try\s+again\s+(later|in)/i,
|
|
||||||
/access\s+(denied|limit)/i,
|
|
||||||
/\b(blocked|blacklisted|banned)\b/i,
|
|
||||||
/^\s*<(!doctype|html)\b/i,
|
/^\s*<(!doctype|html)\b/i,
|
||||||
];
|
];
|
||||||
|
|
||||||
|
// Permanent: this client is refused outright, so retrying will not help.
|
||||||
|
const BLOCK_PATTERNS: RegExp[] = [
|
||||||
|
/requests\s+of\s+this\s+client\s+are\s+not\s+permitted/i, // .ch/.li
|
||||||
|
/\b(your|this)\s+(ip|address|client)\b.{0,60}\b(blocked|banned|blacklisted|not\s+permitted)\b/i,
|
||||||
|
];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Heuristic: does this WHOIS response look like a throttle notice or error page rather than
|
* Classify a short WHOIS reply that is a refusal rather than a record: `rate_limited` for a
|
||||||
* a domain record? Only short responses qualify, so a real record that mentions "rate limit"
|
* transient throttle, `blocked` for a permanent block. Availability notices win, and long
|
||||||
* in a remark is not rejected.
|
* responses are never refusals, so a real record that mentions "rate limit" is not rejected.
|
||||||
*/
|
*/
|
||||||
export function detectWhoisThrottle(text: string | undefined): boolean {
|
export function detectWhoisRefusal(
|
||||||
if (!text) return false;
|
text: string | undefined,
|
||||||
if (text.length > MAX_THROTTLE_TEXT_LENGTH) return false;
|
): "rate_limited" | "blocked" | undefined {
|
||||||
return THROTTLE_PATTERNS.some((re) => re.test(text));
|
if (!text || text.length > MAX_REFUSAL_TEXT_LENGTH) return undefined;
|
||||||
|
if (isAvailableByWhois(text)) return undefined;
|
||||||
|
if (BLOCK_PATTERNS.some((re) => re.test(text))) return "blocked";
|
||||||
|
if (THROTTLE_PATTERNS.some((re) => re.test(text))) return "rate_limited";
|
||||||
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** True when a normalized WHOIS record carries none of the fields a real registration would. */
|
/** True when a normalized WHOIS record carries none of the fields a real registration would. */
|
||||||
|
|||||||
Reference in New Issue
Block a user