mirror of
https://github.com/jakejarvis/rdapper.git
synced 2026-09-23 00:15:31 -04:00
fix: extend Retry-After parsing to RDAP 503 and fix block pattern across newlines
- Parse `Retry-After` on RDAP `503` responses (previously only `429`), exposing `retryAfterMs` on the attempt and on the top-level result when `rdapOnly` makes it terminal - Fix the WHOIS `BLOCK_PATTERNS` regex to use `[\s\S]` instead of `.` so a block notice that spans multiple lines (e.g. `"Your IP address\nhas been blocked"`) is still classified as `blocked` - Update README to document `retryAfterMs` on `LookupResult`, clarify `rate_limited` / `blocked` / `unparseable` semantics, and note referral-host validation behaviour
This commit is contained in:
@@ -449,11 +449,20 @@ interface LookupResult {
|
|||||||
errorCode?: LookupErrorCode; // machine-readable, present when ok is false
|
errorCode?: LookupErrorCode; // machine-readable, present when ok is false
|
||||||
errorPhase?: "rdap_bootstrap" | "rdap" | "rdap_link" | "iana" | "whois";
|
errorPhase?: "rdap_bootstrap" | "rdap" | "rdap_link" | "iana" | "whois";
|
||||||
errorServer?: string; // RDAP URL or WHOIS host involved in the failure
|
errorServer?: string; // RDAP URL or WHOIS host involved in the failure
|
||||||
|
retryAfterMs?: number; // server's Retry-After, when it was the terminal failure (see below)
|
||||||
attempts: LookupAttempt[]; // every network operation, in order (always present)
|
attempts: LookupAttempt[]; // every network operation, in order (always present)
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
`errorCode` is one of `invalid_input`, `invalid_tld`, `timeout`, `aborted`, `connect_failed`, `http_error`, `rdap_unavailable`, `no_server`, `no_data`, `unsupported_runtime`, or `unknown`. Prefer it over matching `error` text. `timeout` covers every timeout, including `deadlineMs`; `aborted` means your own `signal` fired.
|
`errorCode` is one of `invalid_input`, `invalid_tld`, `timeout`, `aborted`, `connect_failed`, `http_error`, `rdap_unavailable`, `no_server`, `no_data`, `rate_limited`, `blocked`, `unparseable`, `unsupported_runtime`, or `unknown`. Prefer it over matching `error` text. `timeout` covers every timeout, including `deadlineMs`; `aborted` means your own `signal` fired.
|
||||||
|
|
||||||
|
- `rate_limited`: the server throttled the query (RDAP `429`, or a short WHOIS notice such as `WHOIS LIMIT EXCEEDED`). Retrying later may work.
|
||||||
|
- `blocked`: a WHOIS server refuses this client outright (e.g. `.ch`: "Requests of this client are not permitted"). Retrying will not help.
|
||||||
|
- `unparseable`: WHOIS replied with text that is neither an availability notice nor a domain record (no registrar, dates, nameservers, statuses or contacts). It is reported as a failure rather than a "registered" record.
|
||||||
|
|
||||||
|
`retryAfterMs` carries an RDAP `Retry-After` header (on `429` or `503`, as seconds or an HTTP date). It is set on the matching entry in `attempts`, and on the top-level result only when that RDAP attempt is the terminal failure, as with `rdapOnly`. Normally an RDAP failure falls through to WHOIS, so the value stays in `attempts`. The value is passed through as sent and is not capped, so clamp it before using it as a delay.
|
||||||
|
|
||||||
|
WHOIS referral hosts (from `Registrar WHOIS Server:` and similar fields) come from upstream response text, so they are validated (hostname syntax, no private/loopback/link-local IP literals) and their resolved address is checked at connect time. An unsafe, blocked or throttled referral is skipped with an entry in `record.warnings`. The first WHOIS server, from IANA or `whoisHints`, is trusted and not checked.
|
||||||
|
|
||||||
Each entry in `attempts` describes one operation, successful or not, so a failure that was recovered from (say, an RDAP server that was down before WHOIS answered) is still visible:
|
Each entry in `attempts` describes one operation, successful or not, so a failure that was recovered from (say, an RDAP server that was down before WHOIS answered) is still visible:
|
||||||
|
|
||||||
|
|||||||
@@ -91,6 +91,19 @@ describe("throttle and empty-response guards", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("rdapOnly rate limiting", () => {
|
describe("rdapOnly rate limiting", () => {
|
||||||
|
it("parses Retry-After on a 503", async () => {
|
||||||
|
const customFetch: FetchLike = vi.fn(
|
||||||
|
async () => new Response("", { status: 503, headers: { "retry-after": "5" } }),
|
||||||
|
);
|
||||||
|
const res = await lookup("example.com", {
|
||||||
|
customBootstrapData: bootstrap,
|
||||||
|
customFetch,
|
||||||
|
rdapOnly: true,
|
||||||
|
});
|
||||||
|
expect(res.errorCode).toBe("rdap_unavailable");
|
||||||
|
expect(res.retryAfterMs).toBe(5_000);
|
||||||
|
});
|
||||||
|
|
||||||
it("surfaces retryAfterMs on the result", async () => {
|
it("surfaces retryAfterMs on the result", async () => {
|
||||||
const customFetch: FetchLike = vi.fn(
|
const customFetch: FetchLike = vi.fn(
|
||||||
async () => new Response("", { status: 429, headers: { "retry-after": "12" } }),
|
async () => new Response("", { status: 429, headers: { "retry-after": "12" } }),
|
||||||
|
|||||||
+7
-1
@@ -64,7 +64,13 @@ export async function fetchRdapDomain(
|
|||||||
}
|
}
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
const bodyText = await res.text().catch(() => "");
|
const bodyText = await res.text().catch(() => "");
|
||||||
throw new RdapperError("http_error", `RDAP ${res.status}: ${bodyText.slice(0, 500)}`);
|
const retryAfterMs =
|
||||||
|
res.status === 503 ? parseRetryAfterMs(res.headers.get("retry-after")) : undefined;
|
||||||
|
throw new RdapperError(
|
||||||
|
"http_error",
|
||||||
|
`RDAP ${res.status}: ${bodyText.slice(0, 500)}`,
|
||||||
|
retryAfterMs !== undefined ? { retryAfterMs } : undefined,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
const json = await res.json();
|
const json = await res.json();
|
||||||
return { url, json };
|
return { url, json };
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ describe("detectWhoisRefusal", () => {
|
|||||||
it.each([
|
it.each([
|
||||||
"Requests of this client are not permitted. Please use https://www.nic.ch/whois/ for queries.",
|
"Requests of this client are not permitted. Please use https://www.nic.ch/whois/ for queries.",
|
||||||
"Your IP address has been blocked",
|
"Your IP address has been blocked",
|
||||||
|
"Your IP address\nhas been blocked",
|
||||||
])("classifies %j as blocked", (text) => {
|
])("classifies %j as blocked", (text) => {
|
||||||
expect(detectWhoisRefusal(text)).toBe("blocked");
|
expect(detectWhoisRefusal(text)).toBe("blocked");
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ const THROTTLE_PATTERNS: RegExp[] = [
|
|||||||
// Permanent: this client is refused outright, so retrying will not help.
|
// Permanent: this client is refused outright, so retrying will not help.
|
||||||
const BLOCK_PATTERNS: RegExp[] = [
|
const BLOCK_PATTERNS: RegExp[] = [
|
||||||
/requests\s+of\s+this\s+client\s+are\s+not\s+permitted/i, // .ch/.li
|
/requests\s+of\s+this\s+client\s+are\s+not\s+permitted/i, // .ch/.li
|
||||||
/\b(your|this)\s+(ip|address|client)\b.{0,60}\b(blocked|banned|blacklisted|not\s+permitted)\b/i,
|
/\b(your|this)\s+(ip|address|client)\b[\s\S]{0,60}\b(blocked|banned|blacklisted|not\s+permitted)\b/i,
|
||||||
];
|
];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in New Issue
Block a user