diff --git a/README.md b/README.md index e659f1e..7f4999b 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ import { toRegistrableDomain } from "rdapper"; toRegistrableDomain("https://sub.example.co.uk/page"); // => "example.co.uk" toRegistrableDomain("spark-public.s3.amazonaws.com"); // => "amazonaws.com" (ICANN-only default) +toRegistrableDomain("www.google.uk.com"); // => "google.uk.com" (CentralNic sub-registry) toRegistrableDomain("192.168.0.1"); // => null ``` @@ -60,13 +61,15 @@ await isAvailable("likely-unregistered-thing-320485230458.com"); // => true - `domain` may be Unicode (`münchen.de`) and may carry surrounding whitespace or a trailing dot; the lowercase punycode form is what gets queried. - `domain` must be a registrable name (`example.com`, `example.co.uk`), not a subdomain or a bare public suffix: `www.example.com` fails with `invalid_input` naming `example.com`, since the registry would otherwise answer "not registered". Use `toRegistrableDomain` to normalize hostnames and URLs first. - `toRegistrableDomain(input, options?) => string | null` - - Normalizes a domain or URL to its registrable domain (eTLD+1). + - Normalizes a domain or URL to its registrable domain (eTLD+1), the name `lookup()` accepts. A name under a sub-registry keeps its own label (`google.uk.com`, not `uk.com`). - Returns the registrable domain string, or `null` for IPs/invalid input; [options](https://github.com/remusao/tldts/blob/master/packages/tldts-core/src/options.ts) are forwarded to `tldts` (e.g., `allowPrivateDomains`). +- `getDomainTld(domain, options?) => string | null` + - The suffix a name is registered under, matching the `tld` a lookup reports: the ICANN public suffix (`co.uk`), or a sub-registry's (`uk.com` for `google.uk.com`). - `isRegistered(domain, options?) => Promise` - `isAvailable(domain, options?) => Promise` - Reject when the lookup fails, with an `RdapperError` (exported) whose `code`, `phase`, `server`, and `retryAfterMs` mirror the result's `errorCode`, `errorPhase`, `errorServer`, and `retryAfterMs`. An abort keeps the name `"AbortError"`. - `finalizeContact(contact, redactedHint?) => Contact` - - Cleans a contact: drops placeholder values (recording them in `redactedFields`), sets `privacyService`, resolves `country`/`countryCode`, and sets `redacted`. Safe to re-run on an already-cleaned contact, e.g. to upgrade contacts stored by an older version. + - Cleans a contact: drops placeholder values (recording them in `redactedFields`), drops a registry handle (`JJ1234-IS`) given as the name or organization, sets `privacyService`, resolves `country`/`countryCode`, and sets `redacted`. Safe to re-run on an already-cleaned contact, e.g. to upgrade contacts stored by an older version. - `isPrivacyContact(contact) => boolean` - True when the contact hides who it is: a privacy service is named, or something was withheld and no name or organization is left. - `normalizeEppStatus(status) => string` @@ -441,7 +444,7 @@ const result = await lookup("example.com", { - `rdapFollowLinks?: boolean` – Follow related/entity RDAP links to enrich data (default `true`). - `maxRdapLinkHops?: number` – Maximum RDAP related link fetches, following links found in fetched documents too (default `2`). - `rdapLinkRels?: string[]` – RDAP link rel values to consider (default `["related","entity","registrar","alternate"]`). -- `customBootstrapData?: BootstrapData` – Pre-loaded RDAP bootstrap data for caching control (see [Bootstrap Data Caching](#bootstrap-data-caching)). +- `customBootstrapData?: BootstrapData` – Pre-loaded RDAP bootstrap data for caching control (see [Bootstrap Data Caching](#bootstrap-data-caching)). `undefined` or `null` counts as not provided, so a failed cache read can be passed straight through; an object without a `services` array fails the lookup with `invalid_input`. - `customBootstrapUrl?: string` – Override RDAP bootstrap URL (ignored if `customBootstrapData` is provided). - `customFetch?: FetchLike` – Custom fetch implementation for all HTTP requests (see [Custom Fetch Implementation](#custom-fetch-implementation)). - `whoisHints?: Record` – Override/add authoritative WHOIS per TLD (keys are lowercase TLDs, values may include or omit `whois://`). @@ -476,6 +479,8 @@ interface LookupResult { WHOIS referral hosts (from `Registrar WHOIS Server:` and similar fields) come from upstream response text, so they are validated (hostname syntax, no private/loopback/link-local IP literals) and their resolved address is checked at connect time. An unsafe, blocked or throttled referral is skipped with an entry in `record.warnings`. The first WHOIS server, from IANA or `whoisHints`, is trusted and not checked. +RDAP links (`related` and the other `rdapLinkRels`) come from upstream responses too, so a link is only followed when it is `http(s)` to a public hostname or IP literal, without credentials. This is a literal check: a hostname that resolves to a private address, or a redirect to one, still gets through, so pass a `customFetch` that checks resolved addresses when lookups take untrusted input. The RDAP servers from the IANA bootstrap (or `customBootstrapData`) are trusted. + Each entry in `attempts` describes one operation, successful or not, so a failure that was recovered from (say, an RDAP server that was down before WHOIS answered) is still visible: ```json @@ -663,7 +668,7 @@ Project layout: - Some TLDs provide no RDAP service; `rdapOnly: true` will fail for them. - Registries may throttle or block WHOIS; respect rate limits and usage policies. - Field presence depends on source and privacy policies (e.g., redaction/withholding). -- Public suffix detection uses `tldts` with ICANN‑only defaults (Private section is ignored). You can pass options through to `tldts` via `toRegistrableDomain`/`getDomainParts`/`getDomainTld` (e.g., `allowPrivateDomains`) to customize behavior. `lookup()` additionally routes names under CentralNic’s sub-registries (e.g., `google.uk.com`) to CentralNic rather than the `.com` registry. See: [tldts migration notes](https://github.com/remusao/tldts#migrating-from-other-libraries). +- Public suffix detection uses `tldts` with ICANN‑only defaults (Private section is ignored). You can pass options through to `tldts` via `toRegistrableDomain`/`getDomainParts`/`getDomainTld` (e.g., `allowPrivateDomains`) to customize behavior. `lookup()` additionally routes names under CentralNic’s sub-registries (e.g., `google.uk.com`) to CentralNic rather than the `.com` registry, and `toRegistrableDomain`/`getDomainTld` treat those names the same way. See: [tldts migration notes](https://github.com/remusao/tldts#migrating-from-other-libraries). ## License diff --git a/src/index.ts b/src/index.ts index ce9d291..2604fb0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,12 @@ import { linkSignals, throwIfAborted } from "./lib/async"; -import { getDomainParts, isLikelyDomain, toAsciiDomain } from "./lib/domain"; +import { + getDomainParts, + getDomainTld, + isLikelyDomain, + toAsciiDomain, + toRegistrableDomain, +} from "./lib/domain"; import { classifyError, RdapperError } from "./lib/errors"; -import { getSubRegistrySuffix } from "./lib/subregistries"; import { attemptForError, type LookupContext } from "./lib/trace"; import { getRdapBaseUrlsForPublicSuffix } from "./rdap/bootstrap"; import { fetchRdapDomain } from "./rdap/client"; @@ -26,14 +31,7 @@ import { collectWhoisReferralChain } from "./whois/referral"; * from the Public Suffix List the input is taken as is. */ function registrableNameOf(domain: string): string | undefined { - const sub = getSubRegistrySuffix(domain); - if (sub) - return domain - .split(".") - .slice(-(sub.split(".").length + 1)) - .join("."); - const parts = getDomainParts(domain); - return parts.isIcann ? (parts.domain ?? undefined) : domain; + return toRegistrableDomain(domain) ?? (getDomainParts(domain).isIcann ? undefined : domain); } function failure( @@ -129,7 +127,7 @@ async function runLookup( } // Names under a sub-registry (e.g. google.uk.com) are routed by that suffix, not the ICANN one - const tld = getSubRegistrySuffix(domain) ?? getDomainParts(domain).publicSuffix; + const tld = getDomainTld(domain); if (!tld) { return failure(ctx, "invalid_tld", "Invalid TLD"); } diff --git a/src/lib/contacts.test.ts b/src/lib/contacts.test.ts index 154c00b..ce09583 100644 --- a/src/lib/contacts.test.ts +++ b/src/lib/contacts.test.ts @@ -29,6 +29,24 @@ test("finalizeContact keeps privacy-service names and flags privacyService", () expect(c.redactedFields).toBeUndefined(); }); +test("finalizeContact drops a registry handle given as the name, without calling it redacted", () => { + const c = finalizeContact({ + type: "registrant", + name: "JJ1234-IS", + organization: "AB1-NORID", + country: "IS", + }); + expect(c.name).toBeUndefined(); + expect(c.organization).toBeUndefined(); + expect(c.countryCode).toBe("IS"); + expect(c.redacted).toBeUndefined(); + expect(c.redactedFields).toBeUndefined(); + expect(isPrivacyContact(c)).toBe(false); + + // A name that only resembles a handle stays + expect(finalizeContact({ type: "registrant", organization: "3M" }).organization).toBe("3M"); +}); + test("finalizeContact cleans address fields and leaves real contacts untouched", () => { const c = finalizeContact({ type: "admin", diff --git a/src/lib/contacts.ts b/src/lib/contacts.ts index deeb694..eb01cef 100644 --- a/src/lib/contacts.ts +++ b/src/lib/contacts.ts @@ -1,5 +1,6 @@ import type { Contact, ContactField } from "../types"; import { countryCodeFromName, resolveCountry } from "./countries"; +import { isRegistryHandle } from "./handles"; import { isPlaceholderValue, isPrivacyName } from "./privacy"; function cleanValue( @@ -96,8 +97,9 @@ export function redactionFields(name: string, prePath?: string): ContactField[] /** * Post-process a parsed contact: drop placeholder values from every field (recording each in - * `redactedFields`), flag privacy-service names (`privacyService`), resolve country/countryCode, - * and set `redacted` when any redaction signal is present. + * `redactedFields`), drop a registry handle given as the name or organization, flag + * privacy-service names (`privacyService`), resolve country/countryCode, and set `redacted` when + * any redaction signal is present. * * `redactedHint` lets callers pass format-specific signals (e.g. RFC 9537 entries): `true` when * something was redacted but the fields are unknown, or the list of fields known to be redacted. @@ -126,6 +128,13 @@ export function finalizeContact( if (dropped) redacted = true; } + // A registry handle ("JJ1234-IS") in place of a name identifies no one. It isn't a redaction + // either: the registry published a reference, not a withheld value. + for (const key of ["name", "organization"] as const) { + const value = contact[key]; + if (value && isRegistryHandle(value)) contact[key] = undefined; + } + // Remaining name/organization text that names a privacy service is kept, since it is useful to show. if ([contact.name, contact.organization].some((v) => v && isPrivacyName(v))) { contact.privacyService = true; diff --git a/src/lib/domain.test.ts b/src/lib/domain.test.ts index d3d6f0b..81850fc 100644 --- a/src/lib/domain.test.ts +++ b/src/lib/domain.test.ts @@ -1,5 +1,11 @@ import { expect, test } from "vitest"; -import { getDomainParts, isLikelyDomain, toAsciiDomain, toRegistrableDomain } from "./domain"; +import { + getDomainParts, + getDomainTld, + isLikelyDomain, + toAsciiDomain, + toRegistrableDomain, +} from "./domain"; test("getDomainParts.tld basic", () => { expect(getDomainParts("example.com").publicSuffix).toBe("com"); @@ -51,3 +57,21 @@ test("toRegistrableDomain honors allowPrivateDomains", () => { "example.co.uk", ); }); + +test("toRegistrableDomain keeps the name under a sub-registry, as lookup() expects", () => { + expect(toRegistrableDomain("google.uk.com")).toBe("google.uk.com"); + expect(toRegistrableDomain("https://www.Google.UK.com/path")).toBe("google.uk.com"); + expect(toRegistrableDomain("www.google.uk.com.")).toBe("google.uk.com"); + expect(toRegistrableDomain("foo.nhs.uk")).toBe("foo.nhs.uk"); + expect(toRegistrableDomain("www.ox.ac.uk")).toBe("ox.ac.uk"); + // The sub-registry's own domain is an ordinary .com name + expect(toRegistrableDomain("uk.com")).toBe("uk.com"); +}); + +test("getDomainTld reports the sub-registry suffix a name is registered under", () => { + expect(getDomainTld("example.co.uk")).toBe("co.uk"); + expect(getDomainTld("google.uk.com")).toBe("uk.com"); + expect(getDomainTld("www.google.uk.com")).toBe("uk.com"); + expect(getDomainTld("uk.com")).toBe("com"); + expect(getDomainTld("192.168.0.1")).toBeNull(); +}); diff --git a/src/lib/domain.ts b/src/lib/domain.ts index aee85ae..12bd3cb 100644 --- a/src/lib/domain.ts +++ b/src/lib/domain.ts @@ -1,4 +1,5 @@ import { parse } from "tldts"; +import { getSubRegistrySuffix } from "./subregistries"; type ParseOptions = Parameters[1]; @@ -11,7 +12,9 @@ export function getDomainParts(domain: string, opts?: ParseOptions): ReturnType< } /** - * Get the TLD (ICANN-only public suffix) of a domain. Passes options to `tldts.parse()`. + * Get the TLD of a domain: the ICANN public suffix, or for a name under a sub-registry the suffix + * it is registered under ("uk.com" for google.uk.com), matching the `tld` a lookup reports. + * Passes options to `tldts.parse()`. * @see https://github.com/remusao/tldts/blob/master/packages/tldts-core/src/options.ts */ export function getDomainTld(domain: string, opts?: ParseOptions): string | null { @@ -19,7 +22,8 @@ export function getDomainTld(domain: string, opts?: ParseOptions): string | null allowPrivateDomains: false, ...opts, }); - return result.publicSuffix ?? null; + const sub = result.hostname && !result.isIp ? getSubRegistrySuffix(result.hostname) : undefined; + return sub ?? result.publicSuffix ?? null; } /** @@ -52,8 +56,9 @@ export function isLikelyDomain(value: string): boolean { } /** - * Normalize arbitrary input (domain or URL) to its registrable domain (eTLD+1). - * Passes options to `tldts.parse()`. + * Normalize arbitrary input (domain or URL) to its registrable domain (eTLD+1), the name a + * `lookup()` accepts. A name under a sub-registry keeps its own label ("google.uk.com", not + * "uk.com"). Passes options to `tldts.parse()`. * Returns null when the input is not a valid ICANN domain (e.g., invalid TLD, IPs), or with * `allowPrivateDomains`, a valid ICANN or private one. * @see https://github.com/remusao/tldts/blob/master/packages/tldts-core/src/options.ts @@ -71,6 +76,17 @@ export function toRegistrableDomain(input: string, opts?: ParseOptions): string if (result.isIp) return null; if (!result.isIcann && !(opts?.allowPrivateDomains && result.isPrivate)) return null; + // Names under a sub-registry (google.uk.com) are registered with its operator, one label below + // the suffix; the Public Suffix List's ICANN section stops at "uk.com" + const host = result.hostname?.toLowerCase().replace(/\.$/, ""); + const sub = host ? getSubRegistrySuffix(host) : undefined; + if (host && sub) { + return host + .split(".") + .slice(-(sub.split(".").length + 1)) + .join("."); + } + const domain = result.domain ?? ""; if (domain === "") return null; return domain.toLowerCase(); diff --git a/src/lib/handles.test.ts b/src/lib/handles.test.ts new file mode 100644 index 0000000..87e8377 --- /dev/null +++ b/src/lib/handles.test.ts @@ -0,0 +1,38 @@ +import { expect, test } from "vitest"; +import { isRegistryHandle, looksLikeHandle } from "./handles"; + +test("isRegistryHandle recognizes NIC handles", () => { + for (const handle of [ + "JJ1234-IS", + "NG8867695-NICAT", + "C123-LRMS", + "ABC123-RIPE", + "AB1-NORID", + "G31071", + "C12345678", + " JJ1234-IS ", + ]) { + expect(isRegistryHandle(handle), handle).toBe(true); + } +}); + +test("isRegistryHandle keeps names that only resemble handles", () => { + for (const name of [ + "3M", + "ACME-CORP", + "Jane Doe", + "Example Org 1234", + "jj1234-is", + "Company123", + ]) { + expect(isRegistryHandle(name), name).toBe(false); + } +}); + +test("looksLikeHandle is broad: any single token with a digit, or an uppercase code", () => { + expect(looksLikeHandle("292")).toBe(true); + expect(looksLikeHandle("CZ-NIC")).toBe(true); + expect(looksLikeHandle("Company123")).toBe(true); + expect(looksLikeHandle("MarkMonitor")).toBe(false); + expect(looksLikeHandle("Example Registrar 2")).toBe(false); +}); diff --git a/src/lib/handles.ts b/src/lib/handles.ts new file mode 100644 index 0000000..51f355e --- /dev/null +++ b/src/lib/handles.ts @@ -0,0 +1,22 @@ +/** + * A single token with a digit, or an uppercase hyphenated code: a registry handle. Broad, so only + * for fields that usually hold a handle (WHOIS's bare "registrant:" key, an RDAP entity handle). + */ +export function looksLikeHandle(value: string): boolean { + const v = value.trim(); + return !/\s/.test(v) && (/\d/.test(v) || /^[A-Z]+(?:-[A-Z]+)+$/.test(v)); +} + +// NIC handles: "JJ1234-IS", "NG8867695-NICAT", "C123-LRMS"; a registry-suffixed code such as +// "ABC123-RIPE" or "AB1-NORID"; or letters and a long number, "G31071", "C12345678". Codes without +// a digit only count with a known registry suffix, so all-caps names like "ACME-CORP" survive. +const NIC_HANDLE = + /^(?:[A-Z]{1,6}\d{1,10}-[A-Z]{2,8}|[A-Z0-9]{2,10}-(?:IS|RIPE|APNIC|AP|ARIN|NORID|SE|NL|DK|FI)|[A-Z]{1,3}\d{5,})$/; + +/** + * Whether a value is a registry handle rather than a name. Strict, for fields that usually hold a + * name (a contact's name or organization), where "3M" or "ACME-CORP" must survive. + */ +export function isRegistryHandle(value: string): boolean { + return NIC_HANDLE.test(value.trim()); +} diff --git a/src/rdap/bootstrap.test.ts b/src/rdap/bootstrap.test.ts index 5774339..9176a46 100644 --- a/src/rdap/bootstrap.test.ts +++ b/src/rdap/bootstrap.test.ts @@ -1,4 +1,5 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { RdapperError } from "../lib/errors"; import type { BootstrapData } from "../types"; import { getRdapBaseUrlsForPublicSuffix, getRdapBaseUrlsForTld } from "./bootstrap"; @@ -150,26 +151,53 @@ describe("getRdapBaseUrlsForTld with customBootstrapData", () => { }); }); + describe("customBootstrapData set to undefined or null", () => { + it.each([undefined, null])( + "fetches the bootstrap as if it were omitted (%s)", + async (value) => { + vi.mocked(fetch).mockResolvedValue({ + ok: true, + json: async () => validBootstrapData, + } as Response); + + const urls = await getRdapBaseUrlsForTld("com", { + customBootstrapData: value as unknown as BootstrapData, + }); + + expect(urls).toEqual(["https://rdap.verisign.com/com/v1/"]); + expect(fetch).toHaveBeenCalledTimes(1); + }, + ); + }); + describe("invalid customBootstrapData validation", () => { - it("should throw when customBootstrapData is null", async () => { - await expect( - getRdapBaseUrlsForTld("com", { - customBootstrapData: null as unknown as BootstrapData, - }), - ).rejects.toThrow( - "Invalid customBootstrapData: expected an object. See BootstrapData type for required structure.", - ); - expect(fetch).not.toHaveBeenCalled(); + it("rejects with an invalid_input RdapperError", async () => { + const err = await getRdapBaseUrlsForTld("com", { + customBootstrapData: "invalid" as unknown as BootstrapData, + }).catch((e: unknown) => e); + + expect(err).toBeInstanceOf(RdapperError); + expect(err).toMatchObject({ code: "invalid_input" }); }); - it("should throw when customBootstrapData is undefined", async () => { - await expect( - getRdapBaseUrlsForTld("com", { - customBootstrapData: undefined as unknown as BootstrapData, - }), - ).rejects.toThrow( - "Invalid customBootstrapData: expected an object. See BootstrapData type for required structure.", - ); + it("skips malformed service entries instead of rejecting", async () => { + const urls = await getRdapBaseUrlsForTld("com", { + customBootstrapData: { + version: "1.0", + publication: "2025-01-15T12:00:00Z", + services: [ + "garbage", + [["com"]], + [["com"], "https://not-a-list.example/"], + [ + ["com", 42], + ["https://rdap.verisign.com/com/v1/", null], + ], + ], + } as unknown as BootstrapData, + }); + + expect(urls).toEqual(["https://rdap.verisign.com/com/v1/"]); expect(fetch).not.toHaveBeenCalled(); }); diff --git a/src/rdap/bootstrap.ts b/src/rdap/bootstrap.ts index abe93d8..3579767 100644 --- a/src/rdap/bootstrap.ts +++ b/src/rdap/bootstrap.ts @@ -11,7 +11,7 @@ import type { BootstrapData, LookupOptions } from "../types"; * recorded in `ctx.attempts` and the caller falls back to WHOIS). * * Bootstrap data is resolved in the following priority order: - * 1. `options.customBootstrapData` - pre-loaded bootstrap data (no fetch) + * 1. `options.customBootstrapData` - pre-loaded bootstrap data (no fetch), unless undefined/null * 2. `options.customBootstrapUrl` - custom URL to fetch bootstrap data from * 3. Default IANA URL - https://data.iana.org/rdap/dns.json */ @@ -21,32 +21,24 @@ async function loadBootstrapData( ): Promise { let data: BootstrapData; - // Priority 1: Use pre-loaded bootstrap data if provided (no fetch) - if (options && "customBootstrapData" in options) { - const provided = options.customBootstrapData; - // Validate the structure to provide helpful error messages - if (!provided || typeof provided !== "object") { - throw new Error( + // Priority 1: Use pre-loaded bootstrap data if provided (no fetch). `undefined`/`null` count as + // not provided, so a caller whose own fetch failed can pass its result straight through. + const provided = options?.customBootstrapData; + if (provided != null) { + // A wrong shape is the caller's bug, not a missing server: fail as invalid_input instead of + // quietly falling back. Malformed service entries are only skipped (see matchBases). + if (typeof provided !== "object") { + throw new RdapperError( + "invalid_input", "Invalid customBootstrapData: expected an object. See BootstrapData type for required structure.", ); } if (!Array.isArray(provided.services)) { - throw new Error( + throw new RdapperError( + "invalid_input", 'Invalid customBootstrapData: missing or invalid "services" array. See BootstrapData type for required structure.', ); } - provided.services.forEach((svc, idx) => { - if ( - !Array.isArray(svc) || - svc.length < 2 || - !Array.isArray(svc[0]) || - !Array.isArray(svc[1]) - ) { - throw new Error( - `Invalid customBootstrapData: services[${idx}] must be a tuple of [string[], string[]].`, - ); - } - }); data = provided; } else { // Priority 2 & 3: Fetch from custom URL or default IANA URL @@ -93,9 +85,10 @@ function matchBases(data: BootstrapData, tld: string): string[] { const target = tld.toLowerCase(); const bases: string[] = []; for (const svc of data.services) { - if (!svc[0] || !svc[1]) continue; - const tlds = svc[0].map((x) => x.toLowerCase()); - const urls = svc[1]; + // Skip an entry that isn't a [string[], string[]] tuple rather than failing the lookup + if (!Array.isArray(svc) || !Array.isArray(svc[0]) || !Array.isArray(svc[1])) continue; + const tlds = svc[0].filter((x) => typeof x === "string").map((x) => x.toLowerCase()); + const urls = svc[1].filter((u) => typeof u === "string"); // Match exact TLD, and also support multi-label public suffixes present in IANA (rare) if (tlds.includes(target)) { for (const u of urls) { diff --git a/src/rdap/links.ts b/src/rdap/links.ts index a34ad88..6415f34 100644 --- a/src/rdap/links.ts +++ b/src/rdap/links.ts @@ -1,4 +1,5 @@ import type { LookupOptions } from "../types"; +import { isSafePublicHost } from "../whois/host"; type RdapLink = { value?: string; @@ -21,10 +22,28 @@ export function selfLinks(doc: unknown): string[] { .map((l) => l.href as string); } +/** + * Whether a link taken from a server's response may be followed: http(s) to a public host, with + * no credentials. A literal check, as for WHOIS referrals: a hostname that resolves to a private + * address, or a redirect to one, is only caught by a guarded `customFetch`. + */ +function isFollowableUrl(url: string): boolean { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return false; + } + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return false; + if (parsed.username || parsed.password) return false; + // IPv6 literals come bracketed ("[::1]") + return isSafePublicHost(parsed.hostname.replace(/^\[(.*)\]$/, "$1")); +} + /** * Extract candidate RDAP link URLs from an RDAP document, "related" links (to the registrar's * copy of the domain) first. A link's target is its href; its value is the context, the - * document itself. + * document itself. Links to private or local hosts are dropped (see {@link isFollowableUrl}). */ export function extractRdapRelatedLinks( doc: unknown, @@ -41,7 +60,7 @@ export function extractRdapRelatedLinks( if (!rels.includes(rel)) continue; if (type && !/application\/rdap\+json/i.test(type)) continue; const url = link.href; - if (typeof url !== "string" || !/^https?:\/\//i.test(url) || self.has(url)) continue; + if (typeof url !== "string" || self.has(url) || !isFollowableUrl(url)) continue; out.push({ url, related: rel === "related" }); } out.sort((a, b) => Number(b.related) - Number(a.related)); diff --git a/src/rdap/merge.test.ts b/src/rdap/merge.test.ts index f3409a1..2fef3f9 100644 --- a/src/rdap/merge.test.ts +++ b/src/rdap/merge.test.ts @@ -120,4 +120,31 @@ describe("fetchAndMergeRdapRelated", () => { expect(customFetch).not.toHaveBeenCalled(); expect(serversTried).toEqual([]); }); + + it("doesn't follow links to private, local, or credentialed hosts", async () => { + const base = { + ldhName: "example.com", + links: [ + related("http://169.254.169.254/latest/meta-data/"), + related("http://127.1/domain/example.com"), + related("http://[::1]:8080/domain/example.com"), + related("https://localhost/domain/example.com"), + related("https://rdap.internal/domain/example.com"), + related("https://user:pass@rdap.registrar.test/domain/example.com"), + related("ftp://rdap.registrar.test/domain/example.com"), + related("https://rdap.registrar.test:8443/domain/example.com"), + ], + }; + const customFetch = serve({}); + const { serversTried } = await fetchAndMergeRdapRelated("example.com", base, { + customFetch, + maxRdapLinkHops: 10, + }); + // Only the public host is fetched (a port is fine) + expect(customFetch).toHaveBeenCalledTimes(1); + expect(customFetch.mock.calls[0]?.[0]).toBe( + "https://rdap.registrar.test:8443/domain/example.com", + ); + expect(serversTried).toEqual([]); + }); }); diff --git a/src/rdap/normalize.test.ts b/src/rdap/normalize.test.ts index 1f72002..13cf13f 100644 --- a/src/rdap/normalize.test.ts +++ b/src/rdap/normalize.test.ts @@ -377,6 +377,20 @@ test("normalizeRdap resolves registrar countryCode from the country name", () => expect(rec.registrar).toMatchObject({ country: "Canada", countryCode: "CA" }); }); +test("normalizeRdap names the registrar from its handle only when the handle isn't an ID", () => { + const registrarNamed = (handle: string) => + normalizeRdap( + "example.com", + "com", + { ldhName: "example.com", entities: [{ roles: ["registrar"], handle }] }, + [], + ).registrar?.name; + + expect(registrarNamed("292")).toBeUndefined(); + expect(registrarNamed("MMR-1234")).toBeUndefined(); + expect(registrarNamed("MarkMonitor")).toBe("MarkMonitor"); +}); + test("normalizeRdap ties a redaction path to redactedFields", () => { const entity = { roles: ["registrant"], diff --git a/src/rdap/normalize.ts b/src/rdap/normalize.ts index 39ec31a..afc9f8a 100644 --- a/src/rdap/normalize.ts +++ b/src/rdap/normalize.ts @@ -1,6 +1,7 @@ import { resolveCountry } from "../lib/countries"; import { finalizeContact, isPrivacyContact, mergeContacts, redactionFields } from "../lib/contacts"; import { toISO } from "../lib/dates"; +import { looksLikeHandle } from "../lib/handles"; import { mergeNameservers } from "../lib/nameservers"; import { normalizeEppStatus } from "../lib/status"; import { asDateLike, asString, asStringArray, uniqBy } from "../lib/text"; @@ -248,8 +249,11 @@ function registrarInfo(ent: unknown): RegistrarInfo { /iana\s*registrar\s*id/i.test(String(id?.type)), )?.identifier : undefined; + // Some registrars publish their name only as the entity handle, but most handles are IDs + // ("292", "MMR-1234"), which would pass for a registrar called "292" + const handle = asString((ent as RdapDoc)?.handle); return { - name: v.fn || v.org || asString((ent as RdapDoc)?.handle) || undefined, + name: v.fn || v.org || (handle && !looksLikeHandle(handle) ? handle : undefined), ianaId: asString(ianaId), url: v.url ?? undefined, email: v.email?.[0], diff --git a/src/whois/host.test.ts b/src/whois/host.test.ts index 8875ffc..823a585 100644 --- a/src/whois/host.test.ts +++ b/src/whois/host.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; -import { isSafeWhoisReferralHost } from "./host"; +import { isSafePublicHost } from "./host"; -describe("isSafeWhoisReferralHost", () => { +describe("isSafePublicHost", () => { it.each([ "whois.markmonitor.com", "whois.1api.net", @@ -11,7 +11,7 @@ describe("isSafeWhoisReferralHost", () => { "2606:4700:4700::1111", "::ffff:808:808", "whois.example.com.", - ])("accepts %s", (h) => expect(isSafeWhoisReferralHost(h)).toBe(true)); + ])("accepts %s", (h) => expect(isSafePublicHost(h)).toBe(true)); it.each([ "", @@ -49,5 +49,5 @@ describe("isSafeWhoisReferralHost", () => { "-bad.example.com", `${"a".repeat(64)}.example.com`, `${"a.".repeat(130)}com`, - ])("rejects %j", (h) => expect(isSafeWhoisReferralHost(h)).toBe(false)); + ])("rejects %j", (h) => expect(isSafePublicHost(h)).toBe(false)); }); diff --git a/src/whois/host.ts b/src/whois/host.ts index ec69ba9..8c9a2ce 100644 --- a/src/whois/host.ts +++ b/src/whois/host.ts @@ -93,11 +93,11 @@ export function isPrivateIp(ip: string): boolean { } /** - * Whether a WHOIS referral host taken from upstream response text is safe to connect to: - * a well-formed public hostname or public IP literal, with no port, path or userinfo. - * This is a literal check only; the resolved address is checked again at connect time. + * Whether a host taken from upstream response data (a WHOIS referral, an RDAP link) is safe to + * connect to: a well-formed public hostname or public IP literal, with no port, path or userinfo. + * This is a literal check only; WHOIS checks the resolved address again at connect time. */ -export function isSafeWhoisReferralHost(host: string): boolean { +export function isSafePublicHost(host: string): boolean { const value = host.trim().replace(/\.$/, ""); if (!value || value.length > 253) return false; diff --git a/src/whois/normalize.ts b/src/whois/normalize.ts index c967dbc..3aa8ee1 100644 --- a/src/whois/normalize.ts +++ b/src/whois/normalize.ts @@ -1,5 +1,6 @@ import { type ParseDateOptions, toISOFromTokens } from "../lib/dates"; import { finalizeContact, isPrivacyContact } from "../lib/contacts"; +import { looksLikeHandle } from "../lib/handles"; import { mergeNameservers } from "../lib/nameservers"; import { isEppStatus, normalizeEppStatus } from "../lib/status"; import { parseKeyValueBlocks, parseKeyValueLines, uniqBy } from "../lib/text"; @@ -526,12 +527,6 @@ function collectContacts(map: Record): Contact[] | undefined { return contacts.length ? contacts : undefined; } -/** A single token with a digit, or an uppercase hyphenated code: a registry handle. */ -function looksLikeHandle(value: string): boolean { - const v = value.trim(); - return !/\s/.test(v) && (/\d/.test(v) || /^[A-Z]+(?:-[A-Z]+)+$/.test(v)); -} - function multi(map: Record, keys: string[]): string[] | undefined { for (const k of keys) { const v = map[k]; diff --git a/src/whois/referral.ts b/src/whois/referral.ts index 8d39d82..3ded6fa 100644 --- a/src/whois/referral.ts +++ b/src/whois/referral.ts @@ -5,7 +5,7 @@ import type { LookupOptions } from "../types"; import type { WhoisQueryResult } from "./client"; import { whoisQuery } from "./client"; import { extractWhoisReferral } from "./discovery"; -import { isSafeWhoisReferralHost } from "./host"; +import { isSafePublicHost } from "./host"; import { isAvailableByWhois, normalizeWhois } from "./normalize"; import { detectWhoisRefusal, looksEmptyWhois } from "./throttle"; @@ -35,7 +35,7 @@ export async function collectWhoisReferralChain( const next = extractWhoisReferral(current.text); if (!next) break; const normalized = normalize(next); - if (!isSafeWhoisReferralHost(normalized)) { + if (!isSafePublicHost(normalized)) { warnings.push(`Skipped WHOIS referral to unsafe host "${next.slice(0, 100)}"`); break; }