jake/rdapper
1
Fork 0
mirror of https://github.com/jakejarvis/rdapper.git synced 2025-10-18 20:14:27 -04:00
Code Issues Packages Releases Activity

Enhance RDAP and WHOIS functionality with referral handling

Browse Source 
Added support for following registrar WHOIS referrals with configurable hop limits. Introduced new options in LookupOptions for maximum referral hops and RDAP link handling. Updated README to reflect these changes and improved the lookupDomain function to utilize the new referral logic. Added utility functions for merging RDAP documents and extracting related links.
This commit is contained in:
Jake Jarvis
2025-10-08 19:25:51 -04:00
parent 27b3187d33
commit f8280508ff
7 changed files with 294 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
README.md
View File

@@ -49,6 +49,10 @@ await isAvailable("likely-unregistered-thing-320485230458.com"); // => false
- `rdapOnly?: boolean` – Only attempt RDAP; do not fall back to WHOIS.
- `whoisOnly?: boolean` – Skip RDAP and query WHOIS directly.
- `followWhoisReferral?: boolean` – Follow registrar referral from the TLD WHOIS (default `true`).
- `maxWhoisReferralHops?: number` – Maximum registrar WHOIS referral hops to follow (default `2`).
- `rdapFollowLinks?: boolean` – Follow related/entity RDAP links to enrich data (default `true`).
- `maxRdapLinkHops?: number` – Maximum RDAP related link hops to follow (default `2`).
- `rdapLinkRels?: string[]` – RDAP link rel values to consider (default `["related","entity","registrar","alternate"]`).
- `customBootstrapUrl?: string` – Override RDAP bootstrap URL.
- `whoisHints?: Record<string, string>` – Override/add authoritative WHOIS per TLD (keys are lowercase TLDs, values may include or omit `whois://`).
- `includeRaw?: boolean` – Include `rawRdap`/`rawWhois` in the returned record (default `false`).
@@ -144,10 +148,11 @@ interface DomainRecord {
- RDAP
- Discovers base URLs for the TLD via IANA’s RDAP bootstrap JSON.
- Tries each base until one responds successfully; parses standard RDAP domain JSON.
- Optionally follows related/entity links to registrar RDAP resources and merges results (bounded by hop limits).
- Normalizes registrar (from `entities`), contacts (vCard), nameservers (`ipAddresses`), events (created/changed/expiration), statuses, and DNSSEC (`secureDNS`).
- WHOIS
- Discovers the authoritative TLD WHOIS via `whois.iana.org` (TCP 43), with curated exceptions for tricky zones and public SLDs.
- Queries the TLD WHOIS; if a registrar referral is present and `followWhoisReferral !== false`, follows one hop to the registrar WHOIS.
- Queries the TLD WHOIS and follows registrar referrals recursively up to `maxWhoisReferralHops` (unless disabled).
- Normalizes common key/value variants across gTLD/ccTLD formats (dates, statuses, nameservers, contacts). Availability is inferred from common phrases (best‑effort heuristic).
Timeouts are enforced per request using a simple race against `timeoutMs` (default 15s). All network I/O is performed with global `fetch` (RDAP) and a raw TCP socket (WHOIS).

72
src/index.test.ts
View File

@@ -8,7 +8,14 @@ vi.mock("./rdap/bootstrap.js", () => ({
vi.mock("./rdap/client.js", () => ({
fetchRdapDomain: vi.fn(async () => ({
url: "https://rdap.example/domain/example.com",
json: { ldhName: "example.com" },
json: { ldhName: "example.com", links: [] },
})),
}));
vi.mock("./rdap/merge.js", () => ({
fetchAndMergeRdapRelated: vi.fn(async (_domain: string, json: unknown) => ({
merged: json,
serversTried: [],
})),
}));
@@ -19,6 +26,19 @@ vi.mock("./whois/client.js", () => ({
})),
}));
vi.mock("./whois/referral.js", async () => {
const client = await import("./whois/client.js");
return {
followWhoisReferrals: vi.fn(
async (
server: string,
domain: string,
opts?: import("./types").LookupOptions,
) => client.whoisQuery(server, domain, opts),
),
};
});
vi.mock("./whois/discovery.js", async () => {
const actual = await vi.importActual("./whois/discovery.js");
return {
@@ -43,6 +63,7 @@ import * as rdapClient from "./rdap/client";
import type { WhoisQueryResult } from "./whois/client";
import * as whoisClient from "./whois/client";
import * as discovery from "./whois/discovery";
import * as whoisReferral from "./whois/referral";
// 1) Orchestration tests (RDAP path, fallback, whoisOnly)
describe("lookupDomain orchestration", () => {
@@ -93,20 +114,10 @@ describe("WHOIS referral & includeRaw", () => {
});
it("does not follow referral when followWhoisReferral is false", async () => {
vi.mocked(whoisClient.whoisQuery).mockImplementation(
async (server: string): Promise<WhoisQueryResult> => {
if (server === "whois.verisign-grs.com") {
return {
text: "Registrar WHOIS Server: whois.registrar.test\nDomain Name: EXAMPLE.COM",
serverQueried: "whois.verisign-grs.com",
};
}
return {
text: "Domain Name: EXAMPLE.COM\nRegistrar: Registrar LLC",
serverQueried: "whois.registrar.test",
};
},
);
vi.mocked(whoisReferral.followWhoisReferrals).mockResolvedValue({
text: "Registrar WHOIS Server: whois.registrar.test\nDomain Name: EXAMPLE.COM",
serverQueried: "whois.verisign-grs.com",
});
const res = await lookupDomain("example.com", {
timeoutMs: 200,
@@ -114,26 +125,17 @@ describe("WHOIS referral & includeRaw", () => {
followWhoisReferral: false,
});
expect(res.ok, res.error).toBe(true);
expect(vi.mocked(whoisClient.whoisQuery)).toHaveBeenCalledTimes(1);
expect(vi.mocked(whoisClient.whoisQuery).mock.calls[0][0]).toBe(
"whois.verisign-grs.com",
expect(vi.mocked(whoisReferral.followWhoisReferrals)).toHaveBeenCalledTimes(
1,
);
});
it("includes rawWhois when includeRaw is true", async () => {
vi.mocked(whoisClient.whoisQuery).mockImplementation(
async (server: string): Promise<WhoisQueryResult> => {
if (server === "whois.verisign-grs.com") {
return {
text: "Registrar WHOIS Server: whois.registrar.test\nDomain Name: EXAMPLE.COM",
serverQueried: "whois.verisign-grs.com",
};
}
return {
text: "Domain Name: EXAMPLE.COM\nRegistrar: Registrar LLC",
serverQueried: "whois.registrar.test",
};
},
vi.mocked(whoisReferral.followWhoisReferrals).mockImplementation(
async (_server: string, _domain: string): Promise<WhoisQueryResult> => ({
text: "Domain Name: EXAMPLE.COM\nRegistrar: Registrar LLC",
serverQueried: "whois.registrar.test",
}),
);
const res = await lookupDomain("example.com", {
@@ -159,6 +161,14 @@ describe("WHOIS multi-label public suffix fallback", () => {
publicSuffix: "uk.com",
tld: "com",
});
// Ensure referral helper defers to whoisQuery for initial TLD query in this suite
vi.mocked(whoisReferral.followWhoisReferrals).mockImplementation(
async (
server: string,
d: string,
o?: import("./types").LookupOptions,
): Promise<WhoisQueryResult> => whoisClient.whoisQuery(server, d, o),
);
});
it("tries exception server for multi-label public suffix when TLD says no match", async () => {

42
src/index.ts
View File

@@ -2,16 +2,17 @@ import { toISO } from "./lib/dates";
import { getDomainParts, isLikelyDomain } from "./lib/domain";
import { getRdapBaseUrlsForTld } from "./rdap/bootstrap";
import { fetchRdapDomain } from "./rdap/client";
import { fetchAndMergeRdapRelated } from "./rdap/merge";
import { normalizeRdap } from "./rdap/normalize";
import type { DomainRecord, LookupOptions, LookupResult } from "./types";
import { whoisQuery } from "./whois/client";
import {
extractWhoisReferral,
getIanaWhoisTextForTld,
ianaWhoisServerForTld,
parseIanaRegistrationInfoUrl,
} from "./whois/discovery";
import { normalizeWhois } from "./whois/normalize";
import { followWhoisReferrals } from "./whois/referral";
import { WHOIS_TLD_EXCEPTIONS } from "./whois/servers";
/**
@@ -39,11 +40,16 @@ export async function lookupDomain(
tried.push(base);
try {
const { json } = await fetchRdapDomain(domain, base, opts);
const rdapEnriched = await fetchAndMergeRdapRelated(
domain,
json,
opts,
);
const record: DomainRecord = normalizeRdap(
domain,
tld,
json,
tried,
rdapEnriched.merged,
[...tried, ...rdapEnriched.serversTried],
now,
!!opts?.includeRaw,
);
@@ -75,18 +81,8 @@ export async function lookupDomain(
error: `No WHOIS server discovered for TLD '${tld}'. This registry may not publish public WHOIS over port 43.${hint}`,
};
}
// Query the TLD server first; if it returns a referral, we follow it below.
let res = await whoisQuery(whoisServer, domain, opts);
if (opts?.followWhoisReferral !== false) {
const referral = extractWhoisReferral(res.text);
if (referral && referral.toLowerCase() !== whoisServer.toLowerCase()) {
try {
res = await whoisQuery(referral, domain, opts);
} catch {
// keep original
}
}
}
// Query the TLD server first; optionally follow registrar referrals (multi-hop)
const res = await followWhoisReferrals(whoisServer, domain, opts);
// If TLD registry returns no match and there was no referral, try multi-label public suffix candidates
if (
@@ -102,10 +98,18 @@ export async function lookupDomain(
for (const server of candidates) {
try {
const alt = await whoisQuery(server, domain, opts);
if (alt.text && !/error/i.test(alt.text)) {
res = alt;
break;
}
if (alt.text && !/error/i.test(alt.text))
return {
ok: true,
record: normalizeWhois(
domain,
tld,
alt.text,
alt.serverQueried,
now,
!!opts?.includeRaw,
),
};
} catch {
// try next
}

32
src/rdap/links.ts Normal file
View File

@@ -0,0 +1,32 @@
import type { LookupOptions } from "../types";
type RdapLink = {
value?: string;
rel?: string;
href?: string;
type?: string;
};
/** Extract candidate RDAP link URLs from an RDAP document. */
export function extractRdapRelatedLinks(
doc: unknown,
opts?: Pick<LookupOptions, "rdapLinkRels">,
): string[] {
const rels = (
opts?.rdapLinkRels?.length
? opts.rdapLinkRels
: ["related", "entity", "registrar", "alternate"]
).map((r) => r.toLowerCase());
const d = (doc ?? {}) as Record<string, unknown> & { links?: RdapLink[] };
const arr = Array.isArray(d?.links) ? (d.links as RdapLink[]) : [];
const out: string[] = [];
for (const link of arr) {
const rel = String(link.rel || "").toLowerCase();
const type = String(link.type || "").toLowerCase();
if (!rels.includes(rel)) continue;
if (type && !/application\/rdap\+json/i.test(type)) continue;
const url = link.href || link.value;
if (url && /^https?:\/\//i.test(url)) out.push(url);
}
return Array.from(new Set(out));
}

145
src/rdap/merge.ts Normal file
View File

@@ -0,0 +1,145 @@
import { withTimeout } from "../lib/async";
import { DEFAULT_TIMEOUT_MS } from "../lib/constants";
import type { LookupOptions } from "../types";
import { extractRdapRelatedLinks } from "./links";
type Json = Record<string, unknown>;
/** Merge RDAP documents with a conservative, additive strategy. */
export function mergeRdapDocs(baseDoc: unknown, others: unknown[]): unknown {
const merged: Json = { ...(baseDoc as Json) };
for (const doc of others) {
const cur = (doc ?? {}) as Json;
// status: array of strings
merged.status = uniqStrings([
...toStringArray(merged.status),
...toStringArray(cur.status),
]);
// events: array of objects; dedupe by eventAction + eventDate
merged.events = uniqBy(
[...toArray<Json>(merged.events), ...toArray<Json>(cur.events)],
(e) =>
`${String(e?.eventAction ?? "").toLowerCase()}|${String(e?.eventDate ?? "")}`,
);
// nameservers: array of objects; dedupe by ldhName/unicodeName
merged.nameservers = uniqBy(
[...toArray<Json>(merged.nameservers), ...toArray<Json>(cur.nameservers)],
(n) => `${String(n?.ldhName ?? n?.unicodeName ?? "").toLowerCase()}`,
);
// entities: array; dedupe by handle if present, else by roles+vcard hash
merged.entities = uniqBy(
[...toArray<Json>(merged.entities), ...toArray<Json>(cur.entities)],
(e) =>
`${String(e?.handle ?? "").toLowerCase()}|${String(
JSON.stringify(e?.roles || []),
).toLowerCase()}|${String(JSON.stringify(e?.vcardArray || [])).toLowerCase()}`,
);
// secureDNS: prefer existing; fill if missing
if (merged.secureDNS == null && cur.secureDNS != null)
merged.secureDNS = cur.secureDNS;
// port43 (authoritative WHOIS): prefer existing; fill if missing
if (merged.port43 == null && cur.port43 != null) merged.port43 = cur.port43;
// remarks: concat simple strings if present
const mergedRemarks = (merged as { remarks?: Json[] }).remarks;
const curRemarks = (cur as { remarks?: Json[] }).remarks;
if (Array.isArray(mergedRemarks) || Array.isArray(curRemarks)) {
const a = toArray<Json>(mergedRemarks);
const b = toArray<Json>(curRemarks);
(merged as { remarks?: Json[] }).remarks = [...a, ...b];
}
}
return merged;
}
/** Fetch and merge RDAP related documents up to a hop limit. */
export async function fetchAndMergeRdapRelated(
domain: string,
baseDoc: unknown,
opts?: LookupOptions,
): Promise<{ merged: unknown; serversTried: string[] }> {
const tried: string[] = [];
if (opts?.rdapFollowLinks === false)
return { merged: baseDoc, serversTried: tried };
const maxHops = Math.max(0, opts?.maxRdapLinkHops ?? 2);
if (maxHops === 0) return { merged: baseDoc, serversTried: tried };
const visited = new Set<string>();
let current = baseDoc;
let hops = 0;
// BFS: collect links from the latest merged doc only to keep it simple and bounded
while (hops < maxHops) {
const links = extractRdapRelatedLinks(current, {
rdapLinkRels: opts?.rdapLinkRels,
});
const nextBatch = links.filter((u) => !visited.has(u));
if (nextBatch.length === 0) break;
const fetchedDocs: unknown[] = [];
for (const url of nextBatch) {
visited.add(url);
try {
const { json } = await fetchRdapUrl(url, opts);
tried.push(url);
// only accept docs that appear related to the same domain when possible
// if ldhName/unicodeName present, they should match the queried domain (case-insensitive)
const ldh = String((json as Json)?.ldhName ?? "").toLowerCase();
const uni = String((json as Json)?.unicodeName ?? "").toLowerCase();
if (ldh && !sameDomain(ldh, domain)) continue;
if (uni && !sameDomain(uni, domain)) continue;
fetchedDocs.push(json);
} catch {
// ignore failures and continue
}
}
if (fetchedDocs.length === 0) break;
current = mergeRdapDocs(current, fetchedDocs);
hops += 1;
}
return { merged: current, serversTried: tried };
}
async function fetchRdapUrl(
url: string,
options?: LookupOptions,
): Promise<{ url: string; json: unknown }> {
const res = await withTimeout(
fetch(url, {
method: "GET",
headers: { accept: "application/rdap+json, application/json" },
signal: options?.signal,
}),
options?.timeoutMs ?? DEFAULT_TIMEOUT_MS,
"RDAP link fetch timeout",
);
if (!res.ok) {
const bodyText = await res.text();
throw new Error(`RDAP ${res.status}: ${bodyText.slice(0, 500)}`);
}
const json = await res.json();
// Optionally parse Link header for future iterations; the main loop inspects body.links
return { url, json };
}
function toArray<T>(val: unknown): T[] {
return Array.isArray(val) ? (val as T[]) : [];
}
function toStringArray(val: unknown): string[] {
return Array.isArray(val) ? (val as unknown[]).map((v) => String(v)) : [];
}
function uniqStrings(arr: string[]): string[] {
return Array.from(new Set(arr));
}
function uniqBy<T>(arr: T[], key: (t: T) => string): T[] {
const seen = new Set<string>();
const out: T[] = [];
for (const item of arr) {
const k = key(item);
if (seen.has(k)) continue;
seen.add(k);
out.push(item);
}
return out;
}
function sameDomain(a: string, b: string): boolean {
return a.toLowerCase() === b.toLowerCase();
}

4
src/types.ts
View File

@@ -84,6 +84,10 @@ export interface LookupOptions {
rdapOnly?: boolean; // don't fall back to WHOIS
whoisOnly?: boolean; // don't attempt RDAP
followWhoisReferral?: boolean; // follow referral server (default true)
maxWhoisReferralHops?: number; // maximum registrar WHOIS referral hops (default 2)
rdapFollowLinks?: boolean; // follow RDAP related/entity links (default true)
maxRdapLinkHops?: number; // maximum RDAP related link fetches (default 2)
rdapLinkRels?: string[]; // RDAP link rels to consider (default ["related","entity","registrar","alternate"])
customBootstrapUrl?: string; // override IANA bootstrap
// WHOIS discovery and query tuning
whoisHints?: Record<string, string>; // override/add authoritative WHOIS per TLD

43
src/whois/referral.ts Normal file
View File

@@ -0,0 +1,43 @@
import type { LookupOptions } from "../types";
import type { WhoisQueryResult } from "./client";
import { whoisQuery } from "./client";
import { extractWhoisReferral } from "./discovery";
/**
* Follow registrar WHOIS referrals up to a configured hop limit.
* Returns the last successful WHOIS response (best-effort; keeps original on failures).
*/
export async function followWhoisReferrals(
initialServer: string,
domain: string,
opts?: LookupOptions,
): Promise<WhoisQueryResult> {
const maxHops = Math.max(0, opts?.maxWhoisReferralHops ?? 2);
// First query against the provided server
let current = await whoisQuery(initialServer, domain, opts);
if (opts?.followWhoisReferral === false || maxHops === 0) return current;
const visited = new Set<string>([normalize(current.serverQueried)]);
let hops = 0;
// Iterate while we see a new referral and are under hop limit
while (hops < maxHops) {
const next = extractWhoisReferral(current.text);
if (!next) break;
const normalized = normalize(next);
if (visited.has(normalized)) break; // cycle protection / same as current
visited.add(normalized);
try {
const res = await whoisQuery(next, domain, opts);
current = res; // adopt the newer, more specific result
} catch {
// If referral server fails, stop following and keep the last good response
break;
}
hops += 1;
}
return current;
}
function normalize(server: string): string {
return server.replace(/^whois:\/\//i, "").toLowerCase();
}