Enhance RDAP and WHOIS functionality with referral handling

Added support for following registrar WHOIS referrals with configurable hop limits. Introduced new options in LookupOptions for maximum referral hops and RDAP link handling. Updated README to reflect these changes and improved the lookupDomain function to utilize the new referral logic. Added utility functions for merging RDAP documents and extracting related links.

src/index.test.ts

@@ -8,7 +8,14 @@ vi.mock("./rdap/bootstrap.js", () => ({
 vi.mock("./rdap/client.js", () => ({
   fetchRdapDomain: vi.fn(async () => ({
     url: "https://rdap.example/domain/example.com",
-    json: { ldhName: "example.com" },
+    json: { ldhName: "example.com", links: [] },
+  })),
+}));
+
+vi.mock("./rdap/merge.js", () => ({
+  fetchAndMergeRdapRelated: vi.fn(async (_domain: string, json: unknown) => ({
+    merged: json,
+    serversTried: [],
   })),
 }));
 
@@ -19,6 +26,19 @@ vi.mock("./whois/client.js", () => ({
   })),
 }));
 
+vi.mock("./whois/referral.js", async () => {
+  const client = await import("./whois/client.js");
+  return {
+    followWhoisReferrals: vi.fn(
+      async (
+        server: string,
+        domain: string,
+        opts?: import("./types").LookupOptions,
+      ) => client.whoisQuery(server, domain, opts),
+    ),
+  };
+});
+
 vi.mock("./whois/discovery.js", async () => {
   const actual = await vi.importActual("./whois/discovery.js");
   return {
@@ -43,6 +63,7 @@ import * as rdapClient from "./rdap/client";
 import type { WhoisQueryResult } from "./whois/client";
 import * as whoisClient from "./whois/client";
 import * as discovery from "./whois/discovery";
+import * as whoisReferral from "./whois/referral";
 
 // 1) Orchestration tests (RDAP path, fallback, whoisOnly)
 describe("lookupDomain orchestration", () => {
@@ -93,20 +114,10 @@ describe("WHOIS referral & includeRaw", () => {
   });
 
   it("does not follow referral when followWhoisReferral is false", async () => {
-    vi.mocked(whoisClient.whoisQuery).mockImplementation(
-      async (server: string): Promise<WhoisQueryResult> => {
-        if (server === "whois.verisign-grs.com") {
-          return {
-            text: "Registrar WHOIS Server: whois.registrar.test\nDomain Name: EXAMPLE.COM",
-            serverQueried: "whois.verisign-grs.com",
-          };
-        }
-        return {
-          text: "Domain Name: EXAMPLE.COM\nRegistrar: Registrar LLC",
-          serverQueried: "whois.registrar.test",
-        };
-      },
-    );
+    vi.mocked(whoisReferral.followWhoisReferrals).mockResolvedValue({
+      text: "Registrar WHOIS Server: whois.registrar.test\nDomain Name: EXAMPLE.COM",
+      serverQueried: "whois.verisign-grs.com",
+    });
 
     const res = await lookupDomain("example.com", {
       timeoutMs: 200,
@@ -114,26 +125,17 @@ describe("WHOIS referral & includeRaw", () => {
       followWhoisReferral: false,
     });
     expect(res.ok, res.error).toBe(true);
-    expect(vi.mocked(whoisClient.whoisQuery)).toHaveBeenCalledTimes(1);
-    expect(vi.mocked(whoisClient.whoisQuery).mock.calls[0][0]).toBe(
-      "whois.verisign-grs.com",
+    expect(vi.mocked(whoisReferral.followWhoisReferrals)).toHaveBeenCalledTimes(
+      1,
     );
   });
 
   it("includes rawWhois when includeRaw is true", async () => {
-    vi.mocked(whoisClient.whoisQuery).mockImplementation(
-      async (server: string): Promise<WhoisQueryResult> => {
-        if (server === "whois.verisign-grs.com") {
-          return {
-            text: "Registrar WHOIS Server: whois.registrar.test\nDomain Name: EXAMPLE.COM",
-            serverQueried: "whois.verisign-grs.com",
-          };
-        }
-        return {
-          text: "Domain Name: EXAMPLE.COM\nRegistrar: Registrar LLC",
-          serverQueried: "whois.registrar.test",
-        };
-      },
+    vi.mocked(whoisReferral.followWhoisReferrals).mockImplementation(
+      async (_server: string, _domain: string): Promise<WhoisQueryResult> => ({
+        text: "Domain Name: EXAMPLE.COM\nRegistrar: Registrar LLC",
+        serverQueried: "whois.registrar.test",
+      }),
     );
 
     const res = await lookupDomain("example.com", {
@@ -159,6 +161,14 @@ describe("WHOIS multi-label public suffix fallback", () => {
       publicSuffix: "uk.com",
       tld: "com",
     });
+    // Ensure referral helper defers to whoisQuery for initial TLD query in this suite
+    vi.mocked(whoisReferral.followWhoisReferrals).mockImplementation(
+      async (
+        server: string,
+        d: string,
+        o?: import("./types").LookupOptions,
+      ): Promise<WhoisQueryResult> => whoisClient.whoisQuery(server, d, o),
+    );
   });
 
   it("tries exception server for multi-label public suffix when TLD says no match", async () => {

src/index.ts

@@ -2,16 +2,17 @@ import { toISO } from "./lib/dates";
 import { getDomainParts, isLikelyDomain } from "./lib/domain";
 import { getRdapBaseUrlsForTld } from "./rdap/bootstrap";
 import { fetchRdapDomain } from "./rdap/client";
+import { fetchAndMergeRdapRelated } from "./rdap/merge";
 import { normalizeRdap } from "./rdap/normalize";
 import type { DomainRecord, LookupOptions, LookupResult } from "./types";
 import { whoisQuery } from "./whois/client";
 import {
-  extractWhoisReferral,
   getIanaWhoisTextForTld,
   ianaWhoisServerForTld,
   parseIanaRegistrationInfoUrl,
 } from "./whois/discovery";
 import { normalizeWhois } from "./whois/normalize";
+import { followWhoisReferrals } from "./whois/referral";
 import { WHOIS_TLD_EXCEPTIONS } from "./whois/servers";
 
 /**
@@ -39,11 +40,16 @@ export async function lookupDomain(
         tried.push(base);
         try {
           const { json } = await fetchRdapDomain(domain, base, opts);
+          const rdapEnriched = await fetchAndMergeRdapRelated(
+            domain,
+            json,
+            opts,
+          );
           const record: DomainRecord = normalizeRdap(
             domain,
             tld,
-            json,
-            tried,
+            rdapEnriched.merged,
+            [...tried, ...rdapEnriched.serversTried],
             now,
             !!opts?.includeRaw,
           );
@@ -75,18 +81,8 @@ export async function lookupDomain(
         error: `No WHOIS server discovered for TLD '${tld}'. This registry may not publish public WHOIS over port 43.${hint}`,
       };
     }
-    // Query the TLD server first; if it returns a referral, we follow it below.
-    let res = await whoisQuery(whoisServer, domain, opts);
-    if (opts?.followWhoisReferral !== false) {
-      const referral = extractWhoisReferral(res.text);
-      if (referral && referral.toLowerCase() !== whoisServer.toLowerCase()) {
-        try {
-          res = await whoisQuery(referral, domain, opts);
-        } catch {
-          // keep original
-        }
-      }
-    }
+    // Query the TLD server first; optionally follow registrar referrals (multi-hop)
+    const res = await followWhoisReferrals(whoisServer, domain, opts);
 
     // If TLD registry returns no match and there was no referral, try multi-label public suffix candidates
     if (
@@ -102,10 +98,18 @@ export async function lookupDomain(
       for (const server of candidates) {
         try {
           const alt = await whoisQuery(server, domain, opts);
-          if (alt.text && !/error/i.test(alt.text)) {
-            res = alt;
-            break;
-          }
+          if (alt.text && !/error/i.test(alt.text))
+            return {
+              ok: true,
+              record: normalizeWhois(
+                domain,
+                tld,
+                alt.text,
+                alt.serverQueried,
+                now,
+                !!opts?.includeRaw,
+              ),
+            };
         } catch {
           // try next
         }

src/rdap/links.ts

@@ -0,0 +1,32 @@
+import type { LookupOptions } from "../types";
+
+type RdapLink = {
+  value?: string;
+  rel?: string;
+  href?: string;
+  type?: string;
+};
+
+/** Extract candidate RDAP link URLs from an RDAP document. */
+export function extractRdapRelatedLinks(
+  doc: unknown,
+  opts?: Pick<LookupOptions, "rdapLinkRels">,
+): string[] {
+  const rels = (
+    opts?.rdapLinkRels?.length
+      ? opts.rdapLinkRels
+      : ["related", "entity", "registrar", "alternate"]
+  ).map((r) => r.toLowerCase());
+  const d = (doc ?? {}) as Record<string, unknown> & { links?: RdapLink[] };
+  const arr = Array.isArray(d?.links) ? (d.links as RdapLink[]) : [];
+  const out: string[] = [];
+  for (const link of arr) {
+    const rel = String(link.rel || "").toLowerCase();
+    const type = String(link.type || "").toLowerCase();
+    if (!rels.includes(rel)) continue;
+    if (type && !/application\/rdap\+json/i.test(type)) continue;
+    const url = link.href || link.value;
+    if (url && /^https?:\/\//i.test(url)) out.push(url);
+  }
+  return Array.from(new Set(out));
+}

src/rdap/merge.ts

@@ -0,0 +1,145 @@
+import { withTimeout } from "../lib/async";
+import { DEFAULT_TIMEOUT_MS } from "../lib/constants";
+import type { LookupOptions } from "../types";
+import { extractRdapRelatedLinks } from "./links";
+
+type Json = Record<string, unknown>;
+
+/** Merge RDAP documents with a conservative, additive strategy. */
+export function mergeRdapDocs(baseDoc: unknown, others: unknown[]): unknown {
+  const merged: Json = { ...(baseDoc as Json) };
+  for (const doc of others) {
+    const cur = (doc ?? {}) as Json;
+    // status: array of strings
+    merged.status = uniqStrings([
+      ...toStringArray(merged.status),
+      ...toStringArray(cur.status),
+    ]);
+    // events: array of objects; dedupe by eventAction + eventDate
+    merged.events = uniqBy(
+      [...toArray<Json>(merged.events), ...toArray<Json>(cur.events)],
+      (e) =>
+        `${String(e?.eventAction ?? "").toLowerCase()}|${String(e?.eventDate ?? "")}`,
+    );
+    // nameservers: array of objects; dedupe by ldhName/unicodeName
+    merged.nameservers = uniqBy(
+      [...toArray<Json>(merged.nameservers), ...toArray<Json>(cur.nameservers)],
+      (n) => `${String(n?.ldhName ?? n?.unicodeName ?? "").toLowerCase()}`,
+    );
+    // entities: array; dedupe by handle if present, else by roles+vcard hash
+    merged.entities = uniqBy(
+      [...toArray<Json>(merged.entities), ...toArray<Json>(cur.entities)],
+      (e) =>
+        `${String(e?.handle ?? "").toLowerCase()}|${String(
+          JSON.stringify(e?.roles || []),
+        ).toLowerCase()}|${String(JSON.stringify(e?.vcardArray || [])).toLowerCase()}`,
+    );
+    // secureDNS: prefer existing; fill if missing
+    if (merged.secureDNS == null && cur.secureDNS != null)
+      merged.secureDNS = cur.secureDNS;
+    // port43 (authoritative WHOIS): prefer existing; fill if missing
+    if (merged.port43 == null && cur.port43 != null) merged.port43 = cur.port43;
+    // remarks: concat simple strings if present
+    const mergedRemarks = (merged as { remarks?: Json[] }).remarks;
+    const curRemarks = (cur as { remarks?: Json[] }).remarks;
+    if (Array.isArray(mergedRemarks) || Array.isArray(curRemarks)) {
+      const a = toArray<Json>(mergedRemarks);
+      const b = toArray<Json>(curRemarks);
+      (merged as { remarks?: Json[] }).remarks = [...a, ...b];
+    }
+  }
+  return merged;
+}
+
+/** Fetch and merge RDAP related documents up to a hop limit. */
+export async function fetchAndMergeRdapRelated(
+  domain: string,
+  baseDoc: unknown,
+  opts?: LookupOptions,
+): Promise<{ merged: unknown; serversTried: string[] }> {
+  const tried: string[] = [];
+  if (opts?.rdapFollowLinks === false)
+    return { merged: baseDoc, serversTried: tried };
+  const maxHops = Math.max(0, opts?.maxRdapLinkHops ?? 2);
+  if (maxHops === 0) return { merged: baseDoc, serversTried: tried };
+
+  const visited = new Set<string>();
+  let current = baseDoc;
+  let hops = 0;
+
+  // BFS: collect links from the latest merged doc only to keep it simple and bounded
+  while (hops < maxHops) {
+    const links = extractRdapRelatedLinks(current, {
+      rdapLinkRels: opts?.rdapLinkRels,
+    });
+    const nextBatch = links.filter((u) => !visited.has(u));
+    if (nextBatch.length === 0) break;
+    const fetchedDocs: unknown[] = [];
+    for (const url of nextBatch) {
+      visited.add(url);
+      try {
+        const { json } = await fetchRdapUrl(url, opts);
+        tried.push(url);
+        // only accept docs that appear related to the same domain when possible
+        // if ldhName/unicodeName present, they should match the queried domain (case-insensitive)
+        const ldh = String((json as Json)?.ldhName ?? "").toLowerCase();
+        const uni = String((json as Json)?.unicodeName ?? "").toLowerCase();
+        if (ldh && !sameDomain(ldh, domain)) continue;
+        if (uni && !sameDomain(uni, domain)) continue;
+        fetchedDocs.push(json);
+      } catch {
+        // ignore failures and continue
+      }
+    }
+    if (fetchedDocs.length === 0) break;
+    current = mergeRdapDocs(current, fetchedDocs);
+    hops += 1;
+  }
+  return { merged: current, serversTried: tried };
+}
+
+async function fetchRdapUrl(
+  url: string,
+  options?: LookupOptions,
+): Promise<{ url: string; json: unknown }> {
+  const res = await withTimeout(
+    fetch(url, {
+      method: "GET",
+      headers: { accept: "application/rdap+json, application/json" },
+      signal: options?.signal,
+    }),
+    options?.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+    "RDAP link fetch timeout",
+  );
+  if (!res.ok) {
+    const bodyText = await res.text();
+    throw new Error(`RDAP ${res.status}: ${bodyText.slice(0, 500)}`);
+  }
+  const json = await res.json();
+  // Optionally parse Link header for future iterations; the main loop inspects body.links
+  return { url, json };
+}
+
+function toArray<T>(val: unknown): T[] {
+  return Array.isArray(val) ? (val as T[]) : [];
+}
+function toStringArray(val: unknown): string[] {
+  return Array.isArray(val) ? (val as unknown[]).map((v) => String(v)) : [];
+}
+function uniqStrings(arr: string[]): string[] {
+  return Array.from(new Set(arr));
+}
+function uniqBy<T>(arr: T[], key: (t: T) => string): T[] {
+  const seen = new Set<string>();
+  const out: T[] = [];
+  for (const item of arr) {
+    const k = key(item);
+    if (seen.has(k)) continue;
+    seen.add(k);
+    out.push(item);
+  }
+  return out;
+}
+function sameDomain(a: string, b: string): boolean {
+  return a.toLowerCase() === b.toLowerCase();
+}

src/types.ts

@@ -84,6 +84,10 @@ export interface LookupOptions {
   rdapOnly?: boolean; // don't fall back to WHOIS
   whoisOnly?: boolean; // don't attempt RDAP
   followWhoisReferral?: boolean; // follow referral server (default true)
+  maxWhoisReferralHops?: number; // maximum registrar WHOIS referral hops (default 2)
+  rdapFollowLinks?: boolean; // follow RDAP related/entity links (default true)
+  maxRdapLinkHops?: number; // maximum RDAP related link fetches (default 2)
+  rdapLinkRels?: string[]; // RDAP link rels to consider (default ["related","entity","registrar","alternate"])
   customBootstrapUrl?: string; // override IANA bootstrap
   // WHOIS discovery and query tuning
   whoisHints?: Record<string, string>; // override/add authoritative WHOIS per TLD

src/whois/referral.ts

@@ -0,0 +1,43 @@
+import type { LookupOptions } from "../types";
+import type { WhoisQueryResult } from "./client";
+import { whoisQuery } from "./client";
+import { extractWhoisReferral } from "./discovery";
+
+/**
+ * Follow registrar WHOIS referrals up to a configured hop limit.
+ * Returns the last successful WHOIS response (best-effort; keeps original on failures).
+ */
+export async function followWhoisReferrals(
+  initialServer: string,
+  domain: string,
+  opts?: LookupOptions,
+): Promise<WhoisQueryResult> {
+  const maxHops = Math.max(0, opts?.maxWhoisReferralHops ?? 2);
+  // First query against the provided server
+  let current = await whoisQuery(initialServer, domain, opts);
+  if (opts?.followWhoisReferral === false || maxHops === 0) return current;
+
+  const visited = new Set<string>([normalize(current.serverQueried)]);
+  let hops = 0;
+  // Iterate while we see a new referral and are under hop limit
+  while (hops < maxHops) {
+    const next = extractWhoisReferral(current.text);
+    if (!next) break;
+    const normalized = normalize(next);
+    if (visited.has(normalized)) break; // cycle protection / same as current
+    visited.add(normalized);
+    try {
+      const res = await whoisQuery(next, domain, opts);
+      current = res; // adopt the newer, more specific result
+    } catch {
+      // If referral server fails, stop following and keep the last good response
+      break;
+    }
+    hops += 1;
+  }
+  return current;
+}
+
+function normalize(server: string): string {
+  return server.replace(/^whois:\/\//i, "").toLowerCase();
+}