From 9d87ef8ecb1e5f40a9016f9799d88789fd3699c8 Mon Sep 17 00:00:00 2001 From: Jake Jarvis Date: Mon, 27 Oct 2025 14:42:43 -0400 Subject: [PATCH] Add to list of parser strings to improve WHOIS data normalization --- src/lib/privacy.ts | 29 +++++- src/whois/normalize.ts | 215 +++++++++++++++++++++++++++++++---------- 2 files changed, 192 insertions(+), 52 deletions(-) diff --git a/src/lib/privacy.ts b/src/lib/privacy.ts index 0c08662..1cd3879 100644 --- a/src/lib/privacy.ts +++ b/src/lib/privacy.ts @@ -6,9 +6,36 @@ export const PRIVACY_NAME_KEYWORDS = [ "not disclosed", "protected", "protection", + "privado", // Spanish + "datos privados", // Spanish + "data protected", + "data redacted", + "gdpr redacted", + "gdpr masked", + "non-public data", + "statutory masking", + "redacted.forprivacy", + "registration private", + "hidden upon user request", + "not available from registry", +]; + +// Completely unusable/empty values that should be filtered +export const NO_DATA_VALUES = [ + "-", + ".", + "n/a", + "na", + "no data", + "not available", + "not applicable", + "none", ]; export function isPrivacyName(value: string): boolean { - const v = value.toLowerCase(); + const v = value.toLowerCase().trim(); + // Check for complete no-data values + if (NO_DATA_VALUES.includes(v)) return true; + // Check for privacy keywords return PRIVACY_NAME_KEYWORDS.some((k) => v.includes(k)); } diff --git a/src/whois/normalize.ts b/src/whois/normalize.ts index 6358c5f..3c433fa 100644 --- a/src/whois/normalize.ts +++ b/src/whois/normalize.ts @@ -19,11 +19,13 @@ const WHOIS_AVAILABLE_PATTERNS: RegExp[] = [ /\bdomain status[:\s]+available\b/i, /\bobject does not exist\b/i, /\bthe queried object does not exist\b/i, + /\bqueried object does not exist\b/i, + /\breturned 0 objects\b/i, // Common variants across ccTLDs/registrars /\bstatus:\s*free\b/i, /\bstatus:\s*available\b/i, /\bno object found\b/i, - /\bnicht gefunden\b/i, + /\bnicht gefunden\b/i, // German: "not found" /\bpending release\b/i, // often signals not registered/being deleted ]; @@ -52,54 +54,75 @@ export function normalizeWhois( const creationDate = anyValue(map, [ "creation date", "created on", + "created", "registered on", + "registered", + "registration date", "domain registration date", "domain create date", - "created", - "registered", "domain name commencement date", "registration time", // .cn "domain record activated", // .edu + "domain registered", + "registered date", // .co.jp "assigned", // .il ]); const updatedDate = anyValue(map, [ "updated date", + "updated", "last updated", - "last update", + "last updated on", // .mx + "last update", // .co.jp + "last-update", // .fr "last modified", "modified", + "changed", + "modification date", "domain record last updated", // .edu ]); const expirationDate = anyValue(map, [ "registry expiry date", "registry expiration date", - "expiry date", - "expiration date", - "expire date", // .it - "expiration time", // .cn "registrar registration expiration date", "registrar registration expiry date", "registrar expiration date", "registrar expiry date", - "domain expires", // .edu - "paid-till", + "expiry date", + "expiration date", + "expiry", + "expire date", // .it + "expire", + "expired", // .ly "expires on", "expires", - "expire", // .ee - "renewal date", + "expiration time", // .cn + "domain expires", // .edu + "paid-till", + "renewal date", // .pl "validity", // .il + "record will expire on", ]); // Registrar info (thin registries like .com/.net require referral follow for full data) const registrar: RegistrarInfo | undefined = (() => { const name = anyValue(map, [ "registrar", - "sponsoring registrar", "registrar name", + "registrar organization", + "registrar organization name", // .tr + "sponsoring registrar", + "organisation", + "record maintained by", + ]); + const ianaId = anyValue(map, [ + "registrar iana id", + "sponsoring registrar iana id", + "iana id", ]); - const ianaId = anyValue(map, ["registrar iana id", "iana id"]); const url = anyValue(map, [ "registrar url", + "registrar website", + "registrar web", // .it "url of the registrar", "referrer", ]); @@ -123,7 +146,14 @@ export function normalizeWhois( })(); // Statuses: multiple entries are expected; keep raw - const statusLines = map["domain status"] || map.status || []; + const statusLines = + map["domain status"] || + map.status || + map.flags || + map.state || // .ru + map["registration status"] || + map.eppstatus || // .fr + []; const statuses = statusLines.length ? statusLines.map((line) => ({ status: line.split(/\s+/)[0], raw: line })) : undefined; @@ -134,6 +164,17 @@ export function normalizeWhois( ...(map.nameserver || []), ...(map["name servers"] || []), ...(map.nserver || []), + ...(map["name server information"] || []), + ...(map.dns || []), + ...(map.hostname || []), + ...(map["domain nameservers"] || []), + ...(map["domain servers in listed order"] || []), // .ly + ...(map["domain servers"] || []), // .tr + ...(map["name servers dns"] || []), // .mx + ...(map["ns 1"] || []), + ...(map["ns 2"] || []), + ...(map["ns 3"] || []), + ...(map["ns 4"] || []), ]; const nameservers: Nameserver[] | undefined = nsLines.length ? (uniq( @@ -225,45 +266,117 @@ function anyValue( } function collectContacts(map: Record): Contact[] | undefined { - const roles: Array<{ role: Contact["type"]; prefix: string }> = [ - { role: "registrant", prefix: "registrant" }, - { role: "admin", prefix: "admin" }, - { role: "tech", prefix: "tech" }, - { role: "billing", prefix: "billing" }, - { role: "abuse", prefix: "abuse" }, + const roles: Array<{ + role: Contact["type"]; + prefixes: string[]; + }> = [ + { + role: "registrant", + prefixes: ["registrant", "owner", "holder"], + }, + { + role: "admin", + prefixes: ["admin", "administrative"], + }, + { + role: "tech", + prefixes: ["tech", "technical"], + }, + { + role: "billing", + prefixes: ["billing"], + }, + { + role: "abuse", + prefixes: ["abuse"], + }, ]; const contacts: Contact[] = []; for (const r of roles) { - const name = anyValue(map, [ - `${r.prefix} name`, - `${r.prefix} contact name`, - `${r.prefix}`, - ]); - const org = anyValue(map, [`${r.prefix} organization`, `${r.prefix} org`]); - const email = anyValue(map, [ - `${r.prefix} email`, - `${r.prefix} contact email`, - `${r.prefix} e-mail`, - ]); - const phone = anyValue(map, [ - `${r.prefix} phone`, - `${r.prefix} contact phone`, - `${r.prefix} telephone`, - ]); - const fax = anyValue(map, [`${r.prefix} fax`, `${r.prefix} facsimile`]); - const street = multi(map, [`${r.prefix} street`, `${r.prefix} address`]); - const city = anyValue(map, [`${r.prefix} city`]); - const state = anyValue(map, [ - `${r.prefix} state`, - `${r.prefix} province`, - `${r.prefix} state/province`, - ]); - const postalCode = anyValue(map, [ - `${r.prefix} postal code`, - `${r.prefix} postcode`, - `${r.prefix} zip`, - ]); - const country = anyValue(map, [`${r.prefix} country`]); + const nameKeys: string[] = []; + const orgKeys: string[] = []; + const emailKeys: string[] = []; + const phoneKeys: string[] = []; + const faxKeys: string[] = []; + const streetKeys: string[] = []; + const cityKeys: string[] = []; + const stateKeys: string[] = []; + const postalCodeKeys: string[] = []; + const countryKeys: string[] = []; + + for (const prefix of r.prefixes) { + nameKeys.push(`${prefix} name`, `${prefix} contact name`, `${prefix}`); + if (prefix === "registrant") { + nameKeys.push("registrant person"); // .ua + } + if (prefix === "owner") { + nameKeys.push("owner name"); // .tm + } + + orgKeys.push( + `${prefix} organization`, + `${prefix} organisation`, + `${prefix} org`, + ); + if (prefix === "registrant") { + orgKeys.push("trading as"); // .uk, .co.uk + orgKeys.push("org"); // .ru + } + if (prefix === "owner") { + orgKeys.push("owner orgname"); // .tm + } + + emailKeys.push( + `${prefix} email`, + `${prefix} contact email`, + `${prefix} e-mail`, + ); + + phoneKeys.push( + `${prefix} phone`, + `${prefix} contact phone`, + `${prefix} telephone`, + ); + + faxKeys.push(`${prefix} fax`, `${prefix} facsimile`); + + streetKeys.push( + `${prefix} street`, + `${prefix} address`, + `${prefix}'s address`, + ); + if (prefix === "owner") { + streetKeys.push("owner addr"); // .tm + } + + cityKeys.push(`${prefix} city`); + + stateKeys.push( + `${prefix} state`, + `${prefix} province`, + `${prefix} state/province`, + ); + + postalCodeKeys.push( + `${prefix} postal code`, + `${prefix} postcode`, + `${prefix} zip`, + ); + + countryKeys.push(`${prefix} country`); + } + + const name = anyValue(map, nameKeys); + const org = anyValue(map, orgKeys); + const email = anyValue(map, emailKeys); + const phone = anyValue(map, phoneKeys); + const fax = anyValue(map, faxKeys); + const street = multi(map, streetKeys); + const city = anyValue(map, cityKeys); + const state = anyValue(map, stateKeys); + const postalCode = anyValue(map, postalCodeKeys); + const country = anyValue(map, countryKeys); + if (name || org || email || phone || street?.length) { contacts.push({ type: r.role,