mirror of
https://github.com/jakejarvis/rdapper.git
synced 2026-09-11 04:15:32 -04:00
Add to list of parser strings to improve WHOIS data normalization
This commit is contained in:
+28
-1
@@ -6,9 +6,36 @@ export const PRIVACY_NAME_KEYWORDS = [
|
||||
"not disclosed",
|
||||
"protected",
|
||||
"protection",
|
||||
"privado", // Spanish
|
||||
"datos privados", // Spanish
|
||||
"data protected",
|
||||
"data redacted",
|
||||
"gdpr redacted",
|
||||
"gdpr masked",
|
||||
"non-public data",
|
||||
"statutory masking",
|
||||
"redacted.forprivacy",
|
||||
"registration private",
|
||||
"hidden upon user request",
|
||||
"not available from registry",
|
||||
];
|
||||
|
||||
// Completely unusable/empty values that should be filtered
|
||||
export const NO_DATA_VALUES = [
|
||||
"-",
|
||||
".",
|
||||
"n/a",
|
||||
"na",
|
||||
"no data",
|
||||
"not available",
|
||||
"not applicable",
|
||||
"none",
|
||||
];
|
||||
|
||||
export function isPrivacyName(value: string): boolean {
|
||||
const v = value.toLowerCase();
|
||||
const v = value.toLowerCase().trim();
|
||||
// Check for complete no-data values
|
||||
if (NO_DATA_VALUES.includes(v)) return true;
|
||||
// Check for privacy keywords
|
||||
return PRIVACY_NAME_KEYWORDS.some((k) => v.includes(k));
|
||||
}
|
||||
|
||||
+164
-51
@@ -19,11 +19,13 @@ const WHOIS_AVAILABLE_PATTERNS: RegExp[] = [
|
||||
/\bdomain status[:\s]+available\b/i,
|
||||
/\bobject does not exist\b/i,
|
||||
/\bthe queried object does not exist\b/i,
|
||||
/\bqueried object does not exist\b/i,
|
||||
/\breturned 0 objects\b/i,
|
||||
// Common variants across ccTLDs/registrars
|
||||
/\bstatus:\s*free\b/i,
|
||||
/\bstatus:\s*available\b/i,
|
||||
/\bno object found\b/i,
|
||||
/\bnicht gefunden\b/i,
|
||||
/\bnicht gefunden\b/i, // German: "not found"
|
||||
/\bpending release\b/i, // often signals not registered/being deleted
|
||||
];
|
||||
|
||||
@@ -52,54 +54,75 @@ export function normalizeWhois(
|
||||
const creationDate = anyValue(map, [
|
||||
"creation date",
|
||||
"created on",
|
||||
"created",
|
||||
"registered on",
|
||||
"registered",
|
||||
"registration date",
|
||||
"domain registration date",
|
||||
"domain create date",
|
||||
"created",
|
||||
"registered",
|
||||
"domain name commencement date",
|
||||
"registration time", // .cn
|
||||
"domain record activated", // .edu
|
||||
"domain registered",
|
||||
"registered date", // .co.jp
|
||||
"assigned", // .il
|
||||
]);
|
||||
const updatedDate = anyValue(map, [
|
||||
"updated date",
|
||||
"updated",
|
||||
"last updated",
|
||||
"last update",
|
||||
"last updated on", // .mx
|
||||
"last update", // .co.jp
|
||||
"last-update", // .fr
|
||||
"last modified",
|
||||
"modified",
|
||||
"changed",
|
||||
"modification date",
|
||||
"domain record last updated", // .edu
|
||||
]);
|
||||
const expirationDate = anyValue(map, [
|
||||
"registry expiry date",
|
||||
"registry expiration date",
|
||||
"expiry date",
|
||||
"expiration date",
|
||||
"expire date", // .it
|
||||
"expiration time", // .cn
|
||||
"registrar registration expiration date",
|
||||
"registrar registration expiry date",
|
||||
"registrar expiration date",
|
||||
"registrar expiry date",
|
||||
"domain expires", // .edu
|
||||
"paid-till",
|
||||
"expiry date",
|
||||
"expiration date",
|
||||
"expiry",
|
||||
"expire date", // .it
|
||||
"expire",
|
||||
"expired", // .ly
|
||||
"expires on",
|
||||
"expires",
|
||||
"expire", // .ee
|
||||
"renewal date",
|
||||
"expiration time", // .cn
|
||||
"domain expires", // .edu
|
||||
"paid-till",
|
||||
"renewal date", // .pl
|
||||
"validity", // .il
|
||||
"record will expire on",
|
||||
]);
|
||||
|
||||
// Registrar info (thin registries like .com/.net require referral follow for full data)
|
||||
const registrar: RegistrarInfo | undefined = (() => {
|
||||
const name = anyValue(map, [
|
||||
"registrar",
|
||||
"sponsoring registrar",
|
||||
"registrar name",
|
||||
"registrar organization",
|
||||
"registrar organization name", // .tr
|
||||
"sponsoring registrar",
|
||||
"organisation",
|
||||
"record maintained by",
|
||||
]);
|
||||
const ianaId = anyValue(map, [
|
||||
"registrar iana id",
|
||||
"sponsoring registrar iana id",
|
||||
"iana id",
|
||||
]);
|
||||
const ianaId = anyValue(map, ["registrar iana id", "iana id"]);
|
||||
const url = anyValue(map, [
|
||||
"registrar url",
|
||||
"registrar website",
|
||||
"registrar web", // .it
|
||||
"url of the registrar",
|
||||
"referrer",
|
||||
]);
|
||||
@@ -123,7 +146,14 @@ export function normalizeWhois(
|
||||
})();
|
||||
|
||||
// Statuses: multiple entries are expected; keep raw
|
||||
const statusLines = map["domain status"] || map.status || [];
|
||||
const statusLines =
|
||||
map["domain status"] ||
|
||||
map.status ||
|
||||
map.flags ||
|
||||
map.state || // .ru
|
||||
map["registration status"] ||
|
||||
map.eppstatus || // .fr
|
||||
[];
|
||||
const statuses = statusLines.length
|
||||
? statusLines.map((line) => ({ status: line.split(/\s+/)[0], raw: line }))
|
||||
: undefined;
|
||||
@@ -134,6 +164,17 @@ export function normalizeWhois(
|
||||
...(map.nameserver || []),
|
||||
...(map["name servers"] || []),
|
||||
...(map.nserver || []),
|
||||
...(map["name server information"] || []),
|
||||
...(map.dns || []),
|
||||
...(map.hostname || []),
|
||||
...(map["domain nameservers"] || []),
|
||||
...(map["domain servers in listed order"] || []), // .ly
|
||||
...(map["domain servers"] || []), // .tr
|
||||
...(map["name servers dns"] || []), // .mx
|
||||
...(map["ns 1"] || []),
|
||||
...(map["ns 2"] || []),
|
||||
...(map["ns 3"] || []),
|
||||
...(map["ns 4"] || []),
|
||||
];
|
||||
const nameservers: Nameserver[] | undefined = nsLines.length
|
||||
? (uniq(
|
||||
@@ -225,45 +266,117 @@ function anyValue(
|
||||
}
|
||||
|
||||
function collectContacts(map: Record<string, string[]>): Contact[] | undefined {
|
||||
const roles: Array<{ role: Contact["type"]; prefix: string }> = [
|
||||
{ role: "registrant", prefix: "registrant" },
|
||||
{ role: "admin", prefix: "admin" },
|
||||
{ role: "tech", prefix: "tech" },
|
||||
{ role: "billing", prefix: "billing" },
|
||||
{ role: "abuse", prefix: "abuse" },
|
||||
const roles: Array<{
|
||||
role: Contact["type"];
|
||||
prefixes: string[];
|
||||
}> = [
|
||||
{
|
||||
role: "registrant",
|
||||
prefixes: ["registrant", "owner", "holder"],
|
||||
},
|
||||
{
|
||||
role: "admin",
|
||||
prefixes: ["admin", "administrative"],
|
||||
},
|
||||
{
|
||||
role: "tech",
|
||||
prefixes: ["tech", "technical"],
|
||||
},
|
||||
{
|
||||
role: "billing",
|
||||
prefixes: ["billing"],
|
||||
},
|
||||
{
|
||||
role: "abuse",
|
||||
prefixes: ["abuse"],
|
||||
},
|
||||
];
|
||||
const contacts: Contact[] = [];
|
||||
for (const r of roles) {
|
||||
const name = anyValue(map, [
|
||||
`${r.prefix} name`,
|
||||
`${r.prefix} contact name`,
|
||||
`${r.prefix}`,
|
||||
]);
|
||||
const org = anyValue(map, [`${r.prefix} organization`, `${r.prefix} org`]);
|
||||
const email = anyValue(map, [
|
||||
`${r.prefix} email`,
|
||||
`${r.prefix} contact email`,
|
||||
`${r.prefix} e-mail`,
|
||||
]);
|
||||
const phone = anyValue(map, [
|
||||
`${r.prefix} phone`,
|
||||
`${r.prefix} contact phone`,
|
||||
`${r.prefix} telephone`,
|
||||
]);
|
||||
const fax = anyValue(map, [`${r.prefix} fax`, `${r.prefix} facsimile`]);
|
||||
const street = multi(map, [`${r.prefix} street`, `${r.prefix} address`]);
|
||||
const city = anyValue(map, [`${r.prefix} city`]);
|
||||
const state = anyValue(map, [
|
||||
`${r.prefix} state`,
|
||||
`${r.prefix} province`,
|
||||
`${r.prefix} state/province`,
|
||||
]);
|
||||
const postalCode = anyValue(map, [
|
||||
`${r.prefix} postal code`,
|
||||
`${r.prefix} postcode`,
|
||||
`${r.prefix} zip`,
|
||||
]);
|
||||
const country = anyValue(map, [`${r.prefix} country`]);
|
||||
const nameKeys: string[] = [];
|
||||
const orgKeys: string[] = [];
|
||||
const emailKeys: string[] = [];
|
||||
const phoneKeys: string[] = [];
|
||||
const faxKeys: string[] = [];
|
||||
const streetKeys: string[] = [];
|
||||
const cityKeys: string[] = [];
|
||||
const stateKeys: string[] = [];
|
||||
const postalCodeKeys: string[] = [];
|
||||
const countryKeys: string[] = [];
|
||||
|
||||
for (const prefix of r.prefixes) {
|
||||
nameKeys.push(`${prefix} name`, `${prefix} contact name`, `${prefix}`);
|
||||
if (prefix === "registrant") {
|
||||
nameKeys.push("registrant person"); // .ua
|
||||
}
|
||||
if (prefix === "owner") {
|
||||
nameKeys.push("owner name"); // .tm
|
||||
}
|
||||
|
||||
orgKeys.push(
|
||||
`${prefix} organization`,
|
||||
`${prefix} organisation`,
|
||||
`${prefix} org`,
|
||||
);
|
||||
if (prefix === "registrant") {
|
||||
orgKeys.push("trading as"); // .uk, .co.uk
|
||||
orgKeys.push("org"); // .ru
|
||||
}
|
||||
if (prefix === "owner") {
|
||||
orgKeys.push("owner orgname"); // .tm
|
||||
}
|
||||
|
||||
emailKeys.push(
|
||||
`${prefix} email`,
|
||||
`${prefix} contact email`,
|
||||
`${prefix} e-mail`,
|
||||
);
|
||||
|
||||
phoneKeys.push(
|
||||
`${prefix} phone`,
|
||||
`${prefix} contact phone`,
|
||||
`${prefix} telephone`,
|
||||
);
|
||||
|
||||
faxKeys.push(`${prefix} fax`, `${prefix} facsimile`);
|
||||
|
||||
streetKeys.push(
|
||||
`${prefix} street`,
|
||||
`${prefix} address`,
|
||||
`${prefix}'s address`,
|
||||
);
|
||||
if (prefix === "owner") {
|
||||
streetKeys.push("owner addr"); // .tm
|
||||
}
|
||||
|
||||
cityKeys.push(`${prefix} city`);
|
||||
|
||||
stateKeys.push(
|
||||
`${prefix} state`,
|
||||
`${prefix} province`,
|
||||
`${prefix} state/province`,
|
||||
);
|
||||
|
||||
postalCodeKeys.push(
|
||||
`${prefix} postal code`,
|
||||
`${prefix} postcode`,
|
||||
`${prefix} zip`,
|
||||
);
|
||||
|
||||
countryKeys.push(`${prefix} country`);
|
||||
}
|
||||
|
||||
const name = anyValue(map, nameKeys);
|
||||
const org = anyValue(map, orgKeys);
|
||||
const email = anyValue(map, emailKeys);
|
||||
const phone = anyValue(map, phoneKeys);
|
||||
const fax = anyValue(map, faxKeys);
|
||||
const street = multi(map, streetKeys);
|
||||
const city = anyValue(map, cityKeys);
|
||||
const state = anyValue(map, stateKeys);
|
||||
const postalCode = anyValue(map, postalCodeKeys);
|
||||
const country = anyValue(map, countryKeys);
|
||||
|
||||
if (name || org || email || phone || street?.length) {
|
||||
contacts.push({
|
||||
type: r.role,
|
||||
|
||||
Reference in New Issue
Block a user