fix: read more WHOIS date keys and reject vague dates

- Add the keys .th ("Created date", "Exp date"), .sk ("Valid Until"),
  .by ("Update Date"), and .kr ("Last Updated Date") use; all were missing.
- Take the latest value for the updated date: .il lists every "changed:" line
  oldest first, so its updated date was 1996.
- Keep the time in compact "20200427 16:03:40" dates (.at).
- Don't hand the native parser a word next to a year: .ro's "Before 2001"
  became 2001-01-01.

Across the saved corpus only those fields changed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
jakeandClaude Opus 5.5 committed 2026-10-03 15:23:30 -04:00
1 parent 7451b41cf0
commit 81a02a7eb3
3 files changed
+71 -25

No files matched your search

+11 -4
View File
@@ -27,8 +27,11 @@ const FORMATS: Array<[RegExp, (m: RegExpMatchArray) => number | undefined]> = [
[/^([A-Za-z]{3})\s+(\d{1,2})\s+(\d{4})$/, (m) => utc(m[3], monthOf(m[1]), m[2])],
// 20261004161638 (compact YYYYMMDDHHMMSS, used by .ua)
[/^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})$/, (m) => utc(m[1], m[2], m[3], m[4], m[5], m[6])],
// 20260319 (compact YYYYMMDD, used by .br)
[/^(\d{4})(\d{2})(\d{2})$/, (m) => utc(m[1], m[2], m[3])],
// 20260319 (compact YYYYMMDD, used by .br), optionally with a time: 20200427 16:03:40 (.at)
[
/^(\d{4})(\d{2})(\d{2})(?:[ T](\d{2}):(\d{2}):(\d{2}))?$/,
(m) => utc(m[1], m[2], m[3], m[4], m[5], m[6]),
],
// 28th December 2018 [at 05:54:43[.861]] (used by .gg/.je)
[
/^(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3})[A-Za-z]*\s+(\d{4})(?:\s+at\s+(\d{1,2}):(\d{2}):(\d{2})(?:\.\d+)?)?$/,
@@ -51,8 +54,12 @@ export function toISO(dateLike: string | number | Date | undefined | null): stri
const ms = build(m);
if (ms !== undefined) return toIsoFromDate(new Date(ms));
}
// Anything else goes to the native parser (RFC 2822, "Tue Jan 01 2000", ...), which reads a
// value without a zone as local time: pin those to UTC.
// Anything else goes to the native parser (RFC 2822, "Tue Jan 01 2000", ...). It accepts any
// word next to a year ("Before 2001" on .ro, "since 1999"), so require a month name or a
// second number. It reads a value without a zone as local time: pin those to UTC.
if (!/\b(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/i.test(raw) && !/\d\D+\d/.test(raw)) {
return undefined;
}
const hasZone = /\b(?:UTC?|GMT|[ECMP][SD]T)\b|(?:\dZ|[+-]\d{2}:?\d{2})$/i.test(raw);
for (const candidate of hasZone ? [raw] : [`${raw} UTC`, `${raw}Z`]) {
const native = new Date(candidate);
+24
View File
@@ -582,3 +582,27 @@ test("WHOIS nameserver keys and continuation lines across ccTLD layouts", () =>
).toEqual(["auth01.ns.dk-hostmaster.dk"]);
expect(hosts("nic.lv", "Domain: nic.lv\nNserver: -\n")).toBeUndefined();
});
test("WHOIS date keys and forms across ccTLDs", () => {
const rec = (text: string) =>
normalizeWhois("example.test", "test", `domain: example.test\n${text}`, undefined);
// .th, .sk, .by, .kr keys
expect(rec("Created date: 30 Mar 1999\nExp date: 01 Apr 2030\n")).toMatchObject({
creationDate: "1999-03-30T00:00:00Z",
expirationDate: "2030-04-01T00:00:00Z",
});
expect(rec("Valid Until: 2027-06-10\n").expirationDate).toBe("2027-06-10T00:00:00Z");
expect(rec("Update Date: 2025-10-21\n").updatedDate).toBe("2025-10-21T00:00:00Z");
expect(rec("Last Updated Date : 2025. 05. 09.\n").updatedDate).toBe(
"2025-05-09T00:00:00Z",
);
// .il lists every change, oldest first
expect(
rec(
"changed: domain-registrar AT isoc.org.il 19960111 (Assigned)\nchanged: domain-registrar AT isoc.org.il 20260930 (Changed)\n",
).updatedDate,
).toBe("2026-09-30T00:00:00Z");
// .at keeps the time; .ro's "Before 2001" is not a date
expect(rec("changed: 20200427 16:03:40\n").updatedDate).toBe("2020-04-27T16:03:40Z");
expect(rec("Registered On: Before 2001\n").creationDate).toBeUndefined();
});
+36 -21
View File
@@ -76,7 +76,8 @@ export function normalizeWhois(
// block beats a higher-priority key in a later one: FRED registries (.cz, .mk) repeat "created"
// in every contact and nameserver block. The whole-reply map covers a header whose value sits
// past a blank line.
const dateOf = (keys: string[]) => firstDate(blocks, keys) ?? firstDate([map], keys);
const dateOf = (keys: string[], latest = false) =>
firstDate(blocks, keys, latest) ?? firstDate([map], keys, latest);
// .gg/.je list dates as sentences under "Relevant dates:", e.g. "Registered on 28th December 2018 at 05:54:43.861"
const relevantDate = (label: RegExp) =>
toISOFromTokens(map["relevant dates"]?.find((l) => label.test(l))?.replace(label, ""));
@@ -96,25 +97,31 @@ export function normalizeWhois(
"registration time", // .cn
"domain record activated", // .edu
"domain registered",
"registered date", // .co.jp
"registered date", // .co.jp, .kr
"created date", // .th
"assigned", // .il
]) ??
relevantDate(/^registered on\s+/i) ??
recordDate(/^[ \t]*Record created on[ \t]+(.+?)\.?$/im);
const updatedDate =
dateOf([
"updated date",
"updated",
"last updated",
"last updated on", // .mx
"last update", // .co.jp
"last-update", // .fr
"last modified",
"modified",
"changed",
"modification date",
"domain record last updated", // .edu
]) ?? recordDate(/^[ \t]*Record last updated on[ \t]+(.+?)\.?$/im);
dateOf(
[
"updated date",
"updated",
"last updated",
"last updated on", // .mx
"last update", // .co.jp
"last-update", // .fr
"last modified",
"modified",
"changed",
"modification date",
"last updated date", // .kr
"update date", // .by
"domain record last updated", // .edu
],
true,
) ?? recordDate(/^[ \t]*Record last updated on[ \t]+(.+?)\.?$/im);
const expirationDate =
dateOf([
"registry expiry date",
@@ -137,6 +144,8 @@ export function normalizeWhois(
"renewal date", // .pl
"validity", // .il
"record will expire on",
"exp date", // .th
"valid until", // .sk
]) ?? recordDate(/^[ \t]*Record expires on[ \t]+(.+?)\.?$/im);
// Registrar info (thin registries like .com/.net require referral follow for full data)
@@ -344,14 +353,20 @@ function parseStatusLine(line: string): string[] {
.filter(Boolean);
}
/** First value that parses as a date, taking blocks in order and keys by priority within each. */
function firstDate(blocks: Array<Record<string, string[]>>, keys: string[]): string | undefined {
/**
* First value that parses as a date, taking blocks in order and keys by priority within each.
* With `latest`, the latest of the first matching key's values instead (.il lists every
* "changed:" line, oldest first).
*/
function firstDate(
blocks: Array<Record<string, string[]>>,
keys: string[],
latest = false,
): string | undefined {
for (const block of blocks) {
for (const k of keys) {
for (const value of block[k] ?? []) {
const iso = toISOFromTokens(value);
if (iso) return iso;
}
const dates = (block[k] ?? []).map((v) => toISOFromTokens(v)).filter((d) => d !== undefined);
if (dates.length) return latest ? dates.sort().at(-1) : dates[0];
}
}
return undefined;