From be8e88592f4a1f282779df15d7155059ea93395f Mon Sep 17 00:00:00 2001 From: Jake Jarvis Date: Sat, 19 Sep 2026 00:51:55 -0400 Subject: [PATCH] feat: add ccTLD-specific date formats, noisy-token parsing, and .gg/.je/.br/.ua normalization, closes #20 and #25 - Add `toISOFromTokens` to extract ISO dates from noisy values (e.g. ".ua" `0-UANIC 20111004161638`, `OK-UNTIL 20261004161638`) - Parse compact `YYYYMMDDHHMMSS` (.ua) and `YYYYMMDD` (.br, including ticket suffix like `20260319 #31066859`) date formats - Parse ordinal/prose dates used by .gg/.je (`28th December 2018 at 05:54:43.861`) - Teach `parseKeyValueLines` to collect indented continuation lines for header-style blocks (.gg/.je) without mistaking URLs or times as key separators - Extract .gg/.je expiry from `Relevant dates:` sentences and split inline registrar URL (`epag (http://www.epag.de)`) - Map .ua `status: OK-UNTIL ` to `expirationDate` when no explicit expiry field is present - Treat RDAP `pending release` / `release process: waiting` statuses as `isRegistered: false` (previously always `true`) - Mark WHOIS `.br` `release process: waiting` as available --- src/lib/dates.test.ts | 17 ++++++++- src/lib/dates.ts | 65 ++++++++++++++++++++++++++++++++- src/lib/text.ts | 15 ++++++++ src/rdap/normalize.test.ts | 23 ++++++++++++ src/rdap/normalize.ts | 9 ++++- src/whois/normalize.test.ts | 73 +++++++++++++++++++++++++++++++++++++ src/whois/normalize.ts | 53 +++++++++++++++++---------- 7 files changed, 232 insertions(+), 23 deletions(-) diff --git a/src/lib/dates.test.ts b/src/lib/dates.test.ts index c1cad44..6a44f71 100644 --- a/src/lib/dates.test.ts +++ b/src/lib/dates.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "vitest"; -import { toISO } from "./dates"; +import { toISO, toISOFromTokens } from "./dates"; test("toISO parses ISO and common whois formats", () => { const iso = toISO("2023-01-02T03:04:05Z"); @@ -42,3 +42,18 @@ test("toISO parses DD-MM-YYYY format (used by .il and .hk)", () => { const dmmmy = toISO("02-Jan-2023"); expect(dmmmy).toBe("2023-01-02T00:00:00Z"); }); + +test("toISO parses compact YYYYMMDDHHMMSS", () => { + expect(toISO("20261004161638")).toBe("2026-10-04T16:16:38Z"); +}); + +test("toISOFromTokens extracts a timestamp from noisy strings", () => { + expect(toISOFromTokens("0-UANIC 20111004161638")).toBe("2011-10-04T16:16:38Z"); + expect(toISOFromTokens("UARR149-UANIC 20251004050127")).toBe("2025-10-04T05:01:27Z"); + expect(toISOFromTokens("0-UANIC")).toBeUndefined(); +}); + +test("toISO parses compact YYYYMMDD, including .br 'created' values with a ticket suffix", () => { + expect(toISO("20260319")).toBe("2026-03-19T00:00:00Z"); + expect(toISOFromTokens("20260319 #31066859")).toBe("2026-03-19T00:00:00Z"); +}); diff --git a/src/lib/dates.ts b/src/lib/dates.ts index 677d807..e6bb8e7 100644 --- a/src/lib/dates.ts +++ b/src/lib/dates.ts @@ -18,6 +18,10 @@ export function toISO(dateLike: string | number | Date | undefined | null): stri /^(\d{2})-(\d{2})-(\d{4})$/, // Jan 02 2023 /^([A-Za-z]{3})\s+(\d{1,2})\s+(\d{4})$/, + // 20261004161638 (compact YYYYMMDDHHMMSS, used by .ua) + /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})$/, + // 20260319 (compact YYYYMMDD, used by .br) + /^(\d{4})(\d{2})(\d{2})$/, ]; for (const re of tryFormats) { const m = raw.match(re); @@ -25,12 +29,66 @@ export function toISO(dateLike: string | number | Date | undefined | null): stri const d = parseDateWithRegex(m, re); if (d) return toIsoFromDate(d); } + // 28th December 2018 [at 05:54:43[.861]] (used by .gg/.je) + const ordinal = raw.match( + /^(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3})[A-Za-z]*\s+(\d{4})(?:\s+at\s+(\d{1,2}):(\d{2}):(\d{2})(?:\.\d+)?)?$/, + ); + if (ordinal) { + const [, dd, mon, yyyy, hh, mm, ss] = ordinal; + const monthIdx = MONTHS[mon?.toLowerCase() ?? ""]; + if (monthIdx !== undefined) { + return toIsoFromDate( + new Date( + Date.UTC( + Number(yyyy), + monthIdx, + Number(dd), + Number(hh ?? 0), + Number(mm ?? 0), + Number(ss ?? 0), + ), + ), + ); + } + } // Fallback to native Date parsing (handles ISO and RFC2822 with TZ) const native = new Date(raw); if (!Number.isNaN(native.getTime())) return toIsoFromDate(native); return undefined; } +/** + * Like toISO, but for values with extra noise around the timestamp + * (e.g. "0-UANIC 20111004161638", "OK-UNTIL 20261004161638"): if the whole string doesn't parse, + * try each whitespace-separated token that looks like a date. + */ +export function toISOFromTokens(value: string | undefined | null): string | undefined { + if (!value) return undefined; + const whole = toISO(value); + if (whole) return whole; + for (const token of value.trim().split(/\s+/)) { + if (!/^\d[\d\-/:.TZ+]{5,}$/.test(token)) continue; + const iso = toISO(token); + if (iso) return iso; + } + return undefined; +} + +const MONTHS: Record = { + jan: 0, + feb: 1, + mar: 2, + apr: 3, + may: 4, + jun: 5, + jul: 6, + aug: 7, + sep: 8, + oct: 9, + nov: 10, + dec: 11, +}; + function toIsoFromDate(d: Date): string | undefined { try { return new Date( @@ -68,7 +126,7 @@ function parseDateWithRegex(m: RegExpMatchArray, _re: RegExp): Date | undefined }; try { // If the matched string contains time components, parse as Y-M-D H:M:S - if (m[0].includes(":")) { + if (m[0].includes(":") || /^\d{14}$/.test(m[0])) { const [_, y, mo, d, hh, mm, ss, offH, offM] = m; if (!y || !mo || !d || !hh || !mm || !ss) return undefined; // Base time as UTC @@ -84,6 +142,11 @@ function parseDateWithRegex(m: RegExpMatchArray, _re: RegExp): Date | undefined } return new Date(dt); } + // Compact YYYYMMDD + if (/^\d{8}$/.test(m[0])) { + const [_, y, mo, d] = m; + return new Date(Date.UTC(Number(y), Number(mo) - 1, Number(d))); + } // If the matched string contains hyphens, check if numeric (DD-MM-YYYY) or alpha (DD-MMM-YYYY) if (m[0].includes("-")) { const [_, dd, monStr, yyyy] = m; diff --git a/src/lib/text.ts b/src/lib/text.ts index 07ebb30..a24ee42 100644 --- a/src/lib/text.ts +++ b/src/lib/text.ts @@ -7,6 +7,8 @@ export function parseKeyValueLines(text: string): Record { const map = new Map(); const lines = text.split(/\r?\n/); let lastKey: string | undefined; + // True while inside a header-style block ("Key:" alone on its line, values indented below) + let inHeaderBlock = false; for (const rawLine of lines) { const line = rawLine.replace(/\s+$/, ""); if (!line.trim()) continue; @@ -19,6 +21,18 @@ export function parseKeyValueLines(text: string): Record { if (value) list.push(value); map.set(key, list); lastKey = key; + inHeaderBlock = false; + continue; + } + // Header-style block (.gg/.je): indented values may contain colons (URLs, times) that are + // not key separators, e.g. " Registered on 28th December 2018 at 05:54:43.861" + if (inHeaderBlock && lastKey && /^\s+/.test(line) && !/:(\s|$)/.test(line)) { + const value = line.trim(); + if (value) { + const list = map.get(lastKey) ?? []; + list.push(value); + map.set(lastKey, list); + } continue; } // Colon form: Key: value @@ -34,6 +48,7 @@ export function parseKeyValueLines(text: string): Record { if (value) list.push(value); map.set(key, list); lastKey = key; + inHeaderBlock = !value; continue; } // Continuation line: starts with indentation after a key appeared diff --git a/src/rdap/normalize.test.ts b/src/rdap/normalize.test.ts index 3a43486..bd0cfa4 100644 --- a/src/rdap/normalize.test.ts +++ b/src/rdap/normalize.test.ts @@ -112,3 +112,26 @@ test("normalizeRdap detects transfer lock with camelCase status", () => { const rec = normalizeRdap("example.com", "com", rdap, []); expect(rec.transferLock).toBe(true); }); + +test("normalizeRdap treats release-pending statuses as not registered", () => { + const rec = normalizeRdap( + "iba.com.br", + "com.br", + { + ldhName: "iba.com.br", + status: ["pending release"], + }, + [], + ); + expect(rec.isRegistered).toBe(false); + const active = normalizeRdap( + "example.com", + "com", + { + ldhName: "example.com", + status: ["active"], + }, + [], + ); + expect(active.isRegistered).toBe(true); +}); diff --git a/src/rdap/normalize.ts b/src/rdap/normalize.ts index cc17878..93f96ff 100644 --- a/src/rdap/normalize.ts +++ b/src/rdap/normalize.ts @@ -98,6 +98,13 @@ export function normalizeRdap( asDateLike(byAction("deletion")?.eventDate) ?? asDateLike(doc.deletionDate), ); + // Registries that keep answering for released/available names signal it via status + const isRegistered = !statuses?.some((s) => + /^(available|free|released|pending release|release process[:\s-]*waiting)$/i.test( + s.status.trim(), + ), + ); + // Derive a simple transfer lock flag from statuses const transferLock = !!statuses?.some((s: { status: string }) => /transfer[-\s]*prohibited/i.test(s.status), @@ -109,7 +116,7 @@ export function normalizeRdap( const record: DomainRecord = { domain: unicodeName || ldhName || inputDomain, tld, - isRegistered: true, + isRegistered, isIDN: /(^|\.)xn--/i.test(ldhName || inputDomain), unicodeName: unicodeName || undefined, punycodeName: ldhName || undefined, diff --git a/src/whois/normalize.test.ts b/src/whois/normalize.test.ts index 24bdb66..ac08ca3 100644 --- a/src/whois/normalize.test.ts +++ b/src/whois/normalize.test.ts @@ -207,3 +207,76 @@ test("isAvailableByWhois correctly identifies availability patterns", () => { const rec = normalizeWhois("example.com", "com", text, "whois.example.com"); expect(rec.isRegistered).toBe(false); }); + +test("WHOIS .gov.ua extracts dates from noisy values and OK-UNTIL", () => { + const text = ` +domain: agrex.gov.ua +status: OK-UNTIL 20261004161638 +created: 0-UANIC 20111004161638 +modified: UARR149-UANIC 20251004050127 +`; + const rec = normalizeWhois("agrex.gov.ua", "gov.ua", text, "whois.gov.ua"); + expect(rec.creationDate).toBe("2011-10-04T16:16:38Z"); + expect(rec.updatedDate).toBe("2025-10-04T05:01:27Z"); + expect(rec.expirationDate).toBe("2026-10-04T16:16:38Z"); +}); + +test("WHOIS .br 'release process: waiting' is not registered", () => { + const text = ` +domain: iba.com.br +release process: waiting +`; + expect(isAvailableByWhois(text)).toBe(true); + expect(normalizeWhois("iba.com.br", "com.br", text, "whois.registro.br").isRegistered).toBe( + false, + ); +}); + +test("WHOIS .br published domain is registered with compact dates", () => { + const text = ` +domain: iba.com.br +nserver: ns11.cloudns.net +created: 20260319 #31066859 +changed: 20260414 +expires: 20270319 +status: published +`; + const rec = normalizeWhois("iba.com.br", "com.br", text, "whois.registro.br"); + expect(rec.isRegistered).toBe(true); + expect(rec.creationDate).toBe("2026-03-19T00:00:00Z"); + expect(rec.updatedDate).toBe("2026-04-14T00:00:00Z"); + expect(rec.expirationDate).toBe("2027-03-19T00:00:00Z"); +}); + +test("WHOIS .gg header-style blocks with colons in values", () => { + const text = ` +Domain: + t3.gg + +Domain Status: + Active + +Registrant: + T3 Tools Inc + +Registrar: + epag (http://www.epag.de) + +Relevant dates: + Registered on 28th December 2018 at 05:54:43.861 + Registry fee due on 28th December each year + +Registration status: + Registered until cancelled + +Name servers: + ns1.vercel-dns.com + ns2.vercel-dns.com +`; + const rec = normalizeWhois("t3.gg", "gg", text, "whois.gg"); + expect(rec.isRegistered).toBe(true); + expect(rec.creationDate).toBe("2018-12-28T05:54:43Z"); + expect(rec.expirationDate).toBeUndefined(); + expect(rec.registrar).toEqual({ name: "epag", url: "http://www.epag.de" }); + expect(rec.nameservers?.map((n) => n.host)).toEqual(["ns1.vercel-dns.com", "ns2.vercel-dns.com"]); +}); diff --git a/src/whois/normalize.ts b/src/whois/normalize.ts index 1e97940..6f07a0a 100644 --- a/src/whois/normalize.ts +++ b/src/whois/normalize.ts @@ -1,4 +1,4 @@ -import { toISO } from "../lib/dates"; +import { toISOFromTokens } from "../lib/dates"; import { isPrivacyName } from "../lib/privacy"; import { parseKeyValueLines, uniq } from "../lib/text"; import type { Contact, DomainRecord, Nameserver, RegistrarInfo } from "../types"; @@ -31,6 +31,7 @@ const WHOIS_AVAILABLE_PATTERNS: RegExp[] = [ /\bno se encuentra registrado\b/i, // Spanish: "not found registered" /\bnicht gefunden\b/i, // German: "not found" /\bpending release\b/i, // often signals not registered/being deleted + /\brelease process:\s*waiting\b/i, // .br: expired, awaiting release ]; /** @@ -55,22 +56,26 @@ export function normalizeWhois( const map = parseKeyValueLines(whoisText); // Date extraction across common synonyms - const creationDate = anyValue(map, [ - "creation date", - "created on", - "created", - "registered on", - "registered", - "registration date", - "domain registration date", - "domain create date", - "domain name commencement date", - "registration time", // .cn - "domain record activated", // .edu - "domain registered", - "registered date", // .co.jp - "assigned", // .il - ]); + // .gg/.je list dates as sentences under "Relevant dates:", e.g. "Registered on 28th December 2018 at 05:54:43.861" + const relevantDate = (label: RegExp) => + map["relevant dates"]?.find((l) => label.test(l))?.replace(label, ""); + const creationDate = + anyValue(map, [ + "creation date", + "created on", + "created", + "registered on", + "registered", + "registration date", + "domain registration date", + "domain create date", + "domain name commencement date", + "registration time", // .cn + "domain record activated", // .edu + "domain registered", + "registered date", // .co.jp + "assigned", // .il + ]) ?? relevantDate(/^registered on\s+/i); const updatedDate = anyValue(map, [ "updated date", "updated", @@ -128,6 +133,11 @@ export function normalizeWhois( ]); const abuseEmail = anyValue(map, ["registrar abuse contact email", "abuse contact email"]); const abusePhone = anyValue(map, ["registrar abuse contact phone", "abuse contact phone"]); + // .gg/.je: "epag (http://www.epag.de)" + const inlineUrl = name?.match(/^(.*?)\s*\((https?:\/\/[^)\s]+)\)$/); + if (inlineUrl?.[1] && !url) { + return { name: inlineUrl[1], url: inlineUrl[2] }; + } if (!name && !ianaId && !url && !abuseEmail && !abusePhone) return undefined; return { name: name || undefined, @@ -156,6 +166,9 @@ export function normalizeWhois( .filter((s): s is { status: string; raw: string } => s !== null) : undefined; + // Some registries (.ua) publish expiry only as a status, e.g. "OK-UNTIL 20261004161638" + const okUntil = statuses?.find((s) => /^ok-until$/i.test(s.status))?.raw; + // Nameservers: also appear as "nserver" on some ccTLDs (.de, .ru) and as "name server" const nsLines: string[] = [ ...(map["name server"] || []), @@ -228,9 +241,9 @@ export function normalizeWhois( registrar, reseller: anyValue(map, ["reseller"]) || undefined, statuses, - creationDate: toISO(creationDate || undefined), - updatedDate: toISO(updatedDate || undefined), - expirationDate: toISO(expirationDate || undefined), + creationDate: toISOFromTokens(creationDate), + updatedDate: toISOFromTokens(updatedDate), + expirationDate: toISOFromTokens(expirationDate) ?? toISOFromTokens(okUntil), deletionDate: undefined, transferLock, dnssec,