1
mirror of https://github.com/jakejarvis/hoot.git synced 2025-10-18 20:14:25 -04:00

Refactor DNS record sorting to be handled server-side and update related tests

This commit is contained in:
2025-10-13 09:48:39 -04:00
parent 59263aae46
commit e56d57b23d
11 changed files with 118 additions and 59 deletions

View File

@@ -21,7 +21,7 @@ vi.mock("@/components/ui/tooltip", () => ({
}));
describe("DnsRecordList", () => {
it("sorts MX by priority and renders TTL badges", () => {
it("renders MX with TTL badges (sorting handled server-side)", () => {
const records = [
{
type: "MX",
@@ -52,11 +52,14 @@ describe("DnsRecordList", () => {
const items = Array.from(
document.querySelectorAll("span.min-w-0.flex-1.truncate"),
).map((el) => (el as HTMLElement).textContent);
expect(items).toEqual([
"mx-a.example.com",
"mx-b.example.com",
"mx-c.example.com",
]);
// Ensure all rendered
expect(items).toEqual(
expect.arrayContaining([
"mx-a.example.com",
"mx-b.example.com",
"mx-c.example.com",
]),
);
// TTL badge presence (not exact text as it's formatted)
expect(document.querySelectorAll('[data-slot="badge"]')).toBeTruthy();

View File

@@ -20,14 +20,6 @@ export function DnsRecordList({
}) {
const filtered = useMemo(() => {
const arr = records.filter((r) => r.type === type);
if (type === "MX") {
arr.sort((a, b) => {
const ap = a.priority ?? Number.MAX_SAFE_INTEGER;
const bp = b.priority ?? Number.MAX_SAFE_INTEGER;
if (ap !== bp) return ap - bp;
return a.value.localeCompare(b.value);
});
}
return arr;
}, [records, type]);

View File

@@ -20,10 +20,10 @@ import { SECTION_DEFS } from "@/lib/sections-meta";
function DnsGroupSkeleton({
title,
rows = 2,
records = 2,
}: {
title: string;
rows?: number;
records?: number;
}) {
return (
<div className="space-y-2">
@@ -35,8 +35,8 @@ function DnsGroupSkeleton({
</div>
<div className="grid grid-cols-1 gap-2 sm:grid-cols-2">
{Array.from(
{ length: rows },
(_, n) => `dns-skel-${title}-${rows}-${n}`,
{ length: records },
(_, n) => `dns-skel-${title}-${records}-${n}`,
).map((id) => (
<KeyValueSkeleton key={id} withTrailing widthClass="w-[100px]" />
))}
@@ -76,11 +76,11 @@ export function DnsRecordsSection({
<Section {...SECTION_DEFS.dns} isError={isError} isLoading={isLoading}>
{isLoading ? (
<div className="space-y-4">
<DnsGroupSkeleton title="A Records" rows={1} />
<DnsGroupSkeleton title="AAAA Records" rows={1} />
<DnsGroupSkeleton title="MX Records" rows={2} />
<DnsGroupSkeleton title="TXT Records" rows={4} />
<DnsGroupSkeleton title="NS Records" rows={2} />
<DnsGroupSkeleton title="A Records" />
<DnsGroupSkeleton title="AAAA Records" />
<DnsGroupSkeleton title="MX Records" />
<DnsGroupSkeleton title="TXT Records" records={4} />
<DnsGroupSkeleton title="NS Records" />
</div>
) : records ? (
<div className="space-y-4">

View File

@@ -3,8 +3,8 @@ import { Redis } from "@upstash/redis";
// Uses KV_REST_API_URL and KV_REST_API_TOKEN set by Vercel integration
export const redis = Redis.fromEnv();
export function ns(n: string, id?: string): string {
return `${n}${id ? `:${id}` : ""}`;
export function ns(...parts: string[]): string {
return parts.join(":");
}
function sleep(ms: number): Promise<void> {

View File

@@ -51,26 +51,32 @@ export async function resolveAll(domain: string): Promise<DnsResolveResult> {
const providers = providerOrderForLookup(lower);
const durationByProvider: Record<string, number> = {};
let lastError: unknown = null;
const aggregateKey = ns("dns:all", lower);
const lockKey = ns("lock", `dns:${lower}`);
const aggregateKey = ns("dns", lower);
const lockKey = ns("lock", "dns", lower);
// Aggregate cache fast-path
try {
const agg = (await redis.get(aggregateKey)) as DnsResolveResult | null;
if (agg && Array.isArray(agg.records)) {
// Normalize sorting for returned aggregate in case older cache entries
// were stored before server-side sorting was added.
const sortedAggRecords = sortDnsRecordsByType(
agg.records,
DnsTypeSchema.options,
);
await captureServer("dns_resolve_all", {
domain: lower,
duration_ms_total: Date.now() - startedAt,
counts: ((): Record<DnsType, number> => {
return (DnsTypeSchema.options as DnsType[]).reduce(
(acc, t) => {
acc[t] = agg.records.filter((r) => r.type === t).length;
acc[t] = sortedAggRecords.filter((r) => r.type === t).length;
return acc;
},
{ A: 0, AAAA: 0, MX: 0, TXT: 0, NS: 0 } as Record<DnsType, number>,
);
})(),
cloudflare_ip_present: agg.records.some(
cloudflare_ip_present: sortedAggRecords.some(
(r) => (r.type === "A" || r.type === "AAAA") && r.isCloudflare,
),
dns_provider_used: agg.resolver,
@@ -82,9 +88,9 @@ export async function resolveAll(domain: string): Promise<DnsResolveResult> {
console.info("[dns] aggregate cache hit", {
domain: lower,
resolver: agg.resolver,
total: agg.records.length,
total: sortedAggRecords.length,
});
return agg;
return { records: sortedAggRecords, resolver: agg.resolver };
}
} catch {}
@@ -121,7 +127,11 @@ export async function resolveAll(domain: string): Promise<DnsResolveResult> {
lock_waited_ms: Date.now() - lockWaitStart,
});
console.info("[dns] waited for aggregate", { domain: lower });
return agg;
const sortedAggRecords = sortDnsRecordsByType(
agg.records,
DnsTypeSchema.options,
);
return { records: sortedAggRecords, resolver: agg.resolver };
}
const acquiredLock = lockResult.acquired;
if (!acquiredLock && !lockResult.cachedResult) {
@@ -160,7 +170,11 @@ export async function resolveAll(domain: string): Promise<DnsResolveResult> {
lock_acquired: false,
lock_waited_ms: Date.now() - start,
});
return agg;
const sortedAggRecords = sortDnsRecordsByType(
agg.records,
DnsTypeSchema.options,
);
return { records: sortedAggRecords, resolver: agg.resolver };
}
await new Promise((r) => setTimeout(r, intervalMs));
}
@@ -175,7 +189,11 @@ export async function resolveAll(domain: string): Promise<DnsResolveResult> {
);
const allCached = cachedByType.every((arr) => Array.isArray(arr));
if (allCached) {
const flat = (cachedByType as DnsRecord[][]).flat();
// Ensure per-type cached arrays are normalized for sorting
const sortedByType = (cachedByType as DnsRecord[][]).map((arr, idx) =>
sortDnsRecordsForType(arr.slice(), types[idx] as DnsType),
);
const flat = (sortedByType as DnsRecord[][]).flat();
const counts = types.reduce(
(acc, t) => {
acc[t] = flat.filter((r) => r.type === t).length;
@@ -187,9 +205,8 @@ export async function resolveAll(domain: string): Promise<DnsResolveResult> {
(r) => (r.type === "A" || r.type === "AAAA") && r.isCloudflare,
);
const resolverUsed =
((await redis.get(
ns("dns:meta", `${lower}:resolver`),
)) as DnsResolver | null) || "cloudflare";
((await redis.get(ns("dns", lower, "resolver"))) as DnsResolver | null) ||
"cloudflare";
try {
await redis.set(
aggregateKey,
@@ -232,9 +249,11 @@ export async function resolveAll(domain: string): Promise<DnsResolveResult> {
let usedFresh = false;
const results = await Promise.all(
types.map(async (type) => {
const key = ns("dns", `${lower}:${type}`);
const key = ns("dns", lower, type);
const cached = await redis.get<DnsRecord[]>(key);
if (cached) return cached;
if (cached) {
return sortDnsRecordsForType(cached.slice(), type as DnsType);
}
const fresh = await resolveTypeWithProvider(domain, type, provider);
await redis.set(key, fresh, { ex: 5 * 60 });
usedFresh = usedFresh || true;
@@ -256,14 +275,14 @@ export async function resolveAll(domain: string): Promise<DnsResolveResult> {
);
// Persist the resolver metadata only when we actually fetched fresh data
if (usedFresh) {
await redis.set(ns("dns:meta", `${lower}:resolver`), provider.key, {
await redis.set(ns("dns", `${lower}:resolver`), provider.key, {
ex: 5 * 60,
});
}
const resolverUsed = usedFresh
? provider.key
: ((await redis.get(
ns("dns:meta", `${lower}:resolver`),
ns("dns", lower, "resolver"),
)) as DnsResolver | null) || provider.key;
try {
await redis.set(
@@ -343,7 +362,8 @@ async function resolveTypeWithProvider(
const normalizedRecords = await Promise.all(
ans.map((a) => normalizeAnswer(domain, type, a)),
);
return normalizedRecords.filter(Boolean) as DnsRecord[];
const records = normalizedRecords.filter(Boolean) as DnsRecord[];
return sortDnsRecordsForType(records, type);
}
async function normalizeAnswer(
@@ -388,6 +408,43 @@ function trimQuotes(s: string) {
return s.replace(/^"|"$/g, "");
}
function sortDnsRecordsByType(
records: DnsRecord[],
order: readonly DnsType[],
): DnsRecord[] {
const byType: Record<DnsType, DnsRecord[]> = {
A: [],
AAAA: [],
MX: [],
TXT: [],
NS: [],
};
for (const r of records) byType[r.type].push(r);
const sorted: DnsRecord[] = [];
for (const t of order) {
sorted.push(...sortDnsRecordsForType(byType[t] as DnsRecord[], t));
}
return sorted;
}
function sortDnsRecordsForType(arr: DnsRecord[], type: DnsType): DnsRecord[] {
if (type === "MX") {
arr.sort((a, b) => {
const ap = (a.priority ?? Number.MAX_SAFE_INTEGER) as number;
const bp = (b.priority ?? Number.MAX_SAFE_INTEGER) as number;
if (ap !== bp) return ap - bp;
return a.value.localeCompare(b.value);
});
return arr;
}
if (type === "TXT" || type === "NS") {
arr.sort((a, b) => a.value.localeCompare(b.value));
return arr;
}
// For A/AAAA retain provider order
return arr;
}
type DnsJson = {
Status: number;
Answer?: DnsAnswer[];

View File

@@ -48,8 +48,8 @@ export async function getOrCreateFaviconBlobUrl(
const startedAt = Date.now();
console.debug("[favicon] start", { domain, size: DEFAULT_SIZE });
const indexKey = ns("favicon:url", `${domain}:${DEFAULT_SIZE}`);
const lockKey = ns("lock", `favicon:${domain}:${DEFAULT_SIZE}`);
const indexKey = ns("favicon", "url", domain, String(DEFAULT_SIZE));
const lockKey = ns("lock", "favicon", domain, String(DEFAULT_SIZE));
// 1) Check Redis index first (supports positive and negative cache)
try {

View File

@@ -6,7 +6,7 @@ export async function probeHeaders(domain: string): Promise<HttpHeader[]> {
const lower = domain.toLowerCase();
const url = `https://${domain}/`;
const key = ns("headers", lower);
const lockKey = ns("lock", `headers:${lower}`);
const lockKey = ns("lock", "headers", lower);
console.debug("[headers] start", { domain: lower });
const cached = await redis.get<HttpHeader[]>(key);

View File

@@ -58,12 +58,16 @@ export async function getOrCreateScreenshotBlobUrl(
const backoffMaxMs = options?.backoffMaxMs ?? CAPTURE_BACKOFF_MAX_MS_DEFAULT;
const indexKey = ns(
"screenshot:url",
`${domain}:${VIEWPORT_WIDTH}x${VIEWPORT_HEIGHT}`,
"screenshot",
"url",
domain,
`${VIEWPORT_WIDTH}x${VIEWPORT_HEIGHT}`,
);
const lockKey = ns(
"lock",
`screenshot:${domain}:${VIEWPORT_WIDTH}x${VIEWPORT_HEIGHT}`,
"screenshot",
domain,
`${VIEWPORT_WIDTH}x${VIEWPORT_HEIGHT}`,
);
// 1) Check Redis index first

View File

@@ -60,8 +60,7 @@ function textResponse(text: string, contentType = "text/plain") {
describe("getSeo", () => {
it("uses cached response when meta exists in cache", async () => {
const { ns, redis } = await import("@/lib/redis");
const baseKey = ns("seo", "example.com");
const metaKey = ns(`${baseKey}:meta`, "v1");
const metaKey = ns("seo", "example.com", "meta");
await redis.set(metaKey, {
meta: null,
robots: null,
@@ -138,8 +137,7 @@ describe("getSeo", () => {
it("uses cached robots when present and avoids second fetch", async () => {
const { ns, redis } = await import("@/lib/redis");
const baseKey = ns("seo", "example.com");
const robotsKey = ns(`${baseKey}:robots`, "v1");
const robotsKey = ns("seo", "example.com", "robots");
await redis.set(robotsKey, {
fetched: true,
groups: [{ userAgents: ["*"], rules: [{ type: "allow", value: "/" }] }],

View File

@@ -33,9 +33,8 @@ async function fetchWithTimeout(
export async function getSeo(domain: string): Promise<SeoResponse> {
const lower = domain.toLowerCase();
const baseKey = ns("seo", lower);
const metaKey = ns(`${baseKey}:meta`, "v1");
const robotsKey = ns(`${baseKey}:robots`, "v1");
const metaKey = ns("seo", lower, "meta");
const robotsKey = ns("seo", lower, "robots");
console.debug("[seo] start", { domain: lower });
const cached = await redis.get<SeoResponse>(metaKey);
@@ -178,12 +177,18 @@ async function getOrCreateSocialPreviewImageUrl(
const hash = deterministicHash(imageUrl);
const indexKey = ns(
"seo:image-url",
`${lower}:${hash}:${SOCIAL_WIDTH}x${SOCIAL_HEIGHT}`,
"seo",
"image-url",
lower,
hash,
`${SOCIAL_WIDTH}x${SOCIAL_HEIGHT}`,
);
const lockKey = ns(
"lock",
`${lower}:seo-image:${hash}:${SOCIAL_WIDTH}x${SOCIAL_HEIGHT}`,
"seo-image",
lower,
hash,
`${SOCIAL_WIDTH}x${SOCIAL_HEIGHT}`,
);
// 1) Check Redis index first

View File

@@ -15,7 +15,7 @@ const __redisImpl = vi.hoisted(() => {
const store = new Map<string, unknown>();
// simple sorted-set implementation: key -> Map(member -> score)
const zsets = new Map<string, Map<string, number>>();
const ns = (n: string, id?: string) => `${n}${id ? `:${id}` : ""}`;
const ns = (...parts: string[]) => parts.join(":");
const get = vi.fn(async (key: string) =>
store.has(key) ? store.get(key) : null,