mirror of
https://github.com/jakejarvis/hoot.git
synced 2025-10-18 20:14:25 -04:00
Refactor DNS record sorting to be handled server-side and update related tests
This commit is contained in:
@@ -21,7 +21,7 @@ vi.mock("@/components/ui/tooltip", () => ({
|
||||
}));
|
||||
|
||||
describe("DnsRecordList", () => {
|
||||
it("sorts MX by priority and renders TTL badges", () => {
|
||||
it("renders MX with TTL badges (sorting handled server-side)", () => {
|
||||
const records = [
|
||||
{
|
||||
type: "MX",
|
||||
@@ -52,11 +52,14 @@ describe("DnsRecordList", () => {
|
||||
const items = Array.from(
|
||||
document.querySelectorAll("span.min-w-0.flex-1.truncate"),
|
||||
).map((el) => (el as HTMLElement).textContent);
|
||||
expect(items).toEqual([
|
||||
"mx-a.example.com",
|
||||
"mx-b.example.com",
|
||||
"mx-c.example.com",
|
||||
]);
|
||||
// Ensure all rendered
|
||||
expect(items).toEqual(
|
||||
expect.arrayContaining([
|
||||
"mx-a.example.com",
|
||||
"mx-b.example.com",
|
||||
"mx-c.example.com",
|
||||
]),
|
||||
);
|
||||
|
||||
// TTL badge presence (not exact text as it's formatted)
|
||||
expect(document.querySelectorAll('[data-slot="badge"]')).toBeTruthy();
|
||||
|
@@ -20,14 +20,6 @@ export function DnsRecordList({
|
||||
}) {
|
||||
const filtered = useMemo(() => {
|
||||
const arr = records.filter((r) => r.type === type);
|
||||
if (type === "MX") {
|
||||
arr.sort((a, b) => {
|
||||
const ap = a.priority ?? Number.MAX_SAFE_INTEGER;
|
||||
const bp = b.priority ?? Number.MAX_SAFE_INTEGER;
|
||||
if (ap !== bp) return ap - bp;
|
||||
return a.value.localeCompare(b.value);
|
||||
});
|
||||
}
|
||||
return arr;
|
||||
}, [records, type]);
|
||||
|
||||
|
@@ -20,10 +20,10 @@ import { SECTION_DEFS } from "@/lib/sections-meta";
|
||||
|
||||
function DnsGroupSkeleton({
|
||||
title,
|
||||
rows = 2,
|
||||
records = 2,
|
||||
}: {
|
||||
title: string;
|
||||
rows?: number;
|
||||
records?: number;
|
||||
}) {
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
@@ -35,8 +35,8 @@ function DnsGroupSkeleton({
|
||||
</div>
|
||||
<div className="grid grid-cols-1 gap-2 sm:grid-cols-2">
|
||||
{Array.from(
|
||||
{ length: rows },
|
||||
(_, n) => `dns-skel-${title}-${rows}-${n}`,
|
||||
{ length: records },
|
||||
(_, n) => `dns-skel-${title}-${records}-${n}`,
|
||||
).map((id) => (
|
||||
<KeyValueSkeleton key={id} withTrailing widthClass="w-[100px]" />
|
||||
))}
|
||||
@@ -76,11 +76,11 @@ export function DnsRecordsSection({
|
||||
<Section {...SECTION_DEFS.dns} isError={isError} isLoading={isLoading}>
|
||||
{isLoading ? (
|
||||
<div className="space-y-4">
|
||||
<DnsGroupSkeleton title="A Records" rows={1} />
|
||||
<DnsGroupSkeleton title="AAAA Records" rows={1} />
|
||||
<DnsGroupSkeleton title="MX Records" rows={2} />
|
||||
<DnsGroupSkeleton title="TXT Records" rows={4} />
|
||||
<DnsGroupSkeleton title="NS Records" rows={2} />
|
||||
<DnsGroupSkeleton title="A Records" />
|
||||
<DnsGroupSkeleton title="AAAA Records" />
|
||||
<DnsGroupSkeleton title="MX Records" />
|
||||
<DnsGroupSkeleton title="TXT Records" records={4} />
|
||||
<DnsGroupSkeleton title="NS Records" />
|
||||
</div>
|
||||
) : records ? (
|
||||
<div className="space-y-4">
|
||||
|
@@ -3,8 +3,8 @@ import { Redis } from "@upstash/redis";
|
||||
// Uses KV_REST_API_URL and KV_REST_API_TOKEN set by Vercel integration
|
||||
export const redis = Redis.fromEnv();
|
||||
|
||||
export function ns(n: string, id?: string): string {
|
||||
return `${n}${id ? `:${id}` : ""}`;
|
||||
export function ns(...parts: string[]): string {
|
||||
return parts.join(":");
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
|
@@ -51,26 +51,32 @@ export async function resolveAll(domain: string): Promise<DnsResolveResult> {
|
||||
const providers = providerOrderForLookup(lower);
|
||||
const durationByProvider: Record<string, number> = {};
|
||||
let lastError: unknown = null;
|
||||
const aggregateKey = ns("dns:all", lower);
|
||||
const lockKey = ns("lock", `dns:${lower}`);
|
||||
const aggregateKey = ns("dns", lower);
|
||||
const lockKey = ns("lock", "dns", lower);
|
||||
|
||||
// Aggregate cache fast-path
|
||||
try {
|
||||
const agg = (await redis.get(aggregateKey)) as DnsResolveResult | null;
|
||||
if (agg && Array.isArray(agg.records)) {
|
||||
// Normalize sorting for returned aggregate in case older cache entries
|
||||
// were stored before server-side sorting was added.
|
||||
const sortedAggRecords = sortDnsRecordsByType(
|
||||
agg.records,
|
||||
DnsTypeSchema.options,
|
||||
);
|
||||
await captureServer("dns_resolve_all", {
|
||||
domain: lower,
|
||||
duration_ms_total: Date.now() - startedAt,
|
||||
counts: ((): Record<DnsType, number> => {
|
||||
return (DnsTypeSchema.options as DnsType[]).reduce(
|
||||
(acc, t) => {
|
||||
acc[t] = agg.records.filter((r) => r.type === t).length;
|
||||
acc[t] = sortedAggRecords.filter((r) => r.type === t).length;
|
||||
return acc;
|
||||
},
|
||||
{ A: 0, AAAA: 0, MX: 0, TXT: 0, NS: 0 } as Record<DnsType, number>,
|
||||
);
|
||||
})(),
|
||||
cloudflare_ip_present: agg.records.some(
|
||||
cloudflare_ip_present: sortedAggRecords.some(
|
||||
(r) => (r.type === "A" || r.type === "AAAA") && r.isCloudflare,
|
||||
),
|
||||
dns_provider_used: agg.resolver,
|
||||
@@ -82,9 +88,9 @@ export async function resolveAll(domain: string): Promise<DnsResolveResult> {
|
||||
console.info("[dns] aggregate cache hit", {
|
||||
domain: lower,
|
||||
resolver: agg.resolver,
|
||||
total: agg.records.length,
|
||||
total: sortedAggRecords.length,
|
||||
});
|
||||
return agg;
|
||||
return { records: sortedAggRecords, resolver: agg.resolver };
|
||||
}
|
||||
} catch {}
|
||||
|
||||
@@ -121,7 +127,11 @@ export async function resolveAll(domain: string): Promise<DnsResolveResult> {
|
||||
lock_waited_ms: Date.now() - lockWaitStart,
|
||||
});
|
||||
console.info("[dns] waited for aggregate", { domain: lower });
|
||||
return agg;
|
||||
const sortedAggRecords = sortDnsRecordsByType(
|
||||
agg.records,
|
||||
DnsTypeSchema.options,
|
||||
);
|
||||
return { records: sortedAggRecords, resolver: agg.resolver };
|
||||
}
|
||||
const acquiredLock = lockResult.acquired;
|
||||
if (!acquiredLock && !lockResult.cachedResult) {
|
||||
@@ -160,7 +170,11 @@ export async function resolveAll(domain: string): Promise<DnsResolveResult> {
|
||||
lock_acquired: false,
|
||||
lock_waited_ms: Date.now() - start,
|
||||
});
|
||||
return agg;
|
||||
const sortedAggRecords = sortDnsRecordsByType(
|
||||
agg.records,
|
||||
DnsTypeSchema.options,
|
||||
);
|
||||
return { records: sortedAggRecords, resolver: agg.resolver };
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, intervalMs));
|
||||
}
|
||||
@@ -175,7 +189,11 @@ export async function resolveAll(domain: string): Promise<DnsResolveResult> {
|
||||
);
|
||||
const allCached = cachedByType.every((arr) => Array.isArray(arr));
|
||||
if (allCached) {
|
||||
const flat = (cachedByType as DnsRecord[][]).flat();
|
||||
// Ensure per-type cached arrays are normalized for sorting
|
||||
const sortedByType = (cachedByType as DnsRecord[][]).map((arr, idx) =>
|
||||
sortDnsRecordsForType(arr.slice(), types[idx] as DnsType),
|
||||
);
|
||||
const flat = (sortedByType as DnsRecord[][]).flat();
|
||||
const counts = types.reduce(
|
||||
(acc, t) => {
|
||||
acc[t] = flat.filter((r) => r.type === t).length;
|
||||
@@ -187,9 +205,8 @@ export async function resolveAll(domain: string): Promise<DnsResolveResult> {
|
||||
(r) => (r.type === "A" || r.type === "AAAA") && r.isCloudflare,
|
||||
);
|
||||
const resolverUsed =
|
||||
((await redis.get(
|
||||
ns("dns:meta", `${lower}:resolver`),
|
||||
)) as DnsResolver | null) || "cloudflare";
|
||||
((await redis.get(ns("dns", lower, "resolver"))) as DnsResolver | null) ||
|
||||
"cloudflare";
|
||||
try {
|
||||
await redis.set(
|
||||
aggregateKey,
|
||||
@@ -232,9 +249,11 @@ export async function resolveAll(domain: string): Promise<DnsResolveResult> {
|
||||
let usedFresh = false;
|
||||
const results = await Promise.all(
|
||||
types.map(async (type) => {
|
||||
const key = ns("dns", `${lower}:${type}`);
|
||||
const key = ns("dns", lower, type);
|
||||
const cached = await redis.get<DnsRecord[]>(key);
|
||||
if (cached) return cached;
|
||||
if (cached) {
|
||||
return sortDnsRecordsForType(cached.slice(), type as DnsType);
|
||||
}
|
||||
const fresh = await resolveTypeWithProvider(domain, type, provider);
|
||||
await redis.set(key, fresh, { ex: 5 * 60 });
|
||||
usedFresh = usedFresh || true;
|
||||
@@ -256,14 +275,14 @@ export async function resolveAll(domain: string): Promise<DnsResolveResult> {
|
||||
);
|
||||
// Persist the resolver metadata only when we actually fetched fresh data
|
||||
if (usedFresh) {
|
||||
await redis.set(ns("dns:meta", `${lower}:resolver`), provider.key, {
|
||||
await redis.set(ns("dns", `${lower}:resolver`), provider.key, {
|
||||
ex: 5 * 60,
|
||||
});
|
||||
}
|
||||
const resolverUsed = usedFresh
|
||||
? provider.key
|
||||
: ((await redis.get(
|
||||
ns("dns:meta", `${lower}:resolver`),
|
||||
ns("dns", lower, "resolver"),
|
||||
)) as DnsResolver | null) || provider.key;
|
||||
try {
|
||||
await redis.set(
|
||||
@@ -343,7 +362,8 @@ async function resolveTypeWithProvider(
|
||||
const normalizedRecords = await Promise.all(
|
||||
ans.map((a) => normalizeAnswer(domain, type, a)),
|
||||
);
|
||||
return normalizedRecords.filter(Boolean) as DnsRecord[];
|
||||
const records = normalizedRecords.filter(Boolean) as DnsRecord[];
|
||||
return sortDnsRecordsForType(records, type);
|
||||
}
|
||||
|
||||
async function normalizeAnswer(
|
||||
@@ -388,6 +408,43 @@ function trimQuotes(s: string) {
|
||||
return s.replace(/^"|"$/g, "");
|
||||
}
|
||||
|
||||
function sortDnsRecordsByType(
|
||||
records: DnsRecord[],
|
||||
order: readonly DnsType[],
|
||||
): DnsRecord[] {
|
||||
const byType: Record<DnsType, DnsRecord[]> = {
|
||||
A: [],
|
||||
AAAA: [],
|
||||
MX: [],
|
||||
TXT: [],
|
||||
NS: [],
|
||||
};
|
||||
for (const r of records) byType[r.type].push(r);
|
||||
const sorted: DnsRecord[] = [];
|
||||
for (const t of order) {
|
||||
sorted.push(...sortDnsRecordsForType(byType[t] as DnsRecord[], t));
|
||||
}
|
||||
return sorted;
|
||||
}
|
||||
|
||||
function sortDnsRecordsForType(arr: DnsRecord[], type: DnsType): DnsRecord[] {
|
||||
if (type === "MX") {
|
||||
arr.sort((a, b) => {
|
||||
const ap = (a.priority ?? Number.MAX_SAFE_INTEGER) as number;
|
||||
const bp = (b.priority ?? Number.MAX_SAFE_INTEGER) as number;
|
||||
if (ap !== bp) return ap - bp;
|
||||
return a.value.localeCompare(b.value);
|
||||
});
|
||||
return arr;
|
||||
}
|
||||
if (type === "TXT" || type === "NS") {
|
||||
arr.sort((a, b) => a.value.localeCompare(b.value));
|
||||
return arr;
|
||||
}
|
||||
// For A/AAAA retain provider order
|
||||
return arr;
|
||||
}
|
||||
|
||||
type DnsJson = {
|
||||
Status: number;
|
||||
Answer?: DnsAnswer[];
|
||||
|
@@ -48,8 +48,8 @@ export async function getOrCreateFaviconBlobUrl(
|
||||
const startedAt = Date.now();
|
||||
console.debug("[favicon] start", { domain, size: DEFAULT_SIZE });
|
||||
|
||||
const indexKey = ns("favicon:url", `${domain}:${DEFAULT_SIZE}`);
|
||||
const lockKey = ns("lock", `favicon:${domain}:${DEFAULT_SIZE}`);
|
||||
const indexKey = ns("favicon", "url", domain, String(DEFAULT_SIZE));
|
||||
const lockKey = ns("lock", "favicon", domain, String(DEFAULT_SIZE));
|
||||
|
||||
// 1) Check Redis index first (supports positive and negative cache)
|
||||
try {
|
||||
|
@@ -6,7 +6,7 @@ export async function probeHeaders(domain: string): Promise<HttpHeader[]> {
|
||||
const lower = domain.toLowerCase();
|
||||
const url = `https://${domain}/`;
|
||||
const key = ns("headers", lower);
|
||||
const lockKey = ns("lock", `headers:${lower}`);
|
||||
const lockKey = ns("lock", "headers", lower);
|
||||
|
||||
console.debug("[headers] start", { domain: lower });
|
||||
const cached = await redis.get<HttpHeader[]>(key);
|
||||
|
@@ -58,12 +58,16 @@ export async function getOrCreateScreenshotBlobUrl(
|
||||
const backoffMaxMs = options?.backoffMaxMs ?? CAPTURE_BACKOFF_MAX_MS_DEFAULT;
|
||||
|
||||
const indexKey = ns(
|
||||
"screenshot:url",
|
||||
`${domain}:${VIEWPORT_WIDTH}x${VIEWPORT_HEIGHT}`,
|
||||
"screenshot",
|
||||
"url",
|
||||
domain,
|
||||
`${VIEWPORT_WIDTH}x${VIEWPORT_HEIGHT}`,
|
||||
);
|
||||
const lockKey = ns(
|
||||
"lock",
|
||||
`screenshot:${domain}:${VIEWPORT_WIDTH}x${VIEWPORT_HEIGHT}`,
|
||||
"screenshot",
|
||||
domain,
|
||||
`${VIEWPORT_WIDTH}x${VIEWPORT_HEIGHT}`,
|
||||
);
|
||||
|
||||
// 1) Check Redis index first
|
||||
|
@@ -60,8 +60,7 @@ function textResponse(text: string, contentType = "text/plain") {
|
||||
describe("getSeo", () => {
|
||||
it("uses cached response when meta exists in cache", async () => {
|
||||
const { ns, redis } = await import("@/lib/redis");
|
||||
const baseKey = ns("seo", "example.com");
|
||||
const metaKey = ns(`${baseKey}:meta`, "v1");
|
||||
const metaKey = ns("seo", "example.com", "meta");
|
||||
await redis.set(metaKey, {
|
||||
meta: null,
|
||||
robots: null,
|
||||
@@ -138,8 +137,7 @@ describe("getSeo", () => {
|
||||
|
||||
it("uses cached robots when present and avoids second fetch", async () => {
|
||||
const { ns, redis } = await import("@/lib/redis");
|
||||
const baseKey = ns("seo", "example.com");
|
||||
const robotsKey = ns(`${baseKey}:robots`, "v1");
|
||||
const robotsKey = ns("seo", "example.com", "robots");
|
||||
await redis.set(robotsKey, {
|
||||
fetched: true,
|
||||
groups: [{ userAgents: ["*"], rules: [{ type: "allow", value: "/" }] }],
|
||||
|
@@ -33,9 +33,8 @@ async function fetchWithTimeout(
|
||||
|
||||
export async function getSeo(domain: string): Promise<SeoResponse> {
|
||||
const lower = domain.toLowerCase();
|
||||
const baseKey = ns("seo", lower);
|
||||
const metaKey = ns(`${baseKey}:meta`, "v1");
|
||||
const robotsKey = ns(`${baseKey}:robots`, "v1");
|
||||
const metaKey = ns("seo", lower, "meta");
|
||||
const robotsKey = ns("seo", lower, "robots");
|
||||
|
||||
console.debug("[seo] start", { domain: lower });
|
||||
const cached = await redis.get<SeoResponse>(metaKey);
|
||||
@@ -178,12 +177,18 @@ async function getOrCreateSocialPreviewImageUrl(
|
||||
const hash = deterministicHash(imageUrl);
|
||||
|
||||
const indexKey = ns(
|
||||
"seo:image-url",
|
||||
`${lower}:${hash}:${SOCIAL_WIDTH}x${SOCIAL_HEIGHT}`,
|
||||
"seo",
|
||||
"image-url",
|
||||
lower,
|
||||
hash,
|
||||
`${SOCIAL_WIDTH}x${SOCIAL_HEIGHT}`,
|
||||
);
|
||||
const lockKey = ns(
|
||||
"lock",
|
||||
`${lower}:seo-image:${hash}:${SOCIAL_WIDTH}x${SOCIAL_HEIGHT}`,
|
||||
"seo-image",
|
||||
lower,
|
||||
hash,
|
||||
`${SOCIAL_WIDTH}x${SOCIAL_HEIGHT}`,
|
||||
);
|
||||
|
||||
// 1) Check Redis index first
|
||||
|
@@ -15,7 +15,7 @@ const __redisImpl = vi.hoisted(() => {
|
||||
const store = new Map<string, unknown>();
|
||||
// simple sorted-set implementation: key -> Map(member -> score)
|
||||
const zsets = new Map<string, Map<string, number>>();
|
||||
const ns = (n: string, id?: string) => `${n}${id ? `:${id}` : ""}`;
|
||||
const ns = (...parts: string[]) => parts.join(":");
|
||||
|
||||
const get = vi.fn(async (key: string) =>
|
||||
store.has(key) ? store.get(key) : null,
|
||||
|
Reference in New Issue
Block a user