Add fetch utility functions for enhanced HTTP request handling

2025-10-18 20:14:25 -04:00 · 2025-10-14 18:43:17 -04:00
parent 368e7bdbe4
commit 43a73a5e27
5 changed files with 132 additions and 137 deletions
--- a/lib/fetch.ts
+++ b/lib/fetch.ts
@@ -0,0 +1,54 @@
+export async function fetchWithTimeout(
+  input: RequestInfo | URL,
+  init: RequestInit = {},
+  opts: { timeoutMs?: number; retries?: number; backoffMs?: number } = {},
+): Promise<Response> {
+  const timeoutMs = opts.timeoutMs ?? 5000;
+  const retries = Math.max(0, opts.retries ?? 0);
+  const backoffMs = Math.max(0, opts.backoffMs ?? 150);
+
+  let lastError: unknown = null;
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+      const res = await fetch(input, { ...init, signal: controller.signal });
+      clearTimeout(timer);
+      return res;
+    } catch (err) {
+      lastError = err;
+      clearTimeout(timer);
+      if (attempt < retries) {
+        await new Promise((r) => setTimeout(r, backoffMs));
+      }
+    }
+  }
+  throw lastError instanceof Error ? lastError : new Error("fetch failed");
+}
+
+export async function headThenGet(
+  url: string,
+  init: RequestInit = {},
+  opts: { timeoutMs?: number } = {},
+): Promise<{ response: Response; usedMethod: "HEAD" | "GET" }> {
+  const timeoutMs = opts.timeoutMs ?? 5000;
+  try {
+    const headRes = await fetchWithTimeout(
+      url,
+      { ...init, method: "HEAD", redirect: "follow" },
+      { timeoutMs },
+    );
+    if (headRes.ok) {
+      return { response: headRes, usedMethod: "HEAD" };
+    }
+  } catch {
+    // fall through to GET
+  }
+
+  const getRes = await fetchWithTimeout(
+    url,
+    { ...init, method: "GET", redirect: "follow" },
+    { timeoutMs },
+  );
+  return { response: getRes, usedMethod: "GET" };
+}
--- a/server/services/dns.ts
+++ b/server/services/dns.ts
@@ -1,6 +1,7 @@
 import { captureServer } from "@/lib/analytics/server";
 import { isCloudflareIpAsync } from "@/lib/cloudflare";
 import { USER_AGENT } from "@/lib/constants";
+import { fetchWithTimeout } from "@/lib/fetch";
 import { acquireLockOrWaitForResult, ns, redis } from "@/lib/redis";
 import {
  type DnsRecord,
@@ -353,9 +354,13 @@ async function resolveTypeWithProvider(
  provider: DohProvider,
 ): Promise<DnsRecord[]> {
  const url = provider.buildUrl(domain, type);
-  const res = await fetchWithTimeout(url, {
-    headers: provider.headers,
-  });
+  const res = await fetchWithTimeout(
+    url,
+    {
+      headers: provider.headers,
+    },
+    { timeoutMs: 2000, retries: 1, backoffMs: 150 },
+  );
  if (!res.ok) throw new Error(`DoH failed: ${provider.key} ${res.status}`);
  const json = (await res.json()) as DnsJson;
  const ans = json.Answer ?? [];
@@ -366,19 +371,25 @@ async function resolveTypeWithProvider(
  return sortDnsRecordsForType(records, type);
 }

-async function normalizeAnswer(
+function normalizeAnswer(
  _domain: string,
  type: DnsType,
  a: DnsAnswer,
-): Promise<DnsRecord | undefined> {
+): Promise<DnsRecord | undefined> | DnsRecord | undefined {
  const name = trimDot(a.name);
  const ttl = a.TTL;
  switch (type) {
    case "A":
    case "AAAA": {
      const value = trimDot(a.data);
-      const isCloudflare = await isCloudflareIpAsync(value);
-      return { type, name, value, ttl, isCloudflare };
+      const isCloudflarePromise = isCloudflareIpAsync(value);
+      return isCloudflarePromise.then((isCloudflare) => ({
+        type,
+        name,
+        value,
+        ttl,
+        isCloudflare,
+      }));
    }
    case "NS": {
      return { type, name, value: trimDot(a.data), ttl };
@@ -467,29 +478,3 @@ function providerOrderForLookup(_domain: string): DohProvider[] {
  }
  return providers;
 }
-
-async function fetchWithTimeout(
-  input: URL | string,
-  init?: RequestInit,
-  timeoutMs: number = 2000,
-): Promise<Response> {
-  // Up to two attempts with independent timeouts
-  let lastError: unknown = null;
-  for (let attempt = 0; attempt < 2; attempt++) {
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), timeoutMs);
-    try {
-      const res = await fetch(input, { ...init, signal: controller.signal });
-      clearTimeout(timer);
-      return res;
-    } catch (err) {
-      lastError = err;
-      clearTimeout(timer);
-      if (attempt === 0) {
-        // small backoff before retry
-        await new Promise((r) => setTimeout(r, 150));
-      }
-    }
-  }
-  throw lastError instanceof Error ? lastError : new Error("fetch failed");
-}
--- a/server/services/favicon.ts
+++ b/server/services/favicon.ts
@@ -1,5 +1,6 @@
 import { getOrCreateCachedAsset } from "@/lib/cache/cached-asset";
 import { FAVICON_TTL_SECONDS, USER_AGENT } from "@/lib/constants";
+import { fetchWithTimeout } from "@/lib/fetch";
 import { convertBufferToImageCover } from "@/lib/image";
 import { ns } from "@/lib/redis";
 import { uploadImage } from "@/lib/storage";
@@ -7,28 +8,6 @@ import { uploadImage } from "@/lib/storage";
 const DEFAULT_SIZE = 32;
 const REQUEST_TIMEOUT_MS = 1500; // per each method

-async function fetchWithTimeout(
-  url: string,
-  init?: RequestInit,
-): Promise<Response> {
-  const controller = new AbortController();
-  const t = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
-  try {
-    const res = await fetch(url, {
-      redirect: "follow",
-      headers: {
-        Accept: "image/avif,image/webp,image/png,image/*;q=0.9,*/*;q=0.8",
-        "User-Agent": USER_AGENT,
-      },
-      signal: controller.signal,
-      ...init,
-    });
-    return res;
-  } finally {
-    clearTimeout(t);
-  }
-}
-
 function buildSources(domain: string): string[] {
  const enc = encodeURIComponent(domain);
  return [
@@ -57,7 +36,18 @@ export async function getOrCreateFaviconBlobUrl(
      const sources = buildSources(domain);
      for (const src of sources) {
        try {
-          const res = await fetchWithTimeout(src);
+          const res = await fetchWithTimeout(
+            src,
+            {
+              redirect: "follow",
+              headers: {
+                Accept:
+                  "image/avif,image/webp,image/png,image/*;q=0.9,*/*;q=0.8",
+                "User-Agent": USER_AGENT,
+              },
+            },
+            { timeoutMs: REQUEST_TIMEOUT_MS },
+          );
          if (!res.ok) continue;
          const contentType = res.headers.get("content-type");
          const ab = await res.arrayBuffer();
--- a/server/services/headers.ts
+++ b/server/services/headers.ts
@@ -1,4 +1,5 @@
 import { captureServer } from "@/lib/analytics/server";
+import { headThenGet } from "@/lib/fetch";
 import { acquireLockOrWaitForResult, ns, redis } from "@/lib/redis";
 import type { HttpHeader } from "@/lib/schemas";

@@ -48,42 +49,11 @@ export async function probeHeaders(domain: string): Promise<HttpHeader[]> {

  const REQUEST_TIMEOUT_MS = 5000;
  try {
-    // Try HEAD first with timeout
-    const headController = new AbortController();
-    const headTimer = setTimeout(
-      () => headController.abort(),
-      REQUEST_TIMEOUT_MS,
+    const { response: final, usedMethod } = await headThenGet(
+      url,
+      {},
+      { timeoutMs: REQUEST_TIMEOUT_MS },
    );
-    let res: Response | null = null;
-    try {
-      res = await fetch(url, {
-        method: "HEAD",
-        redirect: "follow" as RequestRedirect,
-        signal: headController.signal,
-      });
-    } finally {
-      clearTimeout(headTimer);
-    }
-
-    let final: Response | null = res;
-    if (!res || !res.ok) {
-      const getController = new AbortController();
-      const getTimer = setTimeout(
-        () => getController.abort(),
-        REQUEST_TIMEOUT_MS,
-      );
-      try {
-        final = await fetch(url, {
-          method: "GET",
-          redirect: "follow" as RequestRedirect,
-          signal: getController.signal,
-        });
-      } finally {
-        clearTimeout(getTimer);
-      }
-    }
-
-    if (!final) throw new Error("No response");

    const headers: HttpHeader[] = [];
    final.headers.forEach((value, name) => {
@@ -94,7 +64,7 @@ export async function probeHeaders(domain: string): Promise<HttpHeader[]> {
    await captureServer("headers_probe", {
      domain: lower,
      status: final.status,
-      used_method: res?.ok ? "HEAD" : "GET",
+      used_method: usedMethod,
      final_url: final.url,
      lock_acquired: acquiredLock,
      lock_waited_ms: acquiredLock ? 0 : Date.now() - lockWaitStart,
--- a/server/services/seo.ts
+++ b/server/services/seo.ts
@@ -1,5 +1,6 @@
 import { captureServer } from "@/lib/analytics/server";
 import { SOCIAL_PREVIEW_TTL_SECONDS, USER_AGENT } from "@/lib/constants";
+import { fetchWithTimeout } from "@/lib/fetch";
 import { optimizeImageCover } from "@/lib/image";
 import { acquireLockOrWaitForResult, ns, redis } from "@/lib/redis";
 import type { SeoResponse } from "@/lib/schemas";
@@ -11,22 +12,6 @@ const ROBOTS_TTL_SECONDS = 12 * 60 * 60; // 12 hours
 const SOCIAL_WIDTH = 1200;
 const SOCIAL_HEIGHT = 630;

-async function fetchWithTimeout(
-  url: string,
-  opts: RequestInit & { timeoutMs?: number },
-): Promise<Response> {
-  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), opts.timeoutMs ?? 10000);
-  try {
-    const res = await fetch(url, { ...opts, signal: controller.signal });
-    clearTimeout(timer);
-    return res;
-  } catch (err) {
-    clearTimeout(timer);
-    throw err;
-  }
-}
-
 export async function getSeo(domain: string): Promise<SeoResponse> {
  const lower = domain.toLowerCase();
  const metaKey = ns("seo", lower, "meta");
@@ -43,7 +28,7 @@ export async function getSeo(domain: string): Promise<SeoResponse> {
    return cached;
  }

-  let finalUrl: string | null = `https://${lower}/`;
+  let finalUrl: string = `https://${lower}/`;
  let status: number | null = null;
  let htmlError: string | undefined;
  let robotsError: string | undefined;
@@ -53,17 +38,20 @@ export async function getSeo(domain: string): Promise<SeoResponse> {

  // HTML fetch
  try {
-    const res = await fetchWithTimeout(finalUrl, {
-      method: "GET",
-      redirect: "follow",
-      timeoutMs: 10000,
-      headers: {
-        Accept:
-          "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-        "Accept-Language": "en",
-        "User-Agent": USER_AGENT,
+    const res = await fetchWithTimeout(
+      finalUrl,
+      {
+        method: "GET",
+        redirect: "follow",
+        headers: {
+          Accept:
+            "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+          "Accept-Language": "en",
+          "User-Agent": USER_AGENT,
+        },
      },
-    });
+      { timeoutMs: 10000 },
+    );
    status = res.status;
    finalUrl = res.url;
    const contentType = res.headers.get("content-type") ?? "";
@@ -87,11 +75,14 @@ export async function getSeo(domain: string): Promise<SeoResponse> {
      robots = cachedRobots;
    } else {
      const robotsUrl = `https://${lower}/robots.txt`;
-      const res = await fetchWithTimeout(robotsUrl, {
-        method: "GET",
-        timeoutMs: 8000,
-        headers: { Accept: "text/plain", "User-Agent": USER_AGENT },
-      });
+      const res = await fetchWithTimeout(
+        robotsUrl,
+        {
+          method: "GET",
+          headers: { Accept: "text/plain", "User-Agent": USER_AGENT },
+        },
+        { timeoutMs: 8000 },
+      );
      if (res.ok) {
        const ct = res.headers.get("content-type") ?? "";
        if (ct.includes("text/plain") || ct.includes("text/")) {
@@ -109,9 +100,7 @@ export async function getSeo(domain: string): Promise<SeoResponse> {
    robotsError = String(err);
  }

-  const preview = meta
-    ? selectPreview(meta, finalUrl ?? `https://${lower}/`)
-    : null;
+  const preview = meta ? selectPreview(meta, finalUrl) : null;

  // If a social image is present, store a cached copy via UploadThing for privacy
  if (preview?.image) {
@@ -196,7 +185,11 @@ async function getOrCreateSocialPreviewImageUrl(
  // 1) Check Redis index first
  try {
    const raw = (await redis.get(indexKey)) as { url?: unknown } | null;
-    if (raw && typeof raw === "object" && typeof raw.url === "string") {
+    if (
+      raw &&
+      typeof raw === "object" &&
+      typeof (raw as { url?: unknown }).url === "string"
+    ) {
      await captureServer("seo_image", {
        domain: lower,
        width: SOCIAL_WIDTH,
@@ -206,7 +199,7 @@ async function getOrCreateSocialPreviewImageUrl(
        outcome: "ok",
        cache: "hit",
      });
-      return { url: raw.url };
+      return { url: (raw as { url: string }).url };
    }
  } catch {
    // ignore and continue
@@ -237,15 +230,18 @@ async function getOrCreateSocialPreviewImageUrl(

  // 3) We acquired the lock - fetch, process, upload
  try {
-    const res = await fetchWithTimeout(imageUrl, {
-      method: "GET",
-      timeoutMs: 8000,
-      headers: {
-        Accept:
-          "image/avif,image/webp,image/png,image/jpeg,image/*;q=0.9,*/*;q=0.8",
-        "User-Agent": USER_AGENT,
+    const res = await fetchWithTimeout(
+      imageUrl,
+      {
+        method: "GET",
+        headers: {
+          Accept:
+            "image/avif,image/webp,image/png,image/jpeg,image/*;q=0.9,*/*;q=0.8",
+          "User-Agent": USER_AGENT,
+        },
      },
-    });
+      { timeoutMs: 8000 },
+    );

    if (!res.ok) return { url: null };
    const contentType = res.headers.get("content-type") ?? "";