From 450f1ebd5de94ee8fa42547312ad2a27b90a13df Mon Sep 17 00:00:00 2001 From: Jake Jarvis Date: Sun, 21 Sep 2025 15:02:58 -0400 Subject: [PATCH] Refactor favicon fetching to use blob storage and improve API structure (#7) --- .env.example | 6 + .gitignore | 3 +- app/api/favicon/route.ts | 94 ---------------- app/globals.css | 47 +------- components/domain/favicon.tsx | 28 +++-- components/domain/provider-value.tsx | 4 +- .../domain/sections/hosting-email-section.tsx | 18 ++- .../domain/sections/registration-section.tsx | 6 +- lib/blob.ts | 59 ++++++++++ next.config.ts | 8 ++ package.json | 1 + pnpm-lock.yaml | 59 ++++++++++ server/routers/domain.ts | 10 +- server/services/favicon.ts | 105 ++++++------------ server/services/headers.ts | 8 +- server/services/hosting.ts | 9 +- 16 files changed, 225 insertions(+), 240 deletions(-) delete mode 100644 app/api/favicon/route.ts create mode 100644 lib/blob.ts diff --git a/.env.example b/.env.example index 010e8a0..5f7aab7 100644 --- a/.env.example +++ b/.env.example @@ -5,3 +5,9 @@ NEXT_PUBLIC_POSTHOG_HOST= # Upstash credentials via native Vercel integration KV_REST_API_TOKEN= KV_REST_API_URL= + +# Vercel Blob (add integration on Vercel; token for local/dev) +BLOB_READ_WRITE_TOKEN= + +# Secret used to derive unpredictable blob paths for favicons +FAVICON_BLOB_SIGNING_SECRET= diff --git a/.gitignore b/.gitignore index 7b8da95..23bad85 100644 --- a/.gitignore +++ b/.gitignore @@ -31,8 +31,7 @@ yarn-error.log* .pnpm-debug.log* # env files (can opt-in for committing if needed) -.env* -!.env.example +.env*.local # vercel .vercel diff --git a/app/api/favicon/route.ts b/app/api/favicon/route.ts deleted file mode 100644 index 7d6403c..0000000 --- a/app/api/favicon/route.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { createHash } from "node:crypto"; -import { NextResponse } from "next/server"; -import { toRegistrableDomain } from "@/lib/domain-server"; -import { captureServer } from "@/server/analytics/posthog"; -import { - clampFaviconSize, - getFaviconPngForDomain, -} from "@/server/services/favicon"; - -export const runtime = "nodejs"; -export const maxDuration = 10; - -export async function GET(req: Request) { - const url = new URL(req.url); - const raw = url.searchParams.get("domain") ?? url.searchParams.get("d"); - const sizeParam = Number( - url.searchParams.get("size") ?? url.searchParams.get("sz") ?? "", - ); - const size = clampFaviconSize(sizeParam); - - // Attempt to associate events with user via PostHog cookie - let distinctId: string | undefined; - try { - const key = process.env.NEXT_PUBLIC_POSTHOG_KEY; - if (key) { - const cookieName = `ph_${key}_posthog`; - const cookieStr = req.headers.get("cookie") ?? ""; - const m = cookieStr.match(new RegExp(`${cookieName}=([^;]+)`)); - if (m) { - const parsed = JSON.parse(decodeURIComponent(m[1])); - if (parsed && typeof parsed.distinct_id === "string") { - distinctId = parsed.distinct_id; - } - } - } - } catch { - // ignore - } - - const registrable = raw ? toRegistrableDomain(raw) : null; - if (!registrable) { - await captureServer( - "favicon_fetch", - { - domain: raw ?? "", - valid: false, - reason: "invalid_domain", - }, - distinctId, - ); - - return NextResponse.json({ error: "Invalid domain" }, { status: 400 }); - } - - const png = await getFaviconPngForDomain(registrable, size, { distinctId }); - if (png) { - const etag = (() => { - const hash = createHash("sha256").update(png).digest("base64url"); - return `"${hash}"`; - })(); - - const ifNoneMatch = req.headers.get("if-none-match"); - - if (ifNoneMatch) { - const candidates = ifNoneMatch.split(",").map((s) => s.trim()); - const matches = candidates.some((c) => c === etag || c === `W/${etag}`); - - if (matches) { - return new NextResponse(null, { - status: 304, - headers: { - ETag: etag, - "Cache-Control": "public, max-age=604800, s-maxage=604800", - }, - }); - } - } - - return new NextResponse(png as unknown as BodyInit, { - status: 200, - headers: { - "Content-Type": "image/png", - // Cache for a week; allow CDN to keep for a week - "Cache-Control": "public, max-age=604800, s-maxage=604800", - "Content-Length": String(png.length), - ETag: etag, - }, - }); - } - - return new NextResponse(null, { - status: 404, - }); -} diff --git a/app/globals.css b/app/globals.css index 4530c65..8e80d9e 100644 --- a/app/globals.css +++ b/app/globals.css @@ -4,23 +4,10 @@ @custom-variant dark (&:is(.dark *)); @theme inline { - --color-background: var(--background); - --color-foreground: var(--foreground); --font-sans: var(--font-geist-sans); --font-mono: var(--font-geist-mono); - --color-sidebar-ring: var(--sidebar-ring); - --color-sidebar-border: var(--sidebar-border); - --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); - --color-sidebar-accent: var(--sidebar-accent); - --color-sidebar-primary-foreground: var(--sidebar-primary-foreground); - --color-sidebar-primary: var(--sidebar-primary); - --color-sidebar-foreground: var(--sidebar-foreground); - --color-sidebar: var(--sidebar); - --color-chart-5: var(--chart-5); - --color-chart-4: var(--chart-4); - --color-chart-3: var(--chart-3); - --color-chart-2: var(--chart-2); - --color-chart-1: var(--chart-1); + --color-background: var(--background); + --color-foreground: var(--foreground); --color-ring: var(--ring); --color-input: var(--input); --color-border: var(--border); @@ -63,11 +50,7 @@ --border: oklch(0.922 0 0); --input: oklch(0.922 0 0); --ring: oklch(0.708 0 0); - --chart-1: oklch(0.646 0.222 41.116); - --chart-2: oklch(0.6 0.118 184.704); - --chart-3: oklch(0.398 0.07 227.392); - --chart-4: oklch(0.828 0.189 84.429); - --chart-5: oklch(0.769 0.188 70.08); + /* Section accent glow (light) */ --accent-blue-glow: radial-gradient( closest-side, @@ -94,6 +77,7 @@ oklch(0.96 0.16 350), transparent ); + /* DNS badge accent tokens */ --dns-a: oklch(0.62 0.2 255); --dns-aaaa: oklch(0.6 0.19 275); @@ -101,14 +85,6 @@ --dns-cname: oklch(0.68 0.19 310); --dns-txt: oklch(0.78 0.17 70); --dns-ns: oklch(0.64 0.16 195); - --sidebar: hsl(0 0% 98%); - --sidebar-foreground: hsl(240 5.3% 26.1%); - --sidebar-primary: hsl(240 5.9% 10%); - --sidebar-primary-foreground: hsl(0 0% 98%); - --sidebar-accent: hsl(240 4.8% 95.9%); - --sidebar-accent-foreground: hsl(240 5.9% 10%); - --sidebar-border: hsl(220 13% 91%); - --sidebar-ring: hsl(217.2 91.2% 59.8%); } .dark { @@ -130,11 +106,7 @@ --border: oklch(1 0 0 / 10%); --input: oklch(1 0 0 / 15%); --ring: oklch(0.556 0 0); - --chart-1: oklch(0.488 0.243 264.376); - --chart-2: oklch(0.696 0.17 162.48); - --chart-3: oklch(0.769 0.188 70.08); - --chart-4: oklch(0.627 0.265 303.9); - --chart-5: oklch(0.645 0.246 16.439); + /* Section accent glow (dark) */ --accent-blue-glow: radial-gradient( closest-side, @@ -161,6 +133,7 @@ oklch(0.82 0.16 350), transparent ); + /* DNS badge accent tokens (dark) */ --dns-a: oklch(0.72 0.16 255); --dns-aaaa: oklch(0.7 0.16 275); @@ -168,14 +141,6 @@ --dns-cname: oklch(0.8 0.16 310); --dns-txt: oklch(0.86 0.14 70); --dns-ns: oklch(0.78 0.12 195); - --sidebar: hsl(240 5.9% 10%); - --sidebar-foreground: hsl(240 4.8% 95.9%); - --sidebar-primary: hsl(224.3 76.3% 48%); - --sidebar-primary-foreground: hsl(0 0% 100%); - --sidebar-accent: hsl(240 3.7% 15.9%); - --sidebar-accent-foreground: hsl(240 4.8% 95.9%); - --sidebar-border: hsl(240 3.7% 15.9%); - --sidebar-ring: hsl(217.2 91.2% 59.8%); } @layer base { diff --git a/components/domain/favicon.tsx b/components/domain/favicon.tsx index 35d4e24..f440009 100644 --- a/components/domain/favicon.tsx +++ b/components/domain/favicon.tsx @@ -2,7 +2,8 @@ import { Globe } from "lucide-react"; import Image from "next/image"; -import * as React from "react"; +import { Skeleton } from "@/components/ui/skeleton"; +import { trpc } from "@/lib/trpc/client"; import { cn } from "@/lib/utils"; export function Favicon({ @@ -14,14 +15,19 @@ export function Favicon({ size?: number; className?: string; }) { - const apiUrl = React.useMemo( - () => `/api/favicon?domain=${encodeURIComponent(domain)}`, - [domain], - ); - const [failedUrl, setFailedUrl] = React.useState(null); - const failed = failedUrl === apiUrl; + const { data, isLoading } = trpc.domain.faviconUrl.useQuery({ domain }); + const url = data?.url ?? null; - if (failed) { + if (isLoading) { + return ( + + ); + } + + if (!isLoading && !url) { return ( setFailedUrl(apiUrl)} /> ); } diff --git a/components/domain/provider-value.tsx b/components/domain/provider-value.tsx index 93dd659..44e0393 100644 --- a/components/domain/provider-value.tsx +++ b/components/domain/provider-value.tsx @@ -11,7 +11,9 @@ export function ProviderValue({ }) { return (
- {iconDomain ? : null} + {iconDomain ? ( + + ) : null} {name}
); diff --git a/components/domain/sections/hosting-email-section.tsx b/components/domain/sections/hosting-email-section.tsx index bdac9ed..46861e4 100644 --- a/components/domain/sections/hosting-email-section.tsx +++ b/components/domain/sections/hosting-email-section.tsx @@ -47,7 +47,11 @@ export function HostingEmailSection({ value={data.dnsProvider.name} leading={ data.dnsProvider.iconDomain ? ( - + ) : undefined } /> @@ -56,7 +60,11 @@ export function HostingEmailSection({ value={data.hostingProvider.name} leading={ data.hostingProvider.iconDomain ? ( - + ) : undefined } /> @@ -65,7 +73,11 @@ export function HostingEmailSection({ value={data.emailProvider.name} leading={ data.emailProvider.iconDomain ? ( - + ) : undefined } /> diff --git a/components/domain/sections/registration-section.tsx b/components/domain/sections/registration-section.tsx index 3101ad3..5ae6f27 100644 --- a/components/domain/sections/registration-section.tsx +++ b/components/domain/sections/registration-section.tsx @@ -46,7 +46,11 @@ export function RegistrationSection({ value={data.registrar.name} leading={ data.registrar.iconDomain ? ( - + ) : undefined } suffix={ diff --git a/lib/blob.ts b/lib/blob.ts new file mode 100644 index 0000000..976dbac --- /dev/null +++ b/lib/blob.ts @@ -0,0 +1,59 @@ +import "server-only"; + +import { createHmac } from "node:crypto"; +import { head, put } from "@vercel/blob"; + +const ONE_WEEK_SECONDS = 7 * 24 * 60 * 60; + +function getSigningSecret(): string { + if ( + process.env.NODE_ENV === "production" && + !process.env.FAVICON_BLOB_SIGNING_SECRET + ) { + throw new Error("FAVICON_BLOB_SIGNING_SECRET required in production"); + } + + const secret = + process.env.FAVICON_BLOB_SIGNING_SECRET || + process.env.BLOB_READ_WRITE_TOKEN || + "dev-favicon-secret"; + return secret; +} + +export function computeFaviconBlobPath(domain: string, size: number): string { + const input = `${domain}:${size}`; + const secret = getSigningSecret(); + const digest = createHmac("sha256", secret).update(input).digest("hex"); + // Avoid leaking domain; path is deterministic but unpredictable without secret + return `favicons/${digest}/${size}.png`; +} + +export async function headFaviconBlob( + domain: string, + size: number, +): Promise { + const pathname = computeFaviconBlobPath(domain, size); + try { + const res = await head(pathname, { + token: process.env.BLOB_READ_WRITE_TOKEN, + }); + return res?.url ?? null; + } catch { + return null; + } +} + +export async function putFaviconBlob( + domain: string, + size: number, + png: Buffer, +): Promise { + const pathname = computeFaviconBlobPath(domain, size); + const res = await put(pathname, png, { + access: "public", + contentType: "image/png", + cacheControlMaxAge: ONE_WEEK_SECONDS, + token: process.env.BLOB_READ_WRITE_TOKEN, + }); + return res.url; +} diff --git a/next.config.ts b/next.config.ts index 95e7289..ef09c17 100644 --- a/next.config.ts +++ b/next.config.ts @@ -15,6 +15,14 @@ const nextConfig: NextConfig = { dynamic: 0, // disable client-side router cache for dynamic pages }, }, + images: { + remotePatterns: [ + { + protocol: "https", + hostname: "8ubelavjx21gjm2t.public.blob.vercel-storage.com", + }, + ], + }, async rewrites() { return [ { diff --git a/package.json b/package.json index 3215198..be74612 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "@trpc/server": "^11.5.1", "@upstash/redis": "^1.35.4", "@vercel/analytics": "^1.5.0", + "@vercel/blob": "^2.0.0", "@vercel/functions": "^3.1.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8467adc..ef16fee 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,6 +29,9 @@ importers: '@vercel/analytics': specifier: ^1.5.0 version: 1.5.0(next@15.6.0-canary.20(babel-plugin-react-compiler@19.1.0-rc.3)(react-dom@19.1.1(react@19.1.1))(react@19.1.1))(react@19.1.1) + '@vercel/blob': + specifier: ^2.0.0 + version: 2.0.0 '@vercel/functions': specifier: ^3.1.0 version: 3.1.0 @@ -222,6 +225,10 @@ packages: '@emnapi/runtime@1.5.0': resolution: {integrity: sha512-97/BJ3iXHww3djw6hYIfErCZFee7qCtrneuLa20UXFCOTCfBM2cvQHjWJ2EG0s0MtdNwInarqCTz35i4wWXHsQ==} + '@fastify/busboy@2.1.1': + resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==} + engines: {node: '>=14'} + '@floating-ui/core@1.7.3': resolution: {integrity: sha512-sGnvb5dmrJaKEZ+LDIpguvdX3bDlEllmv4/ClQ9awcmCZrlx5jQyyMWFM5kBI+EyNOCDDiKk8il0zeuX3Zlg/w==} @@ -1332,6 +1339,10 @@ packages: vue-router: optional: true + '@vercel/blob@2.0.0': + resolution: {integrity: sha512-oAj7Pdy83YKSwIaMFoM7zFeLYWRc+qUpW3PiDSblxQMnGFb43qs4bmfq7dr/+JIfwhs6PTwe1o2YBwKhyjWxXw==} + engines: {node: '>=20.0.0'} + '@vercel/functions@3.1.0': resolution: {integrity: sha512-V+p8dO+sg1VjiJJUO5rYPp1KG17SzDcR74OWwW7Euyde6L8U5wuTMe9QfEOfLTiWPUPzN1MXZvLcYxqSYhKc4Q==} engines: {node: '>= 20'} @@ -1356,6 +1367,9 @@ packages: resolution: {integrity: sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==} engines: {node: '>=10'} + async-retry@1.3.3: + resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} + babel-plugin-react-compiler@19.1.0-rc.3: resolution: {integrity: sha512-mjRn69WuTz4adL0bXGx8Rsyk1086zFJeKmes6aK0xPuK3aaXmDJdLHqwKKMrpm6KAI1MCoUK72d2VeqQbu8YIA==} @@ -1484,6 +1498,13 @@ packages: resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==} engines: {node: '>= 10'} + is-buffer@2.0.5: + resolution: {integrity: sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==} + engines: {node: '>=4'} + + is-node-process@1.2.0: + resolution: {integrity: sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==} + is-stream@4.0.1: resolution: {integrity: sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==} engines: {node: '>=18'} @@ -1767,6 +1788,10 @@ packages: regenerator-runtime@0.13.11: resolution: {integrity: sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==} + retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} @@ -1837,6 +1862,10 @@ packages: resolution: {integrity: sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==} engines: {node: '>=18'} + throttleit@2.1.0: + resolution: {integrity: sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==} + engines: {node: '>=18'} + tinycolor2@1.6.0: resolution: {integrity: sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==} @@ -1886,6 +1915,10 @@ packages: undici-types@7.12.0: resolution: {integrity: sha512-goOacqME2GYyOZZfb5Lgtu+1IDmAlAEu5xnD3+xTzS10hT0vzpf0SPjkXwAw9Jm+4n/mQGDP3LO8CPbYROeBfQ==} + undici@5.29.0: + resolution: {integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==} + engines: {node: '>=14.0'} + use-callback-ref@1.3.3: resolution: {integrity: sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==} engines: {node: '>=10'} @@ -1996,6 +2029,8 @@ snapshots: tslib: 2.8.1 optional: true + '@fastify/busboy@2.1.1': {} + '@floating-ui/core@1.7.3': dependencies: '@floating-ui/utils': 0.2.10 @@ -3076,6 +3111,14 @@ snapshots: next: 15.6.0-canary.20(babel-plugin-react-compiler@19.1.0-rc.3)(react-dom@19.1.1(react@19.1.1))(react@19.1.1) react: 19.1.1 + '@vercel/blob@2.0.0': + dependencies: + async-retry: 1.3.3 + is-buffer: 2.0.5 + is-node-process: 1.2.0 + throttleit: 2.1.0 + undici: 5.29.0 + '@vercel/functions@3.1.0': dependencies: '@vercel/oidc': 3.0.0 @@ -3095,6 +3138,10 @@ snapshots: dependencies: tslib: 2.8.1 + async-retry@1.3.3: + dependencies: + retry: 0.13.1 + babel-plugin-react-compiler@19.1.0-rc.3: dependencies: '@babel/types': 7.28.4 @@ -3221,6 +3268,10 @@ snapshots: ipaddr.js@2.2.0: {} + is-buffer@2.0.5: {} + + is-node-process@1.2.0: {} + is-stream@4.0.1: {} is-what@4.1.16: {} @@ -3504,6 +3555,8 @@ snapshots: regenerator-runtime@0.13.11: {} + retry@0.13.1: {} + safe-buffer@5.2.1: {} scheduler@0.26.0: {} @@ -3586,6 +3639,8 @@ snapshots: mkdirp: 3.0.1 yallist: 5.0.0 + throttleit@2.1.0: {} + tinycolor2@1.6.0: {} tldts-core@7.0.15: {} @@ -3623,6 +3678,10 @@ snapshots: undici-types@7.12.0: {} + undici@5.29.0: + dependencies: + '@fastify/busboy': 2.1.1 + use-callback-ref@1.3.3(@types/react@19.1.13)(react@19.1.1): dependencies: react: 19.1.1 diff --git a/server/routers/domain.ts b/server/routers/domain.ts index c4d16ef..05e9a33 100644 --- a/server/routers/domain.ts +++ b/server/routers/domain.ts @@ -1,4 +1,5 @@ import { resolveAll } from "../services/dns"; +import { getOrCreateFaviconBlobUrl } from "../services/favicon"; import { probeHeaders } from "../services/headers"; import { detectHosting } from "../services/hosting"; import { fetchWhois } from "../services/rdap"; @@ -14,8 +15,9 @@ export const domainRouter = router({ getCertificates, "Certificate fetch failed", ), - headers: createDomainProcedure(async (domain: string) => { - const res = await probeHeaders(domain); - return res.headers; - }, "Header probe failed"), + headers: createDomainProcedure(probeHeaders, "Header probe failed"), + faviconUrl: createDomainProcedure( + getOrCreateFaviconBlobUrl, + "Favicon fetch failed", + ), }); diff --git a/server/services/favicon.ts b/server/services/favicon.ts index 05f4ac9..729e8e1 100644 --- a/server/services/favicon.ts +++ b/server/services/favicon.ts @@ -1,20 +1,11 @@ import sharp from "sharp"; -import { cacheGet, cacheSet, ns } from "@/lib/redis"; +import { headFaviconBlob, putFaviconBlob } from "@/lib/blob"; import { captureServer } from "@/server/analytics/posthog"; const DEFAULT_SIZE = 32; -const MIN_SIZE = 16; -const MAX_SIZE = 256; const REQUEST_TIMEOUT_MS = 4000; -const CACHE_TTL_SECONDS = 7 * 24 * 60 * 60; // 7 days -const NEGATIVE_CACHE_TTL_SECONDS = 6 * 60 * 60; // 6 hours for not-found -// Use a single Redis key for both positive and negative results; this sentinel marks a negative entry -const NEGATIVE_CACHE_SENTINEL = "!" as const; -export function clampFaviconSize(value: number | null | undefined): number { - if (!value || Number.isNaN(value)) return DEFAULT_SIZE; - return Math.max(MIN_SIZE, Math.min(MAX_SIZE, Math.round(value))); -} +// Legacy Redis-based caching removed; Blob is now the canonical store function isIcoBuffer(buf: Buffer): boolean { return ( @@ -108,63 +99,47 @@ async function convertToPng( return null; } -function buildSources(domain: string, size: number): string[] { +function buildSources(domain: string): string[] { const enc = encodeURIComponent(domain); return [ `https://icons.duckduckgo.com/ip3/${enc}.ico`, - `https://www.google.com/s2/favicons?domain=${enc}&sz=${size}`, + `https://www.google.com/s2/favicons?domain=${enc}&sz=${DEFAULT_SIZE}`, `https://${domain}/favicon.ico`, `http://${domain}/favicon.ico`, ]; } -export async function getFaviconPngForDomain( - domain: string, - size: number, - opts?: { distinctId?: string }, -): Promise { - const cacheKey = ns("favicon", `${domain}:${size}`); - const sources = buildSources(domain, size); - const startedAt = Date.now(); +// Legacy getFaviconPngForDomain removed - // Try cache first +export async function getOrCreateFaviconBlobUrl( + domain: string, + opts?: { distinctId?: string }, +): Promise<{ url: string | null }> { + const startedAt = Date.now(); + // 1) Check blob first try { - const cachedValue = await cacheGet(cacheKey); - if (cachedValue) { - if (cachedValue === NEGATIVE_CACHE_SENTINEL) { - await captureServer( - "favicon_fetch", - { - domain, - size, - source: "cache_redis", - duration_ms: Date.now() - startedAt, - outcome: "not_found", - cache: "hit_negative", - }, - opts?.distinctId, - ); - return null; - } - const pngFromCache = Buffer.from(cachedValue, "base64"); + const existing = await headFaviconBlob(domain, DEFAULT_SIZE); + if (existing) { await captureServer( "favicon_fetch", { domain, - size, - source: "cache_redis", + size: DEFAULT_SIZE, + source: "blob", duration_ms: Date.now() - startedAt, outcome: "ok", - cache: "hit", + cache: "hit_blob", }, opts?.distinctId, ); - return pngFromCache; + return { url: existing }; } } catch { - // ignore cache errors and fall through + // ignore and proceed to fetch } + // 2) Fetch/convert via existing pipeline, then upload + const sources = buildSources(domain); for (const src of sources) { try { const res = await fetchWithTimeout(src); @@ -173,68 +148,50 @@ export async function getFaviconPngForDomain( const ab = await res.arrayBuffer(); const buf = Buffer.from(ab); - const png = await convertToPng(buf, contentType, size); + const png = await convertToPng(buf, contentType, DEFAULT_SIZE); if (!png) continue; const source = (() => { if (src.includes("icons.duckduckgo.com")) return "duckduckgo"; - if (src.includes("www.google.com/s2/favicons")) return "google_s2"; + if (src.includes("www.google.com/s2/favicons")) return "google"; if (src.startsWith("https://")) return "direct_https"; if (src.startsWith("http://")) return "direct_http"; return "unknown"; })(); - // Attempt to store in cache (base64) for subsequent hits - try { - await cacheSet( - cacheKey, - png.toString("base64"), - CACHE_TTL_SECONDS, - ); - } catch { - // ignore cache errors - } + const url = await putFaviconBlob(domain, DEFAULT_SIZE, png); await captureServer( "favicon_fetch", { domain, - size, + size: DEFAULT_SIZE, source, upstream_status: res.status, upstream_content_type: contentType ?? null, duration_ms: Date.now() - startedAt, outcome: "ok", - cache: "store", + cache: "store_blob", }, opts?.distinctId, ); - return png; + return { url }; } catch { - // Try next source + // try next source } } - // Store negative cache using the same key and capture analytics - try { - await cacheSet( - cacheKey, - NEGATIVE_CACHE_SENTINEL, - NEGATIVE_CACHE_TTL_SECONDS, - ); - } catch { - // ignore cache errors - } + await captureServer( "favicon_fetch", { domain, - size, + size: DEFAULT_SIZE, duration_ms: Date.now() - startedAt, outcome: "not_found", - cache: "store_negative", + cache: "miss", }, opts?.distinctId, ); - return null; + return { url: null }; } diff --git a/server/services/headers.ts b/server/services/headers.ts index 87dda9c..ac61ad5 100644 --- a/server/services/headers.ts +++ b/server/services/headers.ts @@ -3,7 +3,7 @@ import { captureServer } from "@/server/analytics/posthog"; export type HttpHeader = { name: string; value: string }; -export async function probeHeaders(domain: string) { +export async function probeHeaders(domain: string): Promise { const key = ns("headers", domain.toLowerCase()); return await getOrSet(key, 10 * 60, async () => { const url = `https://${domain}/`; @@ -28,11 +28,7 @@ export async function probeHeaders(domain: string) { used_method: res.ok ? "HEAD" : "GET", final_url: final.url, }); - return { - url: final.url, - status: final.status, - headers: normalize(headers), - }; + return normalize(headers); }); } diff --git a/server/services/hosting.ts b/server/services/hosting.ts index 826ac0f..74446e3 100644 --- a/server/services/hosting.ts +++ b/server/services/hosting.ts @@ -37,9 +37,10 @@ export async function detectHosting(domain: string): Promise { const ns = dns.filter((d) => d.type === "NS"); const ip = a?.value ?? null; - const headers = await probeHeaders(domain).catch(() => ({ - headers: [] as { name: string; value: string }[], - })); + const headers = await probeHeaders(domain).catch( + () => [] as { name: string; value: string }[], + ); + // Determine email provider, using "none" when MX is unset let emailName = mx.length === 0 @@ -64,7 +65,7 @@ export async function detectHosting(domain: string): Promise { // Hosting provider detection with fallback: // - If no A record/IP → unset → "none" // - Else if unknown → try IP ownership org/ISP - let hostingName = detectHostingProviderFromHeaders(headers.headers); + let hostingName = detectHostingProviderFromHeaders(headers); if (!ip) { hostingName = "none"; } else if (/^unknown$/i.test(hostingName)) {