1
mirror of https://github.com/jakejarvis/hoot.git synced 2025-10-18 14:24:26 -04:00

Refactor favicon fetching to use blob storage and improve API structure (#7)

This commit is contained in:
2025-09-21 15:02:58 -04:00
committed by GitHub
parent 5130562b57
commit 450f1ebd5d
16 changed files with 225 additions and 240 deletions

View File

@@ -5,3 +5,9 @@ NEXT_PUBLIC_POSTHOG_HOST=
# Upstash credentials via native Vercel integration
KV_REST_API_TOKEN=
KV_REST_API_URL=
# Vercel Blob (add integration on Vercel; token for local/dev)
BLOB_READ_WRITE_TOKEN=
# Secret used to derive unpredictable blob paths for favicons
FAVICON_BLOB_SIGNING_SECRET=

3
.gitignore vendored
View File

@@ -31,8 +31,7 @@ yarn-error.log*
.pnpm-debug.log*
# env files (can opt-in for committing if needed)
.env*
!.env.example
.env*.local
# vercel
.vercel

View File

@@ -1,94 +0,0 @@
import { createHash } from "node:crypto";
import { NextResponse } from "next/server";
import { toRegistrableDomain } from "@/lib/domain-server";
import { captureServer } from "@/server/analytics/posthog";
import {
clampFaviconSize,
getFaviconPngForDomain,
} from "@/server/services/favicon";
export const runtime = "nodejs";
export const maxDuration = 10;
export async function GET(req: Request) {
const url = new URL(req.url);
const raw = url.searchParams.get("domain") ?? url.searchParams.get("d");
const sizeParam = Number(
url.searchParams.get("size") ?? url.searchParams.get("sz") ?? "",
);
const size = clampFaviconSize(sizeParam);
// Attempt to associate events with user via PostHog cookie
let distinctId: string | undefined;
try {
const key = process.env.NEXT_PUBLIC_POSTHOG_KEY;
if (key) {
const cookieName = `ph_${key}_posthog`;
const cookieStr = req.headers.get("cookie") ?? "";
const m = cookieStr.match(new RegExp(`${cookieName}=([^;]+)`));
if (m) {
const parsed = JSON.parse(decodeURIComponent(m[1]));
if (parsed && typeof parsed.distinct_id === "string") {
distinctId = parsed.distinct_id;
}
}
}
} catch {
// ignore
}
const registrable = raw ? toRegistrableDomain(raw) : null;
if (!registrable) {
await captureServer(
"favicon_fetch",
{
domain: raw ?? "",
valid: false,
reason: "invalid_domain",
},
distinctId,
);
return NextResponse.json({ error: "Invalid domain" }, { status: 400 });
}
const png = await getFaviconPngForDomain(registrable, size, { distinctId });
if (png) {
const etag = (() => {
const hash = createHash("sha256").update(png).digest("base64url");
return `"${hash}"`;
})();
const ifNoneMatch = req.headers.get("if-none-match");
if (ifNoneMatch) {
const candidates = ifNoneMatch.split(",").map((s) => s.trim());
const matches = candidates.some((c) => c === etag || c === `W/${etag}`);
if (matches) {
return new NextResponse(null, {
status: 304,
headers: {
ETag: etag,
"Cache-Control": "public, max-age=604800, s-maxage=604800",
},
});
}
}
return new NextResponse(png as unknown as BodyInit, {
status: 200,
headers: {
"Content-Type": "image/png",
// Cache for a week; allow CDN to keep for a week
"Cache-Control": "public, max-age=604800, s-maxage=604800",
"Content-Length": String(png.length),
ETag: etag,
},
});
}
return new NextResponse(null, {
status: 404,
});
}

View File

@@ -4,23 +4,10 @@
@custom-variant dark (&:is(.dark *));
@theme inline {
--color-background: var(--background);
--color-foreground: var(--foreground);
--font-sans: var(--font-geist-sans);
--font-mono: var(--font-geist-mono);
--color-sidebar-ring: var(--sidebar-ring);
--color-sidebar-border: var(--sidebar-border);
--color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
--color-sidebar-accent: var(--sidebar-accent);
--color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
--color-sidebar-primary: var(--sidebar-primary);
--color-sidebar-foreground: var(--sidebar-foreground);
--color-sidebar: var(--sidebar);
--color-chart-5: var(--chart-5);
--color-chart-4: var(--chart-4);
--color-chart-3: var(--chart-3);
--color-chart-2: var(--chart-2);
--color-chart-1: var(--chart-1);
--color-background: var(--background);
--color-foreground: var(--foreground);
--color-ring: var(--ring);
--color-input: var(--input);
--color-border: var(--border);
@@ -63,11 +50,7 @@
--border: oklch(0.922 0 0);
--input: oklch(0.922 0 0);
--ring: oklch(0.708 0 0);
--chart-1: oklch(0.646 0.222 41.116);
--chart-2: oklch(0.6 0.118 184.704);
--chart-3: oklch(0.398 0.07 227.392);
--chart-4: oklch(0.828 0.189 84.429);
--chart-5: oklch(0.769 0.188 70.08);
/* Section accent glow (light) */
--accent-blue-glow: radial-gradient(
closest-side,
@@ -94,6 +77,7 @@
oklch(0.96 0.16 350),
transparent
);
/* DNS badge accent tokens */
--dns-a: oklch(0.62 0.2 255);
--dns-aaaa: oklch(0.6 0.19 275);
@@ -101,14 +85,6 @@
--dns-cname: oklch(0.68 0.19 310);
--dns-txt: oklch(0.78 0.17 70);
--dns-ns: oklch(0.64 0.16 195);
--sidebar: hsl(0 0% 98%);
--sidebar-foreground: hsl(240 5.3% 26.1%);
--sidebar-primary: hsl(240 5.9% 10%);
--sidebar-primary-foreground: hsl(0 0% 98%);
--sidebar-accent: hsl(240 4.8% 95.9%);
--sidebar-accent-foreground: hsl(240 5.9% 10%);
--sidebar-border: hsl(220 13% 91%);
--sidebar-ring: hsl(217.2 91.2% 59.8%);
}
.dark {
@@ -130,11 +106,7 @@
--border: oklch(1 0 0 / 10%);
--input: oklch(1 0 0 / 15%);
--ring: oklch(0.556 0 0);
--chart-1: oklch(0.488 0.243 264.376);
--chart-2: oklch(0.696 0.17 162.48);
--chart-3: oklch(0.769 0.188 70.08);
--chart-4: oklch(0.627 0.265 303.9);
--chart-5: oklch(0.645 0.246 16.439);
/* Section accent glow (dark) */
--accent-blue-glow: radial-gradient(
closest-side,
@@ -161,6 +133,7 @@
oklch(0.82 0.16 350),
transparent
);
/* DNS badge accent tokens (dark) */
--dns-a: oklch(0.72 0.16 255);
--dns-aaaa: oklch(0.7 0.16 275);
@@ -168,14 +141,6 @@
--dns-cname: oklch(0.8 0.16 310);
--dns-txt: oklch(0.86 0.14 70);
--dns-ns: oklch(0.78 0.12 195);
--sidebar: hsl(240 5.9% 10%);
--sidebar-foreground: hsl(240 4.8% 95.9%);
--sidebar-primary: hsl(224.3 76.3% 48%);
--sidebar-primary-foreground: hsl(0 0% 100%);
--sidebar-accent: hsl(240 3.7% 15.9%);
--sidebar-accent-foreground: hsl(240 4.8% 95.9%);
--sidebar-border: hsl(240 3.7% 15.9%);
--sidebar-ring: hsl(217.2 91.2% 59.8%);
}
@layer base {

View File

@@ -2,7 +2,8 @@
import { Globe } from "lucide-react";
import Image from "next/image";
import * as React from "react";
import { Skeleton } from "@/components/ui/skeleton";
import { trpc } from "@/lib/trpc/client";
import { cn } from "@/lib/utils";
export function Favicon({
@@ -14,14 +15,19 @@ export function Favicon({
size?: number;
className?: string;
}) {
const apiUrl = React.useMemo(
() => `/api/favicon?domain=${encodeURIComponent(domain)}`,
[domain],
);
const [failedUrl, setFailedUrl] = React.useState<string | null>(null);
const failed = failedUrl === apiUrl;
const { data, isLoading } = trpc.domain.faviconUrl.useQuery({ domain });
const url = data?.url ?? null;
if (failed) {
if (isLoading) {
return (
<Skeleton
className={cn("inline-block bg-input", className)}
style={{ width: size, height: size }}
/>
);
}
if (!isLoading && !url) {
return (
<Globe
className={cn("text-muted-foreground", className)}
@@ -33,13 +39,15 @@ export function Favicon({
return (
<Image
src={apiUrl}
src={
url ??
"data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw=="
}
alt="Favicon"
width={size}
height={size}
className={className}
unoptimized
onError={() => setFailedUrl(apiUrl)}
/>
);
}

View File

@@ -11,7 +11,9 @@ export function ProviderValue({
}) {
return (
<div className="inline-flex items-center gap-2">
{iconDomain ? <Favicon domain={iconDomain} size={16} /> : null}
{iconDomain ? (
<Favicon domain={iconDomain} size={16} className="rounded" />
) : null}
<span>{name}</span>
</div>
);

View File

@@ -47,7 +47,11 @@ export function HostingEmailSection({
value={data.dnsProvider.name}
leading={
data.dnsProvider.iconDomain ? (
<Favicon domain={data.dnsProvider.iconDomain} size={16} />
<Favicon
domain={data.dnsProvider.iconDomain}
size={16}
className="rounded"
/>
) : undefined
}
/>
@@ -56,7 +60,11 @@ export function HostingEmailSection({
value={data.hostingProvider.name}
leading={
data.hostingProvider.iconDomain ? (
<Favicon domain={data.hostingProvider.iconDomain} size={16} />
<Favicon
domain={data.hostingProvider.iconDomain}
size={16}
className="rounded"
/>
) : undefined
}
/>
@@ -65,7 +73,11 @@ export function HostingEmailSection({
value={data.emailProvider.name}
leading={
data.emailProvider.iconDomain ? (
<Favicon domain={data.emailProvider.iconDomain} size={16} />
<Favicon
domain={data.emailProvider.iconDomain}
size={16}
className="rounded"
/>
) : undefined
}
/>

View File

@@ -46,7 +46,11 @@ export function RegistrationSection({
value={data.registrar.name}
leading={
data.registrar.iconDomain ? (
<Favicon domain={data.registrar.iconDomain} size={16} />
<Favicon
domain={data.registrar.iconDomain}
size={16}
className="rounded"
/>
) : undefined
}
suffix={

59
lib/blob.ts Normal file
View File

@@ -0,0 +1,59 @@
import "server-only";
import { createHmac } from "node:crypto";
import { head, put } from "@vercel/blob";
const ONE_WEEK_SECONDS = 7 * 24 * 60 * 60;
function getSigningSecret(): string {
if (
process.env.NODE_ENV === "production" &&
!process.env.FAVICON_BLOB_SIGNING_SECRET
) {
throw new Error("FAVICON_BLOB_SIGNING_SECRET required in production");
}
const secret =
process.env.FAVICON_BLOB_SIGNING_SECRET ||
process.env.BLOB_READ_WRITE_TOKEN ||
"dev-favicon-secret";
return secret;
}
export function computeFaviconBlobPath(domain: string, size: number): string {
const input = `${domain}:${size}`;
const secret = getSigningSecret();
const digest = createHmac("sha256", secret).update(input).digest("hex");
// Avoid leaking domain; path is deterministic but unpredictable without secret
return `favicons/${digest}/${size}.png`;
}
export async function headFaviconBlob(
domain: string,
size: number,
): Promise<string | null> {
const pathname = computeFaviconBlobPath(domain, size);
try {
const res = await head(pathname, {
token: process.env.BLOB_READ_WRITE_TOKEN,
});
return res?.url ?? null;
} catch {
return null;
}
}
export async function putFaviconBlob(
domain: string,
size: number,
png: Buffer,
): Promise<string> {
const pathname = computeFaviconBlobPath(domain, size);
const res = await put(pathname, png, {
access: "public",
contentType: "image/png",
cacheControlMaxAge: ONE_WEEK_SECONDS,
token: process.env.BLOB_READ_WRITE_TOKEN,
});
return res.url;
}

View File

@@ -15,6 +15,14 @@ const nextConfig: NextConfig = {
dynamic: 0, // disable client-side router cache for dynamic pages
},
},
images: {
remotePatterns: [
{
protocol: "https",
hostname: "8ubelavjx21gjm2t.public.blob.vercel-storage.com",
},
],
},
async rewrites() {
return [
{

View File

@@ -27,6 +27,7 @@
"@trpc/server": "^11.5.1",
"@upstash/redis": "^1.35.4",
"@vercel/analytics": "^1.5.0",
"@vercel/blob": "^2.0.0",
"@vercel/functions": "^3.1.0",
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",

59
pnpm-lock.yaml generated
View File

@@ -29,6 +29,9 @@ importers:
'@vercel/analytics':
specifier: ^1.5.0
version: 1.5.0(next@15.6.0-canary.20(babel-plugin-react-compiler@19.1.0-rc.3)(react-dom@19.1.1(react@19.1.1))(react@19.1.1))(react@19.1.1)
'@vercel/blob':
specifier: ^2.0.0
version: 2.0.0
'@vercel/functions':
specifier: ^3.1.0
version: 3.1.0
@@ -222,6 +225,10 @@ packages:
'@emnapi/runtime@1.5.0':
resolution: {integrity: sha512-97/BJ3iXHww3djw6hYIfErCZFee7qCtrneuLa20UXFCOTCfBM2cvQHjWJ2EG0s0MtdNwInarqCTz35i4wWXHsQ==}
'@fastify/busboy@2.1.1':
resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==}
engines: {node: '>=14'}
'@floating-ui/core@1.7.3':
resolution: {integrity: sha512-sGnvb5dmrJaKEZ+LDIpguvdX3bDlEllmv4/ClQ9awcmCZrlx5jQyyMWFM5kBI+EyNOCDDiKk8il0zeuX3Zlg/w==}
@@ -1332,6 +1339,10 @@ packages:
vue-router:
optional: true
'@vercel/blob@2.0.0':
resolution: {integrity: sha512-oAj7Pdy83YKSwIaMFoM7zFeLYWRc+qUpW3PiDSblxQMnGFb43qs4bmfq7dr/+JIfwhs6PTwe1o2YBwKhyjWxXw==}
engines: {node: '>=20.0.0'}
'@vercel/functions@3.1.0':
resolution: {integrity: sha512-V+p8dO+sg1VjiJJUO5rYPp1KG17SzDcR74OWwW7Euyde6L8U5wuTMe9QfEOfLTiWPUPzN1MXZvLcYxqSYhKc4Q==}
engines: {node: '>= 20'}
@@ -1356,6 +1367,9 @@ packages:
resolution: {integrity: sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==}
engines: {node: '>=10'}
async-retry@1.3.3:
resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==}
babel-plugin-react-compiler@19.1.0-rc.3:
resolution: {integrity: sha512-mjRn69WuTz4adL0bXGx8Rsyk1086zFJeKmes6aK0xPuK3aaXmDJdLHqwKKMrpm6KAI1MCoUK72d2VeqQbu8YIA==}
@@ -1484,6 +1498,13 @@ packages:
resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==}
engines: {node: '>= 10'}
is-buffer@2.0.5:
resolution: {integrity: sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==}
engines: {node: '>=4'}
is-node-process@1.2.0:
resolution: {integrity: sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==}
is-stream@4.0.1:
resolution: {integrity: sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==}
engines: {node: '>=18'}
@@ -1767,6 +1788,10 @@ packages:
regenerator-runtime@0.13.11:
resolution: {integrity: sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==}
retry@0.13.1:
resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==}
engines: {node: '>= 4'}
safe-buffer@5.2.1:
resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
@@ -1837,6 +1862,10 @@ packages:
resolution: {integrity: sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==}
engines: {node: '>=18'}
throttleit@2.1.0:
resolution: {integrity: sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==}
engines: {node: '>=18'}
tinycolor2@1.6.0:
resolution: {integrity: sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==}
@@ -1886,6 +1915,10 @@ packages:
undici-types@7.12.0:
resolution: {integrity: sha512-goOacqME2GYyOZZfb5Lgtu+1IDmAlAEu5xnD3+xTzS10hT0vzpf0SPjkXwAw9Jm+4n/mQGDP3LO8CPbYROeBfQ==}
undici@5.29.0:
resolution: {integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==}
engines: {node: '>=14.0'}
use-callback-ref@1.3.3:
resolution: {integrity: sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==}
engines: {node: '>=10'}
@@ -1996,6 +2029,8 @@ snapshots:
tslib: 2.8.1
optional: true
'@fastify/busboy@2.1.1': {}
'@floating-ui/core@1.7.3':
dependencies:
'@floating-ui/utils': 0.2.10
@@ -3076,6 +3111,14 @@ snapshots:
next: 15.6.0-canary.20(babel-plugin-react-compiler@19.1.0-rc.3)(react-dom@19.1.1(react@19.1.1))(react@19.1.1)
react: 19.1.1
'@vercel/blob@2.0.0':
dependencies:
async-retry: 1.3.3
is-buffer: 2.0.5
is-node-process: 1.2.0
throttleit: 2.1.0
undici: 5.29.0
'@vercel/functions@3.1.0':
dependencies:
'@vercel/oidc': 3.0.0
@@ -3095,6 +3138,10 @@ snapshots:
dependencies:
tslib: 2.8.1
async-retry@1.3.3:
dependencies:
retry: 0.13.1
babel-plugin-react-compiler@19.1.0-rc.3:
dependencies:
'@babel/types': 7.28.4
@@ -3221,6 +3268,10 @@ snapshots:
ipaddr.js@2.2.0: {}
is-buffer@2.0.5: {}
is-node-process@1.2.0: {}
is-stream@4.0.1: {}
is-what@4.1.16: {}
@@ -3504,6 +3555,8 @@ snapshots:
regenerator-runtime@0.13.11: {}
retry@0.13.1: {}
safe-buffer@5.2.1: {}
scheduler@0.26.0: {}
@@ -3586,6 +3639,8 @@ snapshots:
mkdirp: 3.0.1
yallist: 5.0.0
throttleit@2.1.0: {}
tinycolor2@1.6.0: {}
tldts-core@7.0.15: {}
@@ -3623,6 +3678,10 @@ snapshots:
undici-types@7.12.0: {}
undici@5.29.0:
dependencies:
'@fastify/busboy': 2.1.1
use-callback-ref@1.3.3(@types/react@19.1.13)(react@19.1.1):
dependencies:
react: 19.1.1

View File

@@ -1,4 +1,5 @@
import { resolveAll } from "../services/dns";
import { getOrCreateFaviconBlobUrl } from "../services/favicon";
import { probeHeaders } from "../services/headers";
import { detectHosting } from "../services/hosting";
import { fetchWhois } from "../services/rdap";
@@ -14,8 +15,9 @@ export const domainRouter = router({
getCertificates,
"Certificate fetch failed",
),
headers: createDomainProcedure(async (domain: string) => {
const res = await probeHeaders(domain);
return res.headers;
}, "Header probe failed"),
headers: createDomainProcedure(probeHeaders, "Header probe failed"),
faviconUrl: createDomainProcedure(
getOrCreateFaviconBlobUrl,
"Favicon fetch failed",
),
});

View File

@@ -1,20 +1,11 @@
import sharp from "sharp";
import { cacheGet, cacheSet, ns } from "@/lib/redis";
import { headFaviconBlob, putFaviconBlob } from "@/lib/blob";
import { captureServer } from "@/server/analytics/posthog";
const DEFAULT_SIZE = 32;
const MIN_SIZE = 16;
const MAX_SIZE = 256;
const REQUEST_TIMEOUT_MS = 4000;
const CACHE_TTL_SECONDS = 7 * 24 * 60 * 60; // 7 days
const NEGATIVE_CACHE_TTL_SECONDS = 6 * 60 * 60; // 6 hours for not-found
// Use a single Redis key for both positive and negative results; this sentinel marks a negative entry
const NEGATIVE_CACHE_SENTINEL = "!" as const;
export function clampFaviconSize(value: number | null | undefined): number {
if (!value || Number.isNaN(value)) return DEFAULT_SIZE;
return Math.max(MIN_SIZE, Math.min(MAX_SIZE, Math.round(value)));
}
// Legacy Redis-based caching removed; Blob is now the canonical store
function isIcoBuffer(buf: Buffer): boolean {
return (
@@ -108,63 +99,47 @@ async function convertToPng(
return null;
}
function buildSources(domain: string, size: number): string[] {
function buildSources(domain: string): string[] {
const enc = encodeURIComponent(domain);
return [
`https://icons.duckduckgo.com/ip3/${enc}.ico`,
`https://www.google.com/s2/favicons?domain=${enc}&sz=${size}`,
`https://www.google.com/s2/favicons?domain=${enc}&sz=${DEFAULT_SIZE}`,
`https://${domain}/favicon.ico`,
`http://${domain}/favicon.ico`,
];
}
export async function getFaviconPngForDomain(
domain: string,
size: number,
opts?: { distinctId?: string },
): Promise<Buffer | null> {
const cacheKey = ns("favicon", `${domain}:${size}`);
const sources = buildSources(domain, size);
const startedAt = Date.now();
// Legacy getFaviconPngForDomain removed
// Try cache first
export async function getOrCreateFaviconBlobUrl(
domain: string,
opts?: { distinctId?: string },
): Promise<{ url: string | null }> {
const startedAt = Date.now();
// 1) Check blob first
try {
const cachedValue = await cacheGet<string>(cacheKey);
if (cachedValue) {
if (cachedValue === NEGATIVE_CACHE_SENTINEL) {
await captureServer(
"favicon_fetch",
{
domain,
size,
source: "cache_redis",
duration_ms: Date.now() - startedAt,
outcome: "not_found",
cache: "hit_negative",
},
opts?.distinctId,
);
return null;
}
const pngFromCache = Buffer.from(cachedValue, "base64");
const existing = await headFaviconBlob(domain, DEFAULT_SIZE);
if (existing) {
await captureServer(
"favicon_fetch",
{
domain,
size,
source: "cache_redis",
size: DEFAULT_SIZE,
source: "blob",
duration_ms: Date.now() - startedAt,
outcome: "ok",
cache: "hit",
cache: "hit_blob",
},
opts?.distinctId,
);
return pngFromCache;
return { url: existing };
}
} catch {
// ignore cache errors and fall through
// ignore and proceed to fetch
}
// 2) Fetch/convert via existing pipeline, then upload
const sources = buildSources(domain);
for (const src of sources) {
try {
const res = await fetchWithTimeout(src);
@@ -173,68 +148,50 @@ export async function getFaviconPngForDomain(
const ab = await res.arrayBuffer();
const buf = Buffer.from(ab);
const png = await convertToPng(buf, contentType, size);
const png = await convertToPng(buf, contentType, DEFAULT_SIZE);
if (!png) continue;
const source = (() => {
if (src.includes("icons.duckduckgo.com")) return "duckduckgo";
if (src.includes("www.google.com/s2/favicons")) return "google_s2";
if (src.includes("www.google.com/s2/favicons")) return "google";
if (src.startsWith("https://")) return "direct_https";
if (src.startsWith("http://")) return "direct_http";
return "unknown";
})();
// Attempt to store in cache (base64) for subsequent hits
try {
await cacheSet<string>(
cacheKey,
png.toString("base64"),
CACHE_TTL_SECONDS,
);
} catch {
// ignore cache errors
}
const url = await putFaviconBlob(domain, DEFAULT_SIZE, png);
await captureServer(
"favicon_fetch",
{
domain,
size,
size: DEFAULT_SIZE,
source,
upstream_status: res.status,
upstream_content_type: contentType ?? null,
duration_ms: Date.now() - startedAt,
outcome: "ok",
cache: "store",
cache: "store_blob",
},
opts?.distinctId,
);
return png;
return { url };
} catch {
// Try next source
// try next source
}
}
// Store negative cache using the same key and capture analytics
try {
await cacheSet<string>(
cacheKey,
NEGATIVE_CACHE_SENTINEL,
NEGATIVE_CACHE_TTL_SECONDS,
);
} catch {
// ignore cache errors
}
await captureServer(
"favicon_fetch",
{
domain,
size,
size: DEFAULT_SIZE,
duration_ms: Date.now() - startedAt,
outcome: "not_found",
cache: "store_negative",
cache: "miss",
},
opts?.distinctId,
);
return null;
return { url: null };
}

View File

@@ -3,7 +3,7 @@ import { captureServer } from "@/server/analytics/posthog";
export type HttpHeader = { name: string; value: string };
export async function probeHeaders(domain: string) {
export async function probeHeaders(domain: string): Promise<HttpHeader[]> {
const key = ns("headers", domain.toLowerCase());
return await getOrSet(key, 10 * 60, async () => {
const url = `https://${domain}/`;
@@ -28,11 +28,7 @@ export async function probeHeaders(domain: string) {
used_method: res.ok ? "HEAD" : "GET",
final_url: final.url,
});
return {
url: final.url,
status: final.status,
headers: normalize(headers),
};
return normalize(headers);
});
}

View File

@@ -37,9 +37,10 @@ export async function detectHosting(domain: string): Promise<HostingInfo> {
const ns = dns.filter((d) => d.type === "NS");
const ip = a?.value ?? null;
const headers = await probeHeaders(domain).catch(() => ({
headers: [] as { name: string; value: string }[],
}));
const headers = await probeHeaders(domain).catch(
() => [] as { name: string; value: string }[],
);
// Determine email provider, using "none" when MX is unset
let emailName =
mx.length === 0
@@ -64,7 +65,7 @@ export async function detectHosting(domain: string): Promise<HostingInfo> {
// Hosting provider detection with fallback:
// - If no A record/IP → unset → "none"
// - Else if unknown → try IP ownership org/ISP
let hostingName = detectHostingProviderFromHeaders(headers.headers);
let hostingName = detectHostingProviderFromHeaders(headers);
if (!ip) {
hostingName = "none";
} else if (/^unknown$/i.test(hostingName)) {