mirror of
https://github.com/jakejarvis/hoot.git
synced 2025-10-18 14:24:26 -04:00
Refactor favicon fetching to use blob storage and improve API structure (#7)
This commit is contained in:
@@ -5,3 +5,9 @@ NEXT_PUBLIC_POSTHOG_HOST=
|
||||
# Upstash credentials via native Vercel integration
|
||||
KV_REST_API_TOKEN=
|
||||
KV_REST_API_URL=
|
||||
|
||||
# Vercel Blob (add integration on Vercel; token for local/dev)
|
||||
BLOB_READ_WRITE_TOKEN=
|
||||
|
||||
# Secret used to derive unpredictable blob paths for favicons
|
||||
FAVICON_BLOB_SIGNING_SECRET=
|
||||
|
3
.gitignore
vendored
3
.gitignore
vendored
@@ -31,8 +31,7 @@ yarn-error.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# env files (can opt-in for committing if needed)
|
||||
.env*
|
||||
!.env.example
|
||||
.env*.local
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
@@ -1,94 +0,0 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { NextResponse } from "next/server";
|
||||
import { toRegistrableDomain } from "@/lib/domain-server";
|
||||
import { captureServer } from "@/server/analytics/posthog";
|
||||
import {
|
||||
clampFaviconSize,
|
||||
getFaviconPngForDomain,
|
||||
} from "@/server/services/favicon";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 10;
|
||||
|
||||
export async function GET(req: Request) {
|
||||
const url = new URL(req.url);
|
||||
const raw = url.searchParams.get("domain") ?? url.searchParams.get("d");
|
||||
const sizeParam = Number(
|
||||
url.searchParams.get("size") ?? url.searchParams.get("sz") ?? "",
|
||||
);
|
||||
const size = clampFaviconSize(sizeParam);
|
||||
|
||||
// Attempt to associate events with user via PostHog cookie
|
||||
let distinctId: string | undefined;
|
||||
try {
|
||||
const key = process.env.NEXT_PUBLIC_POSTHOG_KEY;
|
||||
if (key) {
|
||||
const cookieName = `ph_${key}_posthog`;
|
||||
const cookieStr = req.headers.get("cookie") ?? "";
|
||||
const m = cookieStr.match(new RegExp(`${cookieName}=([^;]+)`));
|
||||
if (m) {
|
||||
const parsed = JSON.parse(decodeURIComponent(m[1]));
|
||||
if (parsed && typeof parsed.distinct_id === "string") {
|
||||
distinctId = parsed.distinct_id;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
const registrable = raw ? toRegistrableDomain(raw) : null;
|
||||
if (!registrable) {
|
||||
await captureServer(
|
||||
"favicon_fetch",
|
||||
{
|
||||
domain: raw ?? "",
|
||||
valid: false,
|
||||
reason: "invalid_domain",
|
||||
},
|
||||
distinctId,
|
||||
);
|
||||
|
||||
return NextResponse.json({ error: "Invalid domain" }, { status: 400 });
|
||||
}
|
||||
|
||||
const png = await getFaviconPngForDomain(registrable, size, { distinctId });
|
||||
if (png) {
|
||||
const etag = (() => {
|
||||
const hash = createHash("sha256").update(png).digest("base64url");
|
||||
return `"${hash}"`;
|
||||
})();
|
||||
|
||||
const ifNoneMatch = req.headers.get("if-none-match");
|
||||
|
||||
if (ifNoneMatch) {
|
||||
const candidates = ifNoneMatch.split(",").map((s) => s.trim());
|
||||
const matches = candidates.some((c) => c === etag || c === `W/${etag}`);
|
||||
|
||||
if (matches) {
|
||||
return new NextResponse(null, {
|
||||
status: 304,
|
||||
headers: {
|
||||
ETag: etag,
|
||||
"Cache-Control": "public, max-age=604800, s-maxage=604800",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return new NextResponse(png as unknown as BodyInit, {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": "image/png",
|
||||
// Cache for a week; allow CDN to keep for a week
|
||||
"Cache-Control": "public, max-age=604800, s-maxage=604800",
|
||||
"Content-Length": String(png.length),
|
||||
ETag: etag,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return new NextResponse(null, {
|
||||
status: 404,
|
||||
});
|
||||
}
|
@@ -4,23 +4,10 @@
|
||||
@custom-variant dark (&:is(.dark *));
|
||||
|
||||
@theme inline {
|
||||
--color-background: var(--background);
|
||||
--color-foreground: var(--foreground);
|
||||
--font-sans: var(--font-geist-sans);
|
||||
--font-mono: var(--font-geist-mono);
|
||||
--color-sidebar-ring: var(--sidebar-ring);
|
||||
--color-sidebar-border: var(--sidebar-border);
|
||||
--color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
|
||||
--color-sidebar-accent: var(--sidebar-accent);
|
||||
--color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
|
||||
--color-sidebar-primary: var(--sidebar-primary);
|
||||
--color-sidebar-foreground: var(--sidebar-foreground);
|
||||
--color-sidebar: var(--sidebar);
|
||||
--color-chart-5: var(--chart-5);
|
||||
--color-chart-4: var(--chart-4);
|
||||
--color-chart-3: var(--chart-3);
|
||||
--color-chart-2: var(--chart-2);
|
||||
--color-chart-1: var(--chart-1);
|
||||
--color-background: var(--background);
|
||||
--color-foreground: var(--foreground);
|
||||
--color-ring: var(--ring);
|
||||
--color-input: var(--input);
|
||||
--color-border: var(--border);
|
||||
@@ -63,11 +50,7 @@
|
||||
--border: oklch(0.922 0 0);
|
||||
--input: oklch(0.922 0 0);
|
||||
--ring: oklch(0.708 0 0);
|
||||
--chart-1: oklch(0.646 0.222 41.116);
|
||||
--chart-2: oklch(0.6 0.118 184.704);
|
||||
--chart-3: oklch(0.398 0.07 227.392);
|
||||
--chart-4: oklch(0.828 0.189 84.429);
|
||||
--chart-5: oklch(0.769 0.188 70.08);
|
||||
|
||||
/* Section accent glow (light) */
|
||||
--accent-blue-glow: radial-gradient(
|
||||
closest-side,
|
||||
@@ -94,6 +77,7 @@
|
||||
oklch(0.96 0.16 350),
|
||||
transparent
|
||||
);
|
||||
|
||||
/* DNS badge accent tokens */
|
||||
--dns-a: oklch(0.62 0.2 255);
|
||||
--dns-aaaa: oklch(0.6 0.19 275);
|
||||
@@ -101,14 +85,6 @@
|
||||
--dns-cname: oklch(0.68 0.19 310);
|
||||
--dns-txt: oklch(0.78 0.17 70);
|
||||
--dns-ns: oklch(0.64 0.16 195);
|
||||
--sidebar: hsl(0 0% 98%);
|
||||
--sidebar-foreground: hsl(240 5.3% 26.1%);
|
||||
--sidebar-primary: hsl(240 5.9% 10%);
|
||||
--sidebar-primary-foreground: hsl(0 0% 98%);
|
||||
--sidebar-accent: hsl(240 4.8% 95.9%);
|
||||
--sidebar-accent-foreground: hsl(240 5.9% 10%);
|
||||
--sidebar-border: hsl(220 13% 91%);
|
||||
--sidebar-ring: hsl(217.2 91.2% 59.8%);
|
||||
}
|
||||
|
||||
.dark {
|
||||
@@ -130,11 +106,7 @@
|
||||
--border: oklch(1 0 0 / 10%);
|
||||
--input: oklch(1 0 0 / 15%);
|
||||
--ring: oklch(0.556 0 0);
|
||||
--chart-1: oklch(0.488 0.243 264.376);
|
||||
--chart-2: oklch(0.696 0.17 162.48);
|
||||
--chart-3: oklch(0.769 0.188 70.08);
|
||||
--chart-4: oklch(0.627 0.265 303.9);
|
||||
--chart-5: oklch(0.645 0.246 16.439);
|
||||
|
||||
/* Section accent glow (dark) */
|
||||
--accent-blue-glow: radial-gradient(
|
||||
closest-side,
|
||||
@@ -161,6 +133,7 @@
|
||||
oklch(0.82 0.16 350),
|
||||
transparent
|
||||
);
|
||||
|
||||
/* DNS badge accent tokens (dark) */
|
||||
--dns-a: oklch(0.72 0.16 255);
|
||||
--dns-aaaa: oklch(0.7 0.16 275);
|
||||
@@ -168,14 +141,6 @@
|
||||
--dns-cname: oklch(0.8 0.16 310);
|
||||
--dns-txt: oklch(0.86 0.14 70);
|
||||
--dns-ns: oklch(0.78 0.12 195);
|
||||
--sidebar: hsl(240 5.9% 10%);
|
||||
--sidebar-foreground: hsl(240 4.8% 95.9%);
|
||||
--sidebar-primary: hsl(224.3 76.3% 48%);
|
||||
--sidebar-primary-foreground: hsl(0 0% 100%);
|
||||
--sidebar-accent: hsl(240 3.7% 15.9%);
|
||||
--sidebar-accent-foreground: hsl(240 4.8% 95.9%);
|
||||
--sidebar-border: hsl(240 3.7% 15.9%);
|
||||
--sidebar-ring: hsl(217.2 91.2% 59.8%);
|
||||
}
|
||||
|
||||
@layer base {
|
||||
|
@@ -2,7 +2,8 @@
|
||||
|
||||
import { Globe } from "lucide-react";
|
||||
import Image from "next/image";
|
||||
import * as React from "react";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { trpc } from "@/lib/trpc/client";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
export function Favicon({
|
||||
@@ -14,14 +15,19 @@ export function Favicon({
|
||||
size?: number;
|
||||
className?: string;
|
||||
}) {
|
||||
const apiUrl = React.useMemo(
|
||||
() => `/api/favicon?domain=${encodeURIComponent(domain)}`,
|
||||
[domain],
|
||||
);
|
||||
const [failedUrl, setFailedUrl] = React.useState<string | null>(null);
|
||||
const failed = failedUrl === apiUrl;
|
||||
const { data, isLoading } = trpc.domain.faviconUrl.useQuery({ domain });
|
||||
const url = data?.url ?? null;
|
||||
|
||||
if (failed) {
|
||||
if (isLoading) {
|
||||
return (
|
||||
<Skeleton
|
||||
className={cn("inline-block bg-input", className)}
|
||||
style={{ width: size, height: size }}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (!isLoading && !url) {
|
||||
return (
|
||||
<Globe
|
||||
className={cn("text-muted-foreground", className)}
|
||||
@@ -33,13 +39,15 @@ export function Favicon({
|
||||
|
||||
return (
|
||||
<Image
|
||||
src={apiUrl}
|
||||
src={
|
||||
url ??
|
||||
"data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw=="
|
||||
}
|
||||
alt="Favicon"
|
||||
width={size}
|
||||
height={size}
|
||||
className={className}
|
||||
unoptimized
|
||||
onError={() => setFailedUrl(apiUrl)}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
@@ -11,7 +11,9 @@ export function ProviderValue({
|
||||
}) {
|
||||
return (
|
||||
<div className="inline-flex items-center gap-2">
|
||||
{iconDomain ? <Favicon domain={iconDomain} size={16} /> : null}
|
||||
{iconDomain ? (
|
||||
<Favicon domain={iconDomain} size={16} className="rounded" />
|
||||
) : null}
|
||||
<span>{name}</span>
|
||||
</div>
|
||||
);
|
||||
|
@@ -47,7 +47,11 @@ export function HostingEmailSection({
|
||||
value={data.dnsProvider.name}
|
||||
leading={
|
||||
data.dnsProvider.iconDomain ? (
|
||||
<Favicon domain={data.dnsProvider.iconDomain} size={16} />
|
||||
<Favicon
|
||||
domain={data.dnsProvider.iconDomain}
|
||||
size={16}
|
||||
className="rounded"
|
||||
/>
|
||||
) : undefined
|
||||
}
|
||||
/>
|
||||
@@ -56,7 +60,11 @@ export function HostingEmailSection({
|
||||
value={data.hostingProvider.name}
|
||||
leading={
|
||||
data.hostingProvider.iconDomain ? (
|
||||
<Favicon domain={data.hostingProvider.iconDomain} size={16} />
|
||||
<Favicon
|
||||
domain={data.hostingProvider.iconDomain}
|
||||
size={16}
|
||||
className="rounded"
|
||||
/>
|
||||
) : undefined
|
||||
}
|
||||
/>
|
||||
@@ -65,7 +73,11 @@ export function HostingEmailSection({
|
||||
value={data.emailProvider.name}
|
||||
leading={
|
||||
data.emailProvider.iconDomain ? (
|
||||
<Favicon domain={data.emailProvider.iconDomain} size={16} />
|
||||
<Favicon
|
||||
domain={data.emailProvider.iconDomain}
|
||||
size={16}
|
||||
className="rounded"
|
||||
/>
|
||||
) : undefined
|
||||
}
|
||||
/>
|
||||
|
@@ -46,7 +46,11 @@ export function RegistrationSection({
|
||||
value={data.registrar.name}
|
||||
leading={
|
||||
data.registrar.iconDomain ? (
|
||||
<Favicon domain={data.registrar.iconDomain} size={16} />
|
||||
<Favicon
|
||||
domain={data.registrar.iconDomain}
|
||||
size={16}
|
||||
className="rounded"
|
||||
/>
|
||||
) : undefined
|
||||
}
|
||||
suffix={
|
||||
|
59
lib/blob.ts
Normal file
59
lib/blob.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
import "server-only";
|
||||
|
||||
import { createHmac } from "node:crypto";
|
||||
import { head, put } from "@vercel/blob";
|
||||
|
||||
const ONE_WEEK_SECONDS = 7 * 24 * 60 * 60;
|
||||
|
||||
function getSigningSecret(): string {
|
||||
if (
|
||||
process.env.NODE_ENV === "production" &&
|
||||
!process.env.FAVICON_BLOB_SIGNING_SECRET
|
||||
) {
|
||||
throw new Error("FAVICON_BLOB_SIGNING_SECRET required in production");
|
||||
}
|
||||
|
||||
const secret =
|
||||
process.env.FAVICON_BLOB_SIGNING_SECRET ||
|
||||
process.env.BLOB_READ_WRITE_TOKEN ||
|
||||
"dev-favicon-secret";
|
||||
return secret;
|
||||
}
|
||||
|
||||
export function computeFaviconBlobPath(domain: string, size: number): string {
|
||||
const input = `${domain}:${size}`;
|
||||
const secret = getSigningSecret();
|
||||
const digest = createHmac("sha256", secret).update(input).digest("hex");
|
||||
// Avoid leaking domain; path is deterministic but unpredictable without secret
|
||||
return `favicons/${digest}/${size}.png`;
|
||||
}
|
||||
|
||||
export async function headFaviconBlob(
|
||||
domain: string,
|
||||
size: number,
|
||||
): Promise<string | null> {
|
||||
const pathname = computeFaviconBlobPath(domain, size);
|
||||
try {
|
||||
const res = await head(pathname, {
|
||||
token: process.env.BLOB_READ_WRITE_TOKEN,
|
||||
});
|
||||
return res?.url ?? null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function putFaviconBlob(
|
||||
domain: string,
|
||||
size: number,
|
||||
png: Buffer,
|
||||
): Promise<string> {
|
||||
const pathname = computeFaviconBlobPath(domain, size);
|
||||
const res = await put(pathname, png, {
|
||||
access: "public",
|
||||
contentType: "image/png",
|
||||
cacheControlMaxAge: ONE_WEEK_SECONDS,
|
||||
token: process.env.BLOB_READ_WRITE_TOKEN,
|
||||
});
|
||||
return res.url;
|
||||
}
|
@@ -15,6 +15,14 @@ const nextConfig: NextConfig = {
|
||||
dynamic: 0, // disable client-side router cache for dynamic pages
|
||||
},
|
||||
},
|
||||
images: {
|
||||
remotePatterns: [
|
||||
{
|
||||
protocol: "https",
|
||||
hostname: "8ubelavjx21gjm2t.public.blob.vercel-storage.com",
|
||||
},
|
||||
],
|
||||
},
|
||||
async rewrites() {
|
||||
return [
|
||||
{
|
||||
|
@@ -27,6 +27,7 @@
|
||||
"@trpc/server": "^11.5.1",
|
||||
"@upstash/redis": "^1.35.4",
|
||||
"@vercel/analytics": "^1.5.0",
|
||||
"@vercel/blob": "^2.0.0",
|
||||
"@vercel/functions": "^3.1.0",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
|
59
pnpm-lock.yaml
generated
59
pnpm-lock.yaml
generated
@@ -29,6 +29,9 @@ importers:
|
||||
'@vercel/analytics':
|
||||
specifier: ^1.5.0
|
||||
version: 1.5.0(next@15.6.0-canary.20(babel-plugin-react-compiler@19.1.0-rc.3)(react-dom@19.1.1(react@19.1.1))(react@19.1.1))(react@19.1.1)
|
||||
'@vercel/blob':
|
||||
specifier: ^2.0.0
|
||||
version: 2.0.0
|
||||
'@vercel/functions':
|
||||
specifier: ^3.1.0
|
||||
version: 3.1.0
|
||||
@@ -222,6 +225,10 @@ packages:
|
||||
'@emnapi/runtime@1.5.0':
|
||||
resolution: {integrity: sha512-97/BJ3iXHww3djw6hYIfErCZFee7qCtrneuLa20UXFCOTCfBM2cvQHjWJ2EG0s0MtdNwInarqCTz35i4wWXHsQ==}
|
||||
|
||||
'@fastify/busboy@2.1.1':
|
||||
resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==}
|
||||
engines: {node: '>=14'}
|
||||
|
||||
'@floating-ui/core@1.7.3':
|
||||
resolution: {integrity: sha512-sGnvb5dmrJaKEZ+LDIpguvdX3bDlEllmv4/ClQ9awcmCZrlx5jQyyMWFM5kBI+EyNOCDDiKk8il0zeuX3Zlg/w==}
|
||||
|
||||
@@ -1332,6 +1339,10 @@ packages:
|
||||
vue-router:
|
||||
optional: true
|
||||
|
||||
'@vercel/blob@2.0.0':
|
||||
resolution: {integrity: sha512-oAj7Pdy83YKSwIaMFoM7zFeLYWRc+qUpW3PiDSblxQMnGFb43qs4bmfq7dr/+JIfwhs6PTwe1o2YBwKhyjWxXw==}
|
||||
engines: {node: '>=20.0.0'}
|
||||
|
||||
'@vercel/functions@3.1.0':
|
||||
resolution: {integrity: sha512-V+p8dO+sg1VjiJJUO5rYPp1KG17SzDcR74OWwW7Euyde6L8U5wuTMe9QfEOfLTiWPUPzN1MXZvLcYxqSYhKc4Q==}
|
||||
engines: {node: '>= 20'}
|
||||
@@ -1356,6 +1367,9 @@ packages:
|
||||
resolution: {integrity: sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
async-retry@1.3.3:
|
||||
resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==}
|
||||
|
||||
babel-plugin-react-compiler@19.1.0-rc.3:
|
||||
resolution: {integrity: sha512-mjRn69WuTz4adL0bXGx8Rsyk1086zFJeKmes6aK0xPuK3aaXmDJdLHqwKKMrpm6KAI1MCoUK72d2VeqQbu8YIA==}
|
||||
|
||||
@@ -1484,6 +1498,13 @@ packages:
|
||||
resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==}
|
||||
engines: {node: '>= 10'}
|
||||
|
||||
is-buffer@2.0.5:
|
||||
resolution: {integrity: sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==}
|
||||
engines: {node: '>=4'}
|
||||
|
||||
is-node-process@1.2.0:
|
||||
resolution: {integrity: sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==}
|
||||
|
||||
is-stream@4.0.1:
|
||||
resolution: {integrity: sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==}
|
||||
engines: {node: '>=18'}
|
||||
@@ -1767,6 +1788,10 @@ packages:
|
||||
regenerator-runtime@0.13.11:
|
||||
resolution: {integrity: sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==}
|
||||
|
||||
retry@0.13.1:
|
||||
resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==}
|
||||
engines: {node: '>= 4'}
|
||||
|
||||
safe-buffer@5.2.1:
|
||||
resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
|
||||
|
||||
@@ -1837,6 +1862,10 @@ packages:
|
||||
resolution: {integrity: sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
throttleit@2.1.0:
|
||||
resolution: {integrity: sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
tinycolor2@1.6.0:
|
||||
resolution: {integrity: sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==}
|
||||
|
||||
@@ -1886,6 +1915,10 @@ packages:
|
||||
undici-types@7.12.0:
|
||||
resolution: {integrity: sha512-goOacqME2GYyOZZfb5Lgtu+1IDmAlAEu5xnD3+xTzS10hT0vzpf0SPjkXwAw9Jm+4n/mQGDP3LO8CPbYROeBfQ==}
|
||||
|
||||
undici@5.29.0:
|
||||
resolution: {integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==}
|
||||
engines: {node: '>=14.0'}
|
||||
|
||||
use-callback-ref@1.3.3:
|
||||
resolution: {integrity: sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==}
|
||||
engines: {node: '>=10'}
|
||||
@@ -1996,6 +2029,8 @@ snapshots:
|
||||
tslib: 2.8.1
|
||||
optional: true
|
||||
|
||||
'@fastify/busboy@2.1.1': {}
|
||||
|
||||
'@floating-ui/core@1.7.3':
|
||||
dependencies:
|
||||
'@floating-ui/utils': 0.2.10
|
||||
@@ -3076,6 +3111,14 @@ snapshots:
|
||||
next: 15.6.0-canary.20(babel-plugin-react-compiler@19.1.0-rc.3)(react-dom@19.1.1(react@19.1.1))(react@19.1.1)
|
||||
react: 19.1.1
|
||||
|
||||
'@vercel/blob@2.0.0':
|
||||
dependencies:
|
||||
async-retry: 1.3.3
|
||||
is-buffer: 2.0.5
|
||||
is-node-process: 1.2.0
|
||||
throttleit: 2.1.0
|
||||
undici: 5.29.0
|
||||
|
||||
'@vercel/functions@3.1.0':
|
||||
dependencies:
|
||||
'@vercel/oidc': 3.0.0
|
||||
@@ -3095,6 +3138,10 @@ snapshots:
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
|
||||
async-retry@1.3.3:
|
||||
dependencies:
|
||||
retry: 0.13.1
|
||||
|
||||
babel-plugin-react-compiler@19.1.0-rc.3:
|
||||
dependencies:
|
||||
'@babel/types': 7.28.4
|
||||
@@ -3221,6 +3268,10 @@ snapshots:
|
||||
|
||||
ipaddr.js@2.2.0: {}
|
||||
|
||||
is-buffer@2.0.5: {}
|
||||
|
||||
is-node-process@1.2.0: {}
|
||||
|
||||
is-stream@4.0.1: {}
|
||||
|
||||
is-what@4.1.16: {}
|
||||
@@ -3504,6 +3555,8 @@ snapshots:
|
||||
|
||||
regenerator-runtime@0.13.11: {}
|
||||
|
||||
retry@0.13.1: {}
|
||||
|
||||
safe-buffer@5.2.1: {}
|
||||
|
||||
scheduler@0.26.0: {}
|
||||
@@ -3586,6 +3639,8 @@ snapshots:
|
||||
mkdirp: 3.0.1
|
||||
yallist: 5.0.0
|
||||
|
||||
throttleit@2.1.0: {}
|
||||
|
||||
tinycolor2@1.6.0: {}
|
||||
|
||||
tldts-core@7.0.15: {}
|
||||
@@ -3623,6 +3678,10 @@ snapshots:
|
||||
|
||||
undici-types@7.12.0: {}
|
||||
|
||||
undici@5.29.0:
|
||||
dependencies:
|
||||
'@fastify/busboy': 2.1.1
|
||||
|
||||
use-callback-ref@1.3.3(@types/react@19.1.13)(react@19.1.1):
|
||||
dependencies:
|
||||
react: 19.1.1
|
||||
|
@@ -1,4 +1,5 @@
|
||||
import { resolveAll } from "../services/dns";
|
||||
import { getOrCreateFaviconBlobUrl } from "../services/favicon";
|
||||
import { probeHeaders } from "../services/headers";
|
||||
import { detectHosting } from "../services/hosting";
|
||||
import { fetchWhois } from "../services/rdap";
|
||||
@@ -14,8 +15,9 @@ export const domainRouter = router({
|
||||
getCertificates,
|
||||
"Certificate fetch failed",
|
||||
),
|
||||
headers: createDomainProcedure(async (domain: string) => {
|
||||
const res = await probeHeaders(domain);
|
||||
return res.headers;
|
||||
}, "Header probe failed"),
|
||||
headers: createDomainProcedure(probeHeaders, "Header probe failed"),
|
||||
faviconUrl: createDomainProcedure(
|
||||
getOrCreateFaviconBlobUrl,
|
||||
"Favicon fetch failed",
|
||||
),
|
||||
});
|
||||
|
@@ -1,20 +1,11 @@
|
||||
import sharp from "sharp";
|
||||
import { cacheGet, cacheSet, ns } from "@/lib/redis";
|
||||
import { headFaviconBlob, putFaviconBlob } from "@/lib/blob";
|
||||
import { captureServer } from "@/server/analytics/posthog";
|
||||
|
||||
const DEFAULT_SIZE = 32;
|
||||
const MIN_SIZE = 16;
|
||||
const MAX_SIZE = 256;
|
||||
const REQUEST_TIMEOUT_MS = 4000;
|
||||
const CACHE_TTL_SECONDS = 7 * 24 * 60 * 60; // 7 days
|
||||
const NEGATIVE_CACHE_TTL_SECONDS = 6 * 60 * 60; // 6 hours for not-found
|
||||
// Use a single Redis key for both positive and negative results; this sentinel marks a negative entry
|
||||
const NEGATIVE_CACHE_SENTINEL = "!" as const;
|
||||
|
||||
export function clampFaviconSize(value: number | null | undefined): number {
|
||||
if (!value || Number.isNaN(value)) return DEFAULT_SIZE;
|
||||
return Math.max(MIN_SIZE, Math.min(MAX_SIZE, Math.round(value)));
|
||||
}
|
||||
// Legacy Redis-based caching removed; Blob is now the canonical store
|
||||
|
||||
function isIcoBuffer(buf: Buffer): boolean {
|
||||
return (
|
||||
@@ -108,63 +99,47 @@ async function convertToPng(
|
||||
return null;
|
||||
}
|
||||
|
||||
function buildSources(domain: string, size: number): string[] {
|
||||
function buildSources(domain: string): string[] {
|
||||
const enc = encodeURIComponent(domain);
|
||||
return [
|
||||
`https://icons.duckduckgo.com/ip3/${enc}.ico`,
|
||||
`https://www.google.com/s2/favicons?domain=${enc}&sz=${size}`,
|
||||
`https://www.google.com/s2/favicons?domain=${enc}&sz=${DEFAULT_SIZE}`,
|
||||
`https://${domain}/favicon.ico`,
|
||||
`http://${domain}/favicon.ico`,
|
||||
];
|
||||
}
|
||||
|
||||
export async function getFaviconPngForDomain(
|
||||
domain: string,
|
||||
size: number,
|
||||
opts?: { distinctId?: string },
|
||||
): Promise<Buffer | null> {
|
||||
const cacheKey = ns("favicon", `${domain}:${size}`);
|
||||
const sources = buildSources(domain, size);
|
||||
const startedAt = Date.now();
|
||||
// Legacy getFaviconPngForDomain removed
|
||||
|
||||
// Try cache first
|
||||
export async function getOrCreateFaviconBlobUrl(
|
||||
domain: string,
|
||||
opts?: { distinctId?: string },
|
||||
): Promise<{ url: string | null }> {
|
||||
const startedAt = Date.now();
|
||||
// 1) Check blob first
|
||||
try {
|
||||
const cachedValue = await cacheGet<string>(cacheKey);
|
||||
if (cachedValue) {
|
||||
if (cachedValue === NEGATIVE_CACHE_SENTINEL) {
|
||||
await captureServer(
|
||||
"favicon_fetch",
|
||||
{
|
||||
domain,
|
||||
size,
|
||||
source: "cache_redis",
|
||||
duration_ms: Date.now() - startedAt,
|
||||
outcome: "not_found",
|
||||
cache: "hit_negative",
|
||||
},
|
||||
opts?.distinctId,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
const pngFromCache = Buffer.from(cachedValue, "base64");
|
||||
const existing = await headFaviconBlob(domain, DEFAULT_SIZE);
|
||||
if (existing) {
|
||||
await captureServer(
|
||||
"favicon_fetch",
|
||||
{
|
||||
domain,
|
||||
size,
|
||||
source: "cache_redis",
|
||||
size: DEFAULT_SIZE,
|
||||
source: "blob",
|
||||
duration_ms: Date.now() - startedAt,
|
||||
outcome: "ok",
|
||||
cache: "hit",
|
||||
cache: "hit_blob",
|
||||
},
|
||||
opts?.distinctId,
|
||||
);
|
||||
return pngFromCache;
|
||||
return { url: existing };
|
||||
}
|
||||
} catch {
|
||||
// ignore cache errors and fall through
|
||||
// ignore and proceed to fetch
|
||||
}
|
||||
|
||||
// 2) Fetch/convert via existing pipeline, then upload
|
||||
const sources = buildSources(domain);
|
||||
for (const src of sources) {
|
||||
try {
|
||||
const res = await fetchWithTimeout(src);
|
||||
@@ -173,68 +148,50 @@ export async function getFaviconPngForDomain(
|
||||
const ab = await res.arrayBuffer();
|
||||
const buf = Buffer.from(ab);
|
||||
|
||||
const png = await convertToPng(buf, contentType, size);
|
||||
const png = await convertToPng(buf, contentType, DEFAULT_SIZE);
|
||||
if (!png) continue;
|
||||
|
||||
const source = (() => {
|
||||
if (src.includes("icons.duckduckgo.com")) return "duckduckgo";
|
||||
if (src.includes("www.google.com/s2/favicons")) return "google_s2";
|
||||
if (src.includes("www.google.com/s2/favicons")) return "google";
|
||||
if (src.startsWith("https://")) return "direct_https";
|
||||
if (src.startsWith("http://")) return "direct_http";
|
||||
return "unknown";
|
||||
})();
|
||||
|
||||
// Attempt to store in cache (base64) for subsequent hits
|
||||
try {
|
||||
await cacheSet<string>(
|
||||
cacheKey,
|
||||
png.toString("base64"),
|
||||
CACHE_TTL_SECONDS,
|
||||
);
|
||||
} catch {
|
||||
// ignore cache errors
|
||||
}
|
||||
const url = await putFaviconBlob(domain, DEFAULT_SIZE, png);
|
||||
|
||||
await captureServer(
|
||||
"favicon_fetch",
|
||||
{
|
||||
domain,
|
||||
size,
|
||||
size: DEFAULT_SIZE,
|
||||
source,
|
||||
upstream_status: res.status,
|
||||
upstream_content_type: contentType ?? null,
|
||||
duration_ms: Date.now() - startedAt,
|
||||
outcome: "ok",
|
||||
cache: "store",
|
||||
cache: "store_blob",
|
||||
},
|
||||
opts?.distinctId,
|
||||
);
|
||||
|
||||
return png;
|
||||
return { url };
|
||||
} catch {
|
||||
// Try next source
|
||||
// try next source
|
||||
}
|
||||
}
|
||||
// Store negative cache using the same key and capture analytics
|
||||
try {
|
||||
await cacheSet<string>(
|
||||
cacheKey,
|
||||
NEGATIVE_CACHE_SENTINEL,
|
||||
NEGATIVE_CACHE_TTL_SECONDS,
|
||||
);
|
||||
} catch {
|
||||
// ignore cache errors
|
||||
}
|
||||
|
||||
await captureServer(
|
||||
"favicon_fetch",
|
||||
{
|
||||
domain,
|
||||
size,
|
||||
size: DEFAULT_SIZE,
|
||||
duration_ms: Date.now() - startedAt,
|
||||
outcome: "not_found",
|
||||
cache: "store_negative",
|
||||
cache: "miss",
|
||||
},
|
||||
opts?.distinctId,
|
||||
);
|
||||
return null;
|
||||
return { url: null };
|
||||
}
|
||||
|
@@ -3,7 +3,7 @@ import { captureServer } from "@/server/analytics/posthog";
|
||||
|
||||
export type HttpHeader = { name: string; value: string };
|
||||
|
||||
export async function probeHeaders(domain: string) {
|
||||
export async function probeHeaders(domain: string): Promise<HttpHeader[]> {
|
||||
const key = ns("headers", domain.toLowerCase());
|
||||
return await getOrSet(key, 10 * 60, async () => {
|
||||
const url = `https://${domain}/`;
|
||||
@@ -28,11 +28,7 @@ export async function probeHeaders(domain: string) {
|
||||
used_method: res.ok ? "HEAD" : "GET",
|
||||
final_url: final.url,
|
||||
});
|
||||
return {
|
||||
url: final.url,
|
||||
status: final.status,
|
||||
headers: normalize(headers),
|
||||
};
|
||||
return normalize(headers);
|
||||
});
|
||||
}
|
||||
|
||||
|
@@ -37,9 +37,10 @@ export async function detectHosting(domain: string): Promise<HostingInfo> {
|
||||
const ns = dns.filter((d) => d.type === "NS");
|
||||
const ip = a?.value ?? null;
|
||||
|
||||
const headers = await probeHeaders(domain).catch(() => ({
|
||||
headers: [] as { name: string; value: string }[],
|
||||
}));
|
||||
const headers = await probeHeaders(domain).catch(
|
||||
() => [] as { name: string; value: string }[],
|
||||
);
|
||||
|
||||
// Determine email provider, using "none" when MX is unset
|
||||
let emailName =
|
||||
mx.length === 0
|
||||
@@ -64,7 +65,7 @@ export async function detectHosting(domain: string): Promise<HostingInfo> {
|
||||
// Hosting provider detection with fallback:
|
||||
// - If no A record/IP → unset → "none"
|
||||
// - Else if unknown → try IP ownership org/ISP
|
||||
let hostingName = detectHostingProviderFromHeaders(headers.headers);
|
||||
let hostingName = detectHostingProviderFromHeaders(headers);
|
||||
if (!ip) {
|
||||
hostingName = "none";
|
||||
} else if (/^unknown$/i.test(hostingName)) {
|
||||
|
Reference in New Issue
Block a user