Add Docker Compose setup for local development environment (#123)

2025-12-02 19:33:48 -05:00 · 2025-10-23 11:48:52 -04:00
parent 80a7813996
commit b9bfa65fd5
8 changed files with 287 additions and 25 deletions
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # 📚 [Domainstack](https://domainstack.io) - Domain Intelligence Tool

-[Domainstack](https://domainstack.io) is an all‑in‑one app for exploring domain names. Search any domain (e.g., [`github.com`](https://domainstack.io/github.com)) and get instant insights including WHOIS/RDAP lookups, DNS records, SSL certificates, HTTP headers, hosting details, geolocation, and SEO signals.
+[Domainstack](https://domainstack.io) is an all-in-one app for exploring domain names. Search any domain (e.g., [`github.com`](https://domainstack.io/github.com)) and get instant insights including WHOIS/RDAP lookups, DNS records, SSL certificates, HTTP headers, hosting details, geolocation, and SEO signals.

 ![Screenshot of Domainstack domain analysis page for GitHub.com](https://github.com/user-attachments/assets/5a13d2c5-2d1c-4f70-bc52-a2742d43ebc6)

@@ -35,37 +35,93 @@

 ## 🌱 Getting Started

-1. Clone & install:
+### 1. Clone & install
+
 ```bash
 git clone https://github.com/jakejarvis/domainstack.io.git
 cd domainstack.io
 pnpm install
 ```

-2. Configure environment (see `.env.example`)
+### 2. Configure environment variables

-3. Run database migrations:
-```bash
-pnpm db:migrate
-# Seed known providers in `lib/providers/rules/`
-pnmm db:seed:providers
+Create `.env.local` (used by `pnpm dev`):
+
+```env
+# --- Database (local) ---
+# TCP URL used by Drizzle CLI & direct TCP usage
+DATABASE_URL=postgres://postgres:postgres@localhost:5432/main
+
+# --- Redis (local via SRH) ---
+# SRH mimics Upstash REST locally; point your app’s Upstash client here.
+KV_REST_API_URL=http://localhost:8079
+KV_REST_API_TOKEN=dev-token
+
+# --- Inngest Dev Server ---
+INNGEST_DEV=1
+INNGEST_BASE_URL=http://localhost:8288
+# If your Inngest handler lives at a custom route, set:
+INNGEST_SERVE_PATH=/api/inngest
 ```

-4. Run dev server:
+### 3. Start local dev services (Docker)
+
+We provide a single [`docker-compose.yml`](docker-compose.yml) and a helper script ([`start-dev-infra.sh`](scripts/start-dev-infra.sh)) that boots all services and waits for them to be ready:
+
+- **Postgres** on `localhost:5432`
+- **Neon wsproxy** on `localhost:5433` (WebSocket proxy used by the Neon serverless driver)
+- **Redis** on `localhost:6379`
+- **Serverless Redis HTTP (SRH)** on `http://localhost:8079` (Upstash-compatible REST proxy)
+- **Inngest Dev Server** on `http://localhost:8288`
+
+Run:
+
+```bash
+pnpm dev:start-docker
+```
+
+> On Linux, if `host.docker.internal` isn’t available, add `extra_hosts` to the Inngest service in `docker-compose.yml`:
+>
+> ```yaml
+> extra_hosts: ["host.docker.internal:host-gateway"]
+> ```
+
+### 4. Run Drizzle database migrations & seeds
+
+```bash
+pnpm db:generate   # generate SQL from schema
+pnpm db:migrate    # apply migrations to local Postgres
+pnpm db:seed:providers  # seed known providers in lib/providers/rules/
+```
+
+### 5. Start the Next.js dev server
+
+Run in a second terminal window:
+
 ```bash
 pnpm dev
 ```
-Open http://localhost:3000
+
+Open [http://localhost:3000](http://localhost:3000)
+
+The Inngest Dev UI will be available at [http://localhost:8288](http://localhost:8288) and is already configured to call the local Next.js server at `http://localhost:3000/api/inngest`.

 ---

 ## 🧰 Useful Commands

 ```bash
-pnpm dev # start dev server
-pnpm lint # Biome lint/format checks
-pnpm typecheck # tsc --noEmit
-pnpm test:run # Vitest
+pnpm dev                  # start dev server (uses .env.development.local)
+pnpm dev:start-docker     # start Dockerized local services and wait until ready
+pnpm lint                 # Biome lint/format checks
+pnpm typecheck            # tsc --noEmit
+pnpm test:run             # Vitest
+
+# Drizzle
+pnpm db:generate    # generate SQL migrations from schema
+pnpm db:migrate     # apply db migrations
+pnpm db:studio      # open Drizzle Studio against your current env URL
+pnpm db:seed:providers
 ```

 ---
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -0,0 +1,63 @@
+services:
+  # Local Postgres (TCP on 5432)
+  postgres:
+    image: postgres:17
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: main
+    ports:
+      - "5432:5432"
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres -d main"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # Neon WebSocket proxy -> Postgres (WS on 5433)
+  # This mirrors the Neon Drizzle guide's example: proxy listens on :80 in-container,
+  # we publish it on localhost:5433, and point it at our local Postgres at postgres:5432
+  # See https://neon.com/guides/drizzle-local-vercel
+  pg_proxy:
+    image: ghcr.io/neondatabase/wsproxy:latest
+    environment:
+      APPEND_PORT: "postgres:5432"
+      ALLOW_ADDR_REGEX: ".*"
+      LOG_TRAFFIC: "true"
+    ports:
+      - "5433:80"
+    depends_on:
+      - postgres
+
+  # Local Redis (binary protocol on 6379)
+  redis:
+    image: redis:7-alpine
+    ports:
+      - "6379:6379"
+
+  # Serverless Redis HTTP (SRH) — Upstash-compatible REST proxy on 8079
+  # Set KV_REST_API_URL to http://localhost:8079
+  srh:
+    image: hiett/serverless-redis-http:latest
+    environment:
+      SRH_MODE: "env"
+      SRH_TOKEN: "dev-token"
+      SRH_CONNECTION_STRING: "redis://redis:6379"
+    ports:
+      - "8079:80"
+    depends_on:
+      - redis
+
+  # Inngest Dev Server on 8288
+  # Next.js runs on the HOST at :3000; the dev server reaches it via host.docker.internal
+  inngest:
+    image: inngest/inngest:latest
+    command: >
+      inngest dev -u http://host.docker.internal:3000/api/inngest
+    ports:
+      - "8288:8288"
+
+volumes:
+  pg_data:
--- a/drizzle.config.ts
+++ b/drizzle.config.ts
@@ -1,10 +1,22 @@
-import type { Config } from "drizzle-kit";
+import * as dotenv from "dotenv";

-export default {
-  schema: "./server/db/schema.ts",
+// Load common local envs first if present, then default .env
+dotenv.config({ path: ".env.local" });
+dotenv.config();
+
+import { defineConfig } from "drizzle-kit";
+
+const url = process.env.DATABASE_URL;
+
+if (!url) {
+  throw new Error("DATABASE_URL is not set");
+}
+
+export default defineConfig({
+  schema: "./lib/db/schema.ts",
  out: "./drizzle",
  dialect: "postgresql",
  dbCredentials: {
-    url: process.env.DATABASE_URL as string,
+    url,
  },
-} satisfies Config;
+});
--- a/lib/db/client.ts
+++ b/lib/db/client.ts
@@ -1,5 +1,6 @@
-import { Pool } from "@neondatabase/serverless";
+import { neonConfig, Pool } from "@neondatabase/serverless";
 import { drizzle } from "drizzle-orm/neon-serverless";
+import ws from "ws";
 import * as schema from "@/lib/db/schema";

 const connectionString = process.env.DATABASE_URL;
@@ -8,5 +9,18 @@ if (!connectionString) {
  throw new Error("DATABASE_URL is not set");
 }

+// Always set the WS constructor (needed in Node)
+neonConfig.webSocketConstructor = ws;
+
+// Local dev: route WebSockets via the Neon wsproxy on localhost:5433
+if (process.env.NODE_ENV !== "production") {
+  // Tell the driver how to build the wsproxy URL from the DB host
+  // With DATABASE_URL using host=localhost, this becomes "localhost:5433/v1"
+  neonConfig.wsProxy = (host) => `${host}:5433/v1`;
+  neonConfig.useSecureWebSocket = false; // ws:// not wss://
+  neonConfig.pipelineTLS = false;
+  neonConfig.pipelineConnect = false;
+}
+
 const pool = new Pool({ connectionString });
 export const db = drizzle(pool, { schema });
--- a/package.json
+++ b/package.json
@@ -13,6 +13,7 @@
  "type": "module",
  "scripts": {
    "dev": "next dev --turbo",
+    "dev:start-docker": "scripts/start-dev-infra.sh",
    "build": "next build",
    "start": "next start",
    "lint": "biome check",
@@ -50,6 +51,7 @@
    "cmdk": "^1.1.1",
    "country-flag-icons": "^1.5.21",
    "date-fns": "^4.1.0",
+    "dotenv": "^17.2.3",
    "drizzle-orm": "^0.44.6",
    "drizzle-zod": "^0.8.3",
    "file-type": "^21.0.0",
@@ -63,6 +65,7 @@
    "motion": "^12.23.24",
    "next": "15.6.0-canary.39",
    "next-themes": "^0.4.6",
+    "postgres": "^3.4.7",
    "posthog-js": "^1.277.0",
    "posthog-node": "^5.10.0",
    "puppeteer-core": "24.22.3",
@@ -78,6 +81,7 @@
    "tailwind-merge": "^3.3.1",
    "uuid": "^13.0.0",
    "vaul": "^1.1.2",
+    "ws": "^8.18.3",
    "zod": "^4.1.12"
  },
  "devDependencies": {
@@ -91,6 +95,7 @@
    "@types/node": "24.8.1",
    "@types/react": "19.1.16",
    "@types/react-dom": "19.1.9",
+    "@types/ws": "^8.18.1",
    "@vitejs/plugin-react": "^5.0.4",
    "@vitest/coverage-v8": "^3.2.4",
    "@vitest/ui": "^3.2.4",
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -71,12 +71,15 @@ importers:
      date-fns:
        specifier: ^4.1.0
        version: 4.1.0
+      dotenv:
+        specifier: ^17.2.3
+        version: 17.2.3
      drizzle-orm:
        specifier: ^0.44.6
-        version: 0.44.6(@electric-sql/pglite@0.3.11)(@neondatabase/serverless@1.0.2)(@opentelemetry/api@1.9.0)(@types/pg@8.15.5)(@upstash/redis@1.35.6)
+        version: 0.44.6(@electric-sql/pglite@0.3.11)(@neondatabase/serverless@1.0.2)(@opentelemetry/api@1.9.0)(@types/pg@8.15.5)(@upstash/redis@1.35.6)(postgres@3.4.7)
      drizzle-zod:
        specifier: ^0.8.3
-        version: 0.8.3(drizzle-orm@0.44.6(@electric-sql/pglite@0.3.11)(@neondatabase/serverless@1.0.2)(@opentelemetry/api@1.9.0)(@types/pg@8.15.5)(@upstash/redis@1.35.6))(zod@4.1.12)
+        version: 0.8.3(drizzle-orm@0.44.6(@electric-sql/pglite@0.3.11)(@neondatabase/serverless@1.0.2)(@opentelemetry/api@1.9.0)(@types/pg@8.15.5)(@upstash/redis@1.35.6)(postgres@3.4.7))(zod@4.1.12)
      file-type:
        specifier: ^21.0.0
        version: 21.0.0
@@ -110,6 +113,9 @@ importers:
      next-themes:
        specifier: ^0.4.6
        version: 0.4.6(react-dom@19.1.1(react@19.1.1))(react@19.1.1)
+      postgres:
+        specifier: ^3.4.7
+        version: 3.4.7
      posthog-js:
        specifier: ^1.277.0
        version: 1.277.0
@@ -155,6 +161,9 @@ importers:
      vaul:
        specifier: ^1.1.2
        version: 1.1.2(@types/react-dom@19.1.9(@types/react@19.1.16))(@types/react@19.1.16)(react-dom@19.1.1(react@19.1.1))(react@19.1.1)
+      ws:
+        specifier: ^8.18.3
+        version: 8.18.3
      zod:
        specifier: ^4.1.12
        version: 4.1.12
@@ -189,6 +198,9 @@ importers:
      '@types/react-dom':
        specifier: 19.1.9
        version: 19.1.9(@types/react@19.1.16)
+      '@types/ws':
+        specifier: ^8.18.1
+        version: 8.18.1
      '@vitejs/plugin-react':
        specifier: ^5.0.4
        version: 5.0.4(vite@7.1.10(@types/node@24.8.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6))
@@ -2995,6 +3007,9 @@ packages:
  '@types/tedious@4.0.14':
    resolution: {integrity: sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw==}

+  '@types/ws@8.18.1':
+    resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==}
+
  '@types/yauzl@2.10.3':
    resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==}

@@ -3528,6 +3543,10 @@ packages:
  domutils@3.2.2:
    resolution: {integrity: sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==}

+  dotenv@17.2.3:
+    resolution: {integrity: sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==}
+    engines: {node: '>=12'}
+
  drizzle-kit@0.31.5:
    resolution: {integrity: sha512-+CHgPFzuoTQTt7cOYCV6MOw2w8vqEn/ap1yv4bpZOWL03u7rlVRQhUY0WYT3rHsgVTXwYQDZaSUJSQrMBUKuWg==}
    hasBin: true
@@ -4524,6 +4543,10 @@ packages:
    resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==}
    engines: {node: '>=0.10.0'}

+  postgres@3.4.7:
+    resolution: {integrity: sha512-Jtc2612XINuBjIl/QTWsV5UvE8UHuNblcO3vVADSrKsrc6RqGX6lOW1cEo3CM2v0XG4Nat8nI+YM7/f26VxXLw==}
+    engines: {node: '>=12'}
+
  posthog-js@1.277.0:
    resolution: {integrity: sha512-whSyov8KH2IwXkeJVbgu07EkPk6AITXnrJN7Mg5rGAHJQ0LS1w6qh2Ib4LMsLHTrR5UAqwYHcufbjDl6snoESw==}
    peerDependencies:
@@ -8576,6 +8599,10 @@ snapshots:
    dependencies:
      '@types/node': 24.8.1

+  '@types/ws@8.18.1':
+    dependencies:
+      '@types/node': 24.8.1
+
  '@types/yauzl@2.10.3':
    dependencies:
      '@types/node': 24.8.1
@@ -9107,6 +9134,8 @@ snapshots:
      domelementtype: 2.3.0
      domhandler: 5.0.3

+  dotenv@17.2.3: {}
+
  drizzle-kit@0.31.5:
    dependencies:
      '@drizzle-team/brocli': 0.10.2
@@ -9116,17 +9145,18 @@ snapshots:
    transitivePeerDependencies:
      - supports-color

-  drizzle-orm@0.44.6(@electric-sql/pglite@0.3.11)(@neondatabase/serverless@1.0.2)(@opentelemetry/api@1.9.0)(@types/pg@8.15.5)(@upstash/redis@1.35.6):
+  drizzle-orm@0.44.6(@electric-sql/pglite@0.3.11)(@neondatabase/serverless@1.0.2)(@opentelemetry/api@1.9.0)(@types/pg@8.15.5)(@upstash/redis@1.35.6)(postgres@3.4.7):
    optionalDependencies:
      '@electric-sql/pglite': 0.3.11
      '@neondatabase/serverless': 1.0.2
      '@opentelemetry/api': 1.9.0
      '@types/pg': 8.15.5
      '@upstash/redis': 1.35.6
+      postgres: 3.4.7

-  drizzle-zod@0.8.3(drizzle-orm@0.44.6(@electric-sql/pglite@0.3.11)(@neondatabase/serverless@1.0.2)(@opentelemetry/api@1.9.0)(@types/pg@8.15.5)(@upstash/redis@1.35.6))(zod@4.1.12):
+  drizzle-zod@0.8.3(drizzle-orm@0.44.6(@electric-sql/pglite@0.3.11)(@neondatabase/serverless@1.0.2)(@opentelemetry/api@1.9.0)(@types/pg@8.15.5)(@upstash/redis@1.35.6)(postgres@3.4.7))(zod@4.1.12):
    dependencies:
-      drizzle-orm: 0.44.6(@electric-sql/pglite@0.3.11)(@neondatabase/serverless@1.0.2)(@opentelemetry/api@1.9.0)(@types/pg@8.15.5)(@upstash/redis@1.35.6)
+      drizzle-orm: 0.44.6(@electric-sql/pglite@0.3.11)(@neondatabase/serverless@1.0.2)(@opentelemetry/api@1.9.0)(@types/pg@8.15.5)(@upstash/redis@1.35.6)(postgres@3.4.7)
      zod: 4.1.12

  dunder-proto@1.0.1:
@@ -10049,6 +10079,8 @@ snapshots:
    dependencies:
      xtend: 4.0.2

+  postgres@3.4.7: {}
+
  posthog-js@1.277.0:
    dependencies:
      '@posthog/core': 1.3.0
--- a/scripts/seed/providers.ts
+++ b/scripts/seed/providers.ts
@@ -1,3 +1,9 @@
+import * as dotenv from "dotenv";
+
+// Load common local envs first if present, then default .env
+dotenv.config({ path: ".env.local" });
+dotenv.config();
+
 import { db } from "@/lib/db/client";
 import { type providerCategory, providers } from "@/lib/db/schema";
 import {
--- a/scripts/start-dev-infra.sh
+++ b/scripts/start-dev-infra.sh
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Change to repo root (this script lives in ./scripts)
+ROOT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT_DIR"
+
+# Allow overriding the compose command (e.g., DOCKER_COMPOSE="docker-compose")
+DOCKER_COMPOSE="${DOCKER_COMPOSE:-docker compose}"
+
+# Check if Docker is installed and on PATH
+if ! command -v docker >/dev/null 2>&1; then
+  echo "❌ Docker is not installed or not on PATH."
+  exit 1
+fi
+
+# Check if Docker Compose v2 is installed and on PATH
+if ! $DOCKER_COMPOSE version >/dev/null 2>&1; then
+  echo "❌ Docker Compose v2 is required (use 'docker compose', not 'docker-compose')."
+  exit 1
+fi
+
+echo "🚀 Starting local infra with Docker Compose…"
+
+# Pull missing images (no-op if already pulled), then start detached
+$DOCKER_COMPOSE pull --ignore-pull-failures || true
+$DOCKER_COMPOSE up -d
+
+# --- Wait helpers ------------------------------------------------------------
+wait_for_port() {
+  local host="$1" port="$2" name="$3" timeout="${4:-60}"
+  local start ts_now
+
+  echo -n "⏳ Waiting for ${name} on ${host}:${port} (timeout ${timeout}s)… "
+  start=$(date +%s)
+  while true; do
+    # bash's /dev/tcp works on most systems without netcat
+    if (exec 3<>"/dev/tcp/${host}/${port}") 2>/dev/null; then
+      exec 3>&- 3<&-
+      echo "✅"
+      return 0
+    fi
+    ts_now=$(date +%s)
+    if (( ts_now - start >= timeout )); then
+      echo "❌ timed out"
+      return 1
+    fi
+    sleep 1
+  done
+}
+
+# --- Wait for exposed services ----------------------------------------------
+# Postgres TCP
+wait_for_port "127.0.0.1" 5432 "Postgres"
+# Neon wsproxy (WebSocket over HTTP)
+wait_for_port "127.0.0.1" 5433 "Neon wsproxy"
+# Redis TCP
+wait_for_port "127.0.0.1" 6379 "Redis"
+# Serverless Redis HTTP (SRH)
+wait_for_port "127.0.0.1" 8079 "SRH (Upstash-compatible HTTP)"
+# Inngest Dev Server
+wait_for_port "127.0.0.1" 8288 "Inngest Dev Server"
+
+echo
+echo "🎉 Local infra is ready!"
+echo "   Postgres:  postgres://postgres:postgres@localhost:5432/main"
+echo "   wsproxy:   ws://localhost:5433/v1  (driver uses this automatically)"
+echo "   Redis:     redis://localhost:6379"
+echo "   SRH:       http://localhost:8079"
+echo "   Inngest:   http://localhost:8288"
+echo
+
+echo "📜 Following logs (Ctrl+C to stop log tail; services keep running)…"
+exec $DOCKER_COMPOSE logs -f --tail=100