commit my messy notes (prometheus/grafana & brotli)

2025-04-25 22:45:22 -04:00 · 2022-12-09 20:45:43 -05:00 · 2022-12-09 20:45:43 -05:00 · ced8e91274
commit ced8e91274
parent 4424f5cb03
3 changed files with 399 additions and 2 deletions
--- a/etc/nginx/sites-available/mastodon.conf
+++ b/etc/nginx/sites-available/mastodon.conf
@ -55,8 +55,24 @@ server {
  gzip_comp_level 6;
  gzip_buffers 16 8k;
  gzip_http_version 1.1;
-  gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml image/x-icon;
-  gzip_min_length 256; # jake: added
+  gzip_types application/atom+xml application/javascript application/json application/rss+xml
+             application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype
+             application/x-font-ttf application/x-javascript application/xhtml+xml application/xml
+             font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon
+             image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml;
+  gzip_min_length 256;
+
+  # jake: added
+  # https://github.com/google/ngx_brotli#sample-configuration
+  brotli on;
+  brotli_comp_level 4;
+  brotli_static on;
+  brotli_types application/atom+xml application/javascript application/json application/rss+xml
+               application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype
+               application/x-font-ttf application/x-javascript application/xhtml+xml application/xml
+               font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon
+               image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml;
+  brotli_min_length 256;

  location / {
    try_files $uri @proxy;
--- a/notes/brotli.md
+++ b/notes/brotli.md
@ -0,0 +1,70 @@
+# Brotli compression for nginx
+
+- https://github.com/google/ngx_brotli
+- https://www.atlantic.net/dedicated-server-hosting/how-to-install-brotli-module-for-nginx-on-ubuntu-20-04/
+- https://linuxhint.com/enable-brotli-compression-nginx/
+- https://www.bowsercache.com/blog/enable-brotli-for-nginx-on-ubuntu-20-04/#install-the-brotli-module-for-nginx
+
+---
+
+/etc/apt/sources.list.d/nginx.list:
+
+```
+deb [arch=amd64 signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] http://nginx.org/packages/ubuntu/ focal nginx
+deb-src [arch=amd64 signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] http://nginx.org/packages/ubuntu/ focal nginx
+```
+
+---
+
+```bash
+cd /usr/local/src
+
+apt-get source nginx
+apt-get build-dep nginx -y
+
+git clone --recursive https://github.com/google/ngx_brotli
+
+cd nginx-1.22.1/
+./configure --with-compat --add-dynamic-module=../ngx_brotli
+make modules
+
+cp ./objs/ngx_http_brotli_*.so /usr/lib/nginx/modules/
+```
+
+---
+
+/etc/nginx/nginx.conf:
+
+```
+load_module modules/ngx_http_brotli_filter_module.so;
+load_module modules/ngx_http_brotli_static_module.so;
+```
+
+---
+
+nginx site config: ([ref](https://github.com/google/ngx_brotli#sample-configuration))
+
+```
+server {
+  # ...
+
+  brotli on;
+  brotli_comp_level 4;
+  brotli_static on;
+  brotli_types application/atom+xml application/javascript application/json application/rss+xml
+               application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype
+               application/x-font-ttf application/x-javascript application/xhtml+xml application/xml
+               font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon
+               image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml;
+  brotli_min_length 256;
+
+  # ...
+}
+```
+
+---
+
+```bash
+nginx -t
+nginx -s reload
+```
--- a/notes/prometheus.md
+++ b/notes/prometheus.md
@ -0,0 +1,311 @@
+# Prometheus & Grafana notes
+
+- https://grafana.pipe.fail/public-dashboards/b5ca7a7c8e844f90b0973d2ab02bad0a
+- https://ipng.ch/s/articles/2022/11/27/mastodon-3.html
+- https://ourcodeworld.com/articles/read/1686/how-to-install-prometheus-node-exporter-on-ubuntu-2004
+
+## Exporters
+
+- https://github.com/prometheus/node_exporter
+- https://github.com/oliver006/redis_exporter
+- https://github.com/prometheus-community/postgres_exporter
+- https://github.com/prometheus/statsd_exporter
+- https://github.com/nginxinc/nginx-prometheus-exporter
+- https://github.com/prometheus-community/json_exporter
+
+---
+
+```bash
+wget https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz
+tar xvf node_exporter-1.3.1.linux-amd64.tar.gz
+cp node_exporter-1.3.1.linux-amd64/node_exporter /usr/local/bin/
+
+useradd --no-create-home --shell /bin/false node_exporter
+chown node_exporter:node_exporter /usr/local/bin/node_exporter
+
+nano /etc/system/systemd/node-exporter.service # see below
+
+systemctl daemon-reload
+systemctl enable --now redis-exporter.service
+```
+
+## Config
+
+/etc/prometheus/prometheus.yml:
+
+```yml
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["localhost:9090"]
+
+  - job_name: "node_exporter_metrics"
+    static_configs:
+      - targets: ["localhost:9100"]
+
+  - job_name: "redis_exporter_targets"
+    static_configs:
+      - targets: ["redis://localhost:6379"]
+    metrics_path: /scrape
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: localhost:9121
+
+  - job_name: "redis_exporter_metrics"
+    static_configs:
+      - targets: ["localhost:9121"]
+
+  - job_name: "postgres_exporter_metrics"
+    static_configs:
+      - targets: ["localhost:9187"]
+
+  - job_name: "nginx_exporter_metrics"
+    static_configs:
+      - targets: ["localhost:9113"]
+
+  - job_name: "statsd_exporter_metrics"
+    static_configs:
+      - targets: ["localhost:9102"]
+
+  - job_name: "elasticsearch_exporter_metrics"
+    static_configs:
+      - targets: ["localhost:9114"]
+
+  - job_name: "json_exporter_metrics"
+    static_configs:
+      - targets: ["localhost:9079"]
+
+  - job_name: "json_exporter_targets"
+    metrics_path: /probe
+    scrape_interval: 30s
+    params:
+      module: [linode]
+    static_configs:
+      - targets:
+        - https://api.linode.com/v4/object-storage/buckets/us-east-1/jarvis-mastodon
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: localhost:9079
+```
+
+---
+
+json-config.yml:
+
+```yml
+modules:
+  linode:
+    metrics:
+    - name: json_linode_size
+      path: "{.size}"
+      labels:
+        bucket: "{.label}"
+        zone: "{.cluster}"
+        hostname: "{.hostname}"
+
+    - name: json_linode_objects
+      path: "{.objects}"
+      labels:
+        bucket: "{.label}"
+        zone: "{.cluster}"
+        hostname: "{.hostname}"
+
+    headers:
+      Authorization: "Bearer XXXXXX"
+```
+
+---
+
+/etc/prometheus/statsd-mapping.yml:
+
+```yml
+## Prometheus Statsd Exporter mapping for Mastodon 4.0+
+##
+## Version 1.0, November 2022
+##
+## Documentation: https://ipng.ch/s/articles/2022/11/27/mastodon-3.html
+
+mappings:
+  ## Web collector
+  - match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.status\.(.+)
+    match_type: regex
+    name: "mastodon_controller_status"
+    labels:
+      controller: $1
+      action: $2
+      format: $3
+      status: $4
+      mastodon: "web"
+  - match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.db_time
+    match_type: regex
+    name: "mastodon_controller_db_time"
+    labels:
+      controller: $1
+      action: $2
+      format: $3
+      mastodon: "web"
+  - match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.view_time
+    match_type: regex
+    name: "mastodon_controller_view_time"
+    labels:
+      controller: $1
+      action: $2
+      format: $3
+      mastodon: "web"
+  - match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.total_duration
+    match_type: regex
+    name: "mastodon_controller_duration"
+    labels:
+      controller: $1
+      action: $2
+      format: $3
+      mastodon: "web"
+
+  ## Database collector
+  - match: Mastodon\.production\.db\.tables\.(.+)\.queries\.(.+)\.duration
+    match_type: regex
+    name: "mastodon_db_operation"
+    labels:
+      table: "$1"
+      operation: "$2"
+      mastodon: "db"
+
+  ## Cache collector
+  - match: Mastodon\.production\.cache\.(.+)\.duration
+    match_type: regex
+    name: "mastodon_cache_duration"
+    labels:
+      operation: "$1"
+      mastodon: "cache"
+
+  ## Sidekiq collector
+  - match: Mastodon\.production\.sidekiq\.(.+)\.processing_time
+    match_type: regex
+    name: "mastodon_sidekiq_worker_processing_time"
+    labels:
+      worker: "$1"
+      mastodon: "sidekiq"
+  - match: Mastodon\.production\.sidekiq\.(.+)\.success
+    match_type: regex
+    name: "mastodon_sidekiq_worker_success_total"
+    labels:
+      worker: "$1"
+      mastodon: "sidekiq"
+  - match: Mastodon\.production\.sidekiq\.(.+)\.failure
+    match_type: regex
+    name: "mastodon_sidekiq_worker_failure_total"
+    labels:
+      worker: "$1"
+      mastodon: "sidekiq"
+  - match: Mastodon\.production\.sidekiq\.queues\.(.+)\.enqueued
+    match_type: regex
+    name: "mastodon_sidekiq_queue_enqueued"
+    labels:
+      queue: "$1"
+      mastodon: "sidekiq"
+  - match: Mastodon\.production\.sidekiq\.queues\.(.+)\.latency
+    match_type: regex
+    name: "mastodon_sidekiq_queue_latency"
+    labels:
+      queue: "$1"
+      mastodon: "sidekiq"
+  - match: Mastodon\.production\.sidekiq\.(.+)
+    match_type: regex
+    name: "mastodon_sidekiq_$1"
+    labels:
+      mastodon: "sidekiq"
+```
+
+---
+
+/home/mastodon/live/.env.production:
+
+```
+STATSD_ADDR=localhost:9125
+```
+
+---
+
+(example) /etc/systemd/system/redis-exporter.service:
+
+```
+[Unit]
+Description=Redis Exporter
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+User=redis_exporter
+Group=redis_exporter
+Type=simple
+ExecStart=/usr/local/bin/redis_exporter
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+```
+
+---
+
+/etc/grafana/grafana.ini:
+
+```ini
+[server]
+http_addr =
+http_port = 3003
+root_url = https://grafana.pipe.fail
+
+[analytics]
+reporting_enabled = false
+check_for_updates = false
+check_for_plugin_updates = false
+feedback_links_enabled = false
+
+[security]
+disable_initial_admin_creation = true
+disable_gravatar = true
+cookie_secure = true
+
+[snapshots]
+external_enabled = false
+
+[dashboards]
+versions_to_keep = 100
+
+[users]
+allow_sign_up = false
+default_theme = dark
+
+[auth]
+disable_login = true
+disable_login_form = true
+
+[auth.grafana_com]
+enabled = true
+allow_sign_up = false
+client_id =
+client_secret =
+scopes = user:email
+allowed_organizations =
+
+[metrics]
+enabled = false
+
+[live]
+max_connections = 10
+
+[feature_toggles]
+publicDashboards = true
+```