jake/mastodon-utils
1
Fork 0
mirror of https://github.com/jakejarvis/mastodon-utils.git synced 2025-04-26 03:25:22 -04:00
Code Issues Releases Wiki Activity

commit my messy notes (prometheus/grafana & brotli)

Browse Source
This commit is contained in:
Jake Jarvis 2022-12-09 20:45:43 -05:00
parent 4424f5cb03
commit ced8e91274
Signed by: jake
GPG Key ID: 2B0C9CF251E69A39
3 changed files with 399 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
etc/nginx/sites-available/mastodon.conf
View File

@ -55,8 +55,24 @@ server {
gzip_comp_level 6;
gzip_buffers 16 8k;
gzip_http_version 1.1;
gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml image/x-icon;
gzip_min_length 256; # jake: added
gzip_types application/atom+xml application/javascript application/json application/rss+xml
application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype
application/x-font-ttf application/x-javascript application/xhtml+xml application/xml
font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon
image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml;
gzip_min_length 256;
# jake: added
# https://github.com/google/ngx_brotli#sample-configuration
brotli on;
brotli_comp_level 4;
brotli_static on;
brotli_types application/atom+xml application/javascript application/json application/rss+xml
application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype
application/x-font-ttf application/x-javascript application/xhtml+xml application/xml
font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon
image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml;
brotli_min_length 256;
location / {
try_files $uri @proxy;

70
notes/brotli.md Normal file
View File

@ -0,0 +1,70 @@
# Brotli compression for nginx
- https://github.com/google/ngx_brotli
- https://www.atlantic.net/dedicated-server-hosting/how-to-install-brotli-module-for-nginx-on-ubuntu-20-04/
- https://linuxhint.com/enable-brotli-compression-nginx/
- https://www.bowsercache.com/blog/enable-brotli-for-nginx-on-ubuntu-20-04/#install-the-brotli-module-for-nginx
---
/etc/apt/sources.list.d/nginx.list:
```
deb [arch=amd64 signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] http://nginx.org/packages/ubuntu/ focal nginx
deb-src [arch=amd64 signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] http://nginx.org/packages/ubuntu/ focal nginx
```
---
```bash
cd /usr/local/src
apt-get source nginx
apt-get build-dep nginx -y
git clone --recursive https://github.com/google/ngx_brotli
cd nginx-1.22.1/
./configure --with-compat --add-dynamic-module=../ngx_brotli
make modules
cp ./objs/ngx_http_brotli_*.so /usr/lib/nginx/modules/
```
---
/etc/nginx/nginx.conf:
```
load_module modules/ngx_http_brotli_filter_module.so;
load_module modules/ngx_http_brotli_static_module.so;
```
---
nginx site config: ([ref](https://github.com/google/ngx_brotli#sample-configuration))
```
server {
# ...
brotli on;
brotli_comp_level 4;
brotli_static on;
brotli_types application/atom+xml application/javascript application/json application/rss+xml
application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype
application/x-font-ttf application/x-javascript application/xhtml+xml application/xml
font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon
image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml;
brotli_min_length 256;
# ...
}
```
---
```bash
nginx -t
nginx -s reload
```

311
notes/prometheus.md Normal file
View File

@ -0,0 +1,311 @@
# Prometheus & Grafana notes
- https://grafana.pipe.fail/public-dashboards/b5ca7a7c8e844f90b0973d2ab02bad0a
- https://ipng.ch/s/articles/2022/11/27/mastodon-3.html
- https://ourcodeworld.com/articles/read/1686/how-to-install-prometheus-node-exporter-on-ubuntu-2004
## Exporters
- https://github.com/prometheus/node_exporter
- https://github.com/oliver006/redis_exporter
- https://github.com/prometheus-community/postgres_exporter
- https://github.com/prometheus/statsd_exporter
- https://github.com/nginxinc/nginx-prometheus-exporter
- https://github.com/prometheus-community/json_exporter
---
```bash
wget https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz
tar xvf node_exporter-1.3.1.linux-amd64.tar.gz
cp node_exporter-1.3.1.linux-amd64/node_exporter /usr/local/bin/
useradd --no-create-home --shell /bin/false node_exporter
chown node_exporter:node_exporter /usr/local/bin/node_exporter
nano /etc/system/systemd/node-exporter.service # see below
systemctl daemon-reload
systemctl enable --now redis-exporter.service
```
## Config
/etc/prometheus/prometheus.yml:
```yml
global:
scrape_interval: 15s
evaluation_interval: 15s
scrape_configs:
- job_name: "prometheus"
static_configs:
- targets: ["localhost:9090"]
- job_name: "node_exporter_metrics"
static_configs:
- targets: ["localhost:9100"]
- job_name: "redis_exporter_targets"
static_configs:
- targets: ["redis://localhost:6379"]
metrics_path: /scrape
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: localhost:9121
- job_name: "redis_exporter_metrics"
static_configs:
- targets: ["localhost:9121"]
- job_name: "postgres_exporter_metrics"
static_configs:
- targets: ["localhost:9187"]
- job_name: "nginx_exporter_metrics"
static_configs:
- targets: ["localhost:9113"]
- job_name: "statsd_exporter_metrics"
static_configs:
- targets: ["localhost:9102"]
- job_name: "elasticsearch_exporter_metrics"
static_configs:
- targets: ["localhost:9114"]
- job_name: "json_exporter_metrics"
static_configs:
- targets: ["localhost:9079"]
- job_name: "json_exporter_targets"
metrics_path: /probe
scrape_interval: 30s
params:
module: [linode]
static_configs:
- targets:
- https://api.linode.com/v4/object-storage/buckets/us-east-1/jarvis-mastodon
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: localhost:9079
```
---
json-config.yml:
```yml
modules:
linode:
metrics:
- name: json_linode_size
path: "{.size}"
labels:
bucket: "{.label}"
zone: "{.cluster}"
hostname: "{.hostname}"
- name: json_linode_objects
path: "{.objects}"
labels:
bucket: "{.label}"
zone: "{.cluster}"
hostname: "{.hostname}"
headers:
Authorization: "Bearer XXXXXX"
```
---
/etc/prometheus/statsd-mapping.yml:
```yml
## Prometheus Statsd Exporter mapping for Mastodon 4.0+
##
## Version 1.0, November 2022
##
## Documentation: https://ipng.ch/s/articles/2022/11/27/mastodon-3.html
mappings:
## Web collector
- match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.status\.(.+)
match_type: regex
name: "mastodon_controller_status"
labels:
controller: $1
action: $2
format: $3
status: $4
mastodon: "web"
- match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.db_time
match_type: regex
name: "mastodon_controller_db_time"
labels:
controller: $1
action: $2
format: $3
mastodon: "web"
- match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.view_time
match_type: regex
name: "mastodon_controller_view_time"
labels:
controller: $1
action: $2
format: $3
mastodon: "web"
- match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.total_duration
match_type: regex
name: "mastodon_controller_duration"
labels:
controller: $1
action: $2
format: $3
mastodon: "web"
## Database collector
- match: Mastodon\.production\.db\.tables\.(.+)\.queries\.(.+)\.duration
match_type: regex
name: "mastodon_db_operation"
labels:
table: "$1"
operation: "$2"
mastodon: "db"
## Cache collector
- match: Mastodon\.production\.cache\.(.+)\.duration
match_type: regex
name: "mastodon_cache_duration"
labels:
operation: "$1"
mastodon: "cache"
## Sidekiq collector
- match: Mastodon\.production\.sidekiq\.(.+)\.processing_time
match_type: regex
name: "mastodon_sidekiq_worker_processing_time"
labels:
worker: "$1"
mastodon: "sidekiq"
- match: Mastodon\.production\.sidekiq\.(.+)\.success
match_type: regex
name: "mastodon_sidekiq_worker_success_total"
labels:
worker: "$1"
mastodon: "sidekiq"
- match: Mastodon\.production\.sidekiq\.(.+)\.failure
match_type: regex
name: "mastodon_sidekiq_worker_failure_total"
labels:
worker: "$1"
mastodon: "sidekiq"
- match: Mastodon\.production\.sidekiq\.queues\.(.+)\.enqueued
match_type: regex
name: "mastodon_sidekiq_queue_enqueued"
labels:
queue: "$1"
mastodon: "sidekiq"
- match: Mastodon\.production\.sidekiq\.queues\.(.+)\.latency
match_type: regex
name: "mastodon_sidekiq_queue_latency"
labels:
queue: "$1"
mastodon: "sidekiq"
- match: Mastodon\.production\.sidekiq\.(.+)
match_type: regex
name: "mastodon_sidekiq_$1"
labels:
mastodon: "sidekiq"
```
---
/home/mastodon/live/.env.production:
```
STATSD_ADDR=localhost:9125
```
---
(example) /etc/systemd/system/redis-exporter.service:
```
[Unit]
Description=Redis Exporter
Wants=network-online.target
After=network-online.target
[Service]
User=redis_exporter
Group=redis_exporter
Type=simple
ExecStart=/usr/local/bin/redis_exporter
Restart=always
[Install]
WantedBy=multi-user.target
```
---
/etc/grafana/grafana.ini:
```ini
[server]
http_addr =
http_port = 3003
root_url = https://grafana.pipe.fail
[analytics]
reporting_enabled = false
check_for_updates = false
check_for_plugin_updates = false
feedback_links_enabled = false
[security]
disable_initial_admin_creation = true
disable_gravatar = true
cookie_secure = true
[snapshots]
external_enabled = false
[dashboards]
versions_to_keep = 100
[users]
allow_sign_up = false
default_theme = dark
[auth]
disable_login = true
disable_login_form = true
[auth.grafana_com]
enabled = true
allow_sign_up = false
client_id =
client_secret =
scopes = user:email
allowed_organizations =
[metrics]
enabled = false
[live]
max_connections = 10
[feature_toggles]
publicDashboards = true
```