diff --git a/etc/grafana/grafana.ini b/etc/grafana/grafana.ini new file mode 100644 index 0000000..1bc5013 --- /dev/null +++ b/etc/grafana/grafana.ini @@ -0,0 +1,51 @@ +##################### Grafana Configuration Example ##################### +# +# Everything has defaults so you only need to uncomment things you want to +# change + +[server] +http_addr = +http_port = 3003 +root_url = https://grafana.pipe.fail + +[analytics] +reporting_enabled = false +check_for_updates = false +check_for_plugin_updates = false +feedback_links_enabled = false + +[security] +disable_initial_admin_creation = true +disable_gravatar = true +cookie_secure = true + +[snapshots] +external_enabled = false + +[dashboards] +versions_to_keep = 100 + +[users] +allow_sign_up = false +default_theme = dark + +[auth] +disable_login = true +disable_login_form = true + +[auth.grafana_com] +enabled = true +allow_sign_up = false +client_id = +client_secret = +scopes = user:email +allowed_organizations = + +[metrics] +enabled = false + +[live] +max_connections = 10 + +[feature_toggles] +publicDashboards = true diff --git a/etc/grafana/public-dashboard.json b/etc/grafana/public-dashboard.json new file mode 100644 index 0000000..110ae49 --- /dev/null +++ b/etc/grafana/public-dashboard.json @@ -0,0 +1,3420 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.3.1" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "state-timeline", + "name": "State timeline", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Public dashboard for fediverse.jarv.is metrics", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 17492, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "fediverse.jarv.is", + "tooltip": "Open Mastodon server", + "type": "link", + "url": "https://fediverse.jarv.is/about" + } + ], + "liveNow": true, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 91, + "links": [ + { + "targetBlank": true, + "title": "Homepage", + "url": "https://fediverse.jarv.is/about" + }, + { + "targetBlank": true, + "title": "@jake", + "url": "https://fediverse.jarv.is/@jake" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n Say hi:\n @jake@jarv.is on fediverse.jarv.is\n
", + "mode": "html" + }, + "pluginVersion": "9.3.1", + "title": "🦣 Welcome to the public metrics for my single-user Mastodon server! 📈", + "transparent": true, + "type": "text" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 77, + "panels": [], + "title": "System Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Busy state of all CPU cores together", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 65, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(sum by(instance) (irate(node_cpu_seconds_total{mode!=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total[$__rate_interval])))) * 100", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "CPU Busy %", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Non available RAM memory", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 3 + }, + "hideTimeOverride": false, + "id": 71, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "100 - ((node_memory_MemAvailable_bytes * 100) / node_memory_MemTotal_bytes)", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "B", + "step": 240 + } + ], + "title": "Memory Used %", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Used Swap", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 15 + }, + { + "color": "red", + "value": 35 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 73, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / (node_memory_SwapTotal_bytes )) * 100", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Swap Used %", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Used Root FS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 3 + }, + "id": 75, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "100 - ((node_filesystem_avail_bytes{mountpoint=\"/\", fstype!=\"rootfs\"} * 100) / node_filesystem_size_bytes{mountpoint=\"/\", fstype!=\"rootfs\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "Disk Used %", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Amount of bandwidth used by Linode servers and media storage out of the monthly quota.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 3 + }, + "id": 99, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(json_linode_transfer_used / ( json_linode_transfer_quota / 2)) * 100", + "hide": false, + "instant": true, + "legendFormat": "", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((json_linode_transfer * 100) / (250 * 1000000000))", + "hide": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Linode Transfer Quota %", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Time since last server restart.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 3600000 + }, + { + "color": "yellow", + "value": 43200000 + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 3 + }, + "hideTimeOverride": true, + "id": 79, + "links": [ + { + "targetBlank": true, + "title": "Status page", + "url": "https://status.pipe.fail/" + } + ], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^uptime$/", + "values": false + }, + "textMode": "value" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_time_seconds - node_boot_time_seconds", + "instant": true, + "intervalFactor": 1, + "legendFormat": "uptime", + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "System Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Basic CPU info", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy System" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy User" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Other" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 81, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total[$__rate_interval])))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Busy System", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total[$__rate_interval])))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Busy User", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total[$__rate_interval])))", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "Busy Iowait", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{mode=~\".*irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total[$__rate_interval])))", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "Busy IRQs", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{mode!='idle', mode!='user', mode!='system',mode!='iowait', mode!='irq', mode!='softirq'}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total[$__rate_interval])))", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "Busy Other", + "range": true, + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total[$__rate_interval])))", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "Idle", + "range": true, + "refId": "F", + "step": 240 + } + ], + "title": "CPU", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Basic memory usage", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "thresholds" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 8000000000, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "RAM Cache + Buffer" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#575757", + "mode": "fixed" + } + }, + { + "id": "unit", + "value": "decbytes" + }, + { + "id": "custom.fillOpacity", + "value": 20 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Used" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + }, + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 7 + }, + "id": 83, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "RAM Total", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes - node_memory_MemFree_bytes - (node_memory_Cached_bytes + node_memory_Buffers_bytes + node_memory_SReclaimable_bytes)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "RAM Used", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "node_memory_Cached_bytes + node_memory_Buffers_bytes + node_memory_SReclaimable_bytes", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "RAM Cache + Buffer", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "node_memory_MemFree_bytes", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "RAM Free", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes)", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "Swap Used", + "range": true, + "refId": "E", + "step": 240 + } + ], + "title": "Memory", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Basic network info per interface", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "eth0 in" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "eth0 out" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 7 + }, + "id": 85, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "irate(node_network_receive_bytes_total{device=\"eth0\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} in", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "irate(node_network_transmit_bytes_total{device=\"eth0\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} out", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic", + "transformations": [], + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 45, + "panels": [], + "title": "Queue", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The total amount of worker jobs that Sidekiq is performing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "dark-blue", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "In Progress" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Scheduled" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Retry" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dead" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 55, + "links": [ + { + "targetBlank": true, + "title": "Open Sidekiq", + "url": "https://fediverse.jarv.is/sidekiq" + } + ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sort(sum by (mastodon)(mastodon_sidekiq_workers_size))", + "instant": true, + "interval": "", + "legendFormat": "In Progress", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sort(sum by (mastodon)(mastodon_sidekiq_scheduled_size))", + "hide": false, + "instant": true, + "legendFormat": "Scheduled", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sort(sum by (mastodon)(mastodon_sidekiq_retry_size))", + "hide": false, + "instant": true, + "legendFormat": "Retry", + "range": false, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sort(sum by (mastodon)(mastodon_sidekiq_dead_size))", + "hide": false, + "instant": true, + "legendFormat": "Dead", + "range": false, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sort(sum by (mastodon)(mastodon_sidekiq_failed))", + "hide": true, + "instant": true, + "legendFormat": "Failed", + "range": false, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sort(sum by (mastodon)(mastodon_sidekiq_processed))", + "hide": true, + "instant": true, + "legendFormat": "Processed", + "range": false, + "refId": "F" + } + ], + "title": "Sidekiq Current Jobs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "A historical graph of the average queue delay for each of the main queues in Sidekiq. If the queues do not have wait-time, they are depicted green. They will go through yellow, orange and become red once the queue is backing up and worker jobs have to wait before being scheduled.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 12, + "y": 14 + }, + "id": 61, + "interval": "30s", + "options": { + "alignValue": "center", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": false, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sort(sum by (queue)(mastodon_sidekiq_queue_latency))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Sidekiq Queue Delay", + "type": "state-timeline" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The running average latency of HTTP Queries that Mastodon is serving.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.75 + }, + { + "color": "red", + "value": 1.25 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 14 + }, + "id": 38, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (mastodon) (rate(mastodon_sidekiq_worker_processing_time_sum[$__rate_interval])) /\nsum by (mastodon) (rate(mastodon_sidekiq_worker_processing_time_count[$__rate_interval]))", + "instant": true, + "legendFormat": "Average", + "range": false, + "refId": "A" + } + ], + "title": "Sidekiq Avg Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of sidekiq worker invocations by worker type, split out by success and failure.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 18 + }, + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (worker)(rate(mastodon_sidekiq_worker_success_total{worker=~\"(ActivityPub..DeliveryWorker|ActivityPub..ProcessingWorker|ActivityPub..SynchronizeFeaturedCollectionWorker|ActivityPub..SynchronizeFeaturedTagsCollectionWorker|DistributionWorker|FeedInsertWorker|FetchReplyWorker|LinkCrawlWorker|LocalNotificationWorker|PollExpirationNotifyWorker|PushUpdateWorker|RedownloadMediaWorker|Scheduler..AccountsStatusesCleanupScheduler|Scheduler..IndexingScheduler|Scheduler..InstanceRefreshScheduler|Scheduler..ScheduledStatusesScheduler|Scheduler..SuspendedUserCleanupScheduler|Scheduler..Trends..RefreshScheduler|ThreadResolveWorker|VerifyAccountLinksWorker|Web..PushNotificationWorker)\"}[$__rate_interval]))", + "legendFormat": "{{worker}} Success", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (worker)(rate(mastodon_sidekiq_worker_failure_total{worker=~\"(ActivityPub..DeliveryWorker|ActivityPub..ProcessingWorker|ActivityPub..SynchronizeFeaturedCollectionWorker|ActivityPub..SynchronizeFeaturedTagsCollectionWorker|DistributionWorker|FeedInsertWorker|FetchReplyWorker|LinkCrawlWorker|LocalNotificationWorker|PollExpirationNotifyWorker|PushUpdateWorker|RedownloadMediaWorker|Scheduler..AccountsStatusesCleanupScheduler|Scheduler..IndexingScheduler|Scheduler..InstanceRefreshScheduler|Scheduler..ScheduledStatusesScheduler|Scheduler..SuspendedUserCleanupScheduler|Scheduler..Trends..RefreshScheduler|ThreadResolveWorker|VerifyAccountLinksWorker|Web..PushNotificationWorker)\"}[$__rate_interval]))", + "hide": false, + "legendFormat": "{{worker}} Failure", + "range": true, + "refId": "B" + } + ], + "title": "Sidekiq Workers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Running latency of sidekiq worker invocations by worker type.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 18 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (worker)(rate(mastodon_sidekiq_worker_processing_time_sum{worker=~\"(ActivityPub..DeliveryWorker|ActivityPub..ProcessingWorker|ActivityPub..SynchronizeFeaturedCollectionWorker|ActivityPub..SynchronizeFeaturedTagsCollectionWorker|DistributionWorker|FeedInsertWorker|FetchReplyWorker|LinkCrawlWorker|LocalNotificationWorker|PollExpirationNotifyWorker|PushUpdateWorker|RedownloadMediaWorker|Scheduler..AccountsStatusesCleanupScheduler|Scheduler..IndexingScheduler|Scheduler..InstanceRefreshScheduler|Scheduler..ScheduledStatusesScheduler|Scheduler..SuspendedUserCleanupScheduler|Scheduler..Trends..RefreshScheduler|ThreadResolveWorker|VerifyAccountLinksWorker|Web..PushNotificationWorker)\"}[$__rate_interval]))\n/ sum by (worker)(rate(mastodon_sidekiq_worker_processing_time_count{worker=~\"(ActivityPub..DeliveryWorker|ActivityPub..ProcessingWorker|ActivityPub..SynchronizeFeaturedCollectionWorker|ActivityPub..SynchronizeFeaturedTagsCollectionWorker|DistributionWorker|FeedInsertWorker|FetchReplyWorker|LinkCrawlWorker|LocalNotificationWorker|PollExpirationNotifyWorker|PushUpdateWorker|RedownloadMediaWorker|Scheduler..AccountsStatusesCleanupScheduler|Scheduler..IndexingScheduler|Scheduler..InstanceRefreshScheduler|Scheduler..ScheduledStatusesScheduler|Scheduler..SuspendedUserCleanupScheduler|Scheduler..Trends..RefreshScheduler|ThreadResolveWorker|VerifyAccountLinksWorker|Web..PushNotificationWorker)\"}[$__rate_interval]))\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Sidekiq Workers Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "A total number of worker jobs waiting in the queue, usually due to there not being enough worker threads or sidekiq jobs to handle them in time. This graph here shows the n-count of elements in the queue.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 18 + }, + "id": 63, + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sort(sum by (queue)(mastodon_sidekiq_queue_enqueued))", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Sidekiq Queue Depth", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 89, + "panels": [], + "title": "Web", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The running average latency of HTTP Queries that Mastodon is serving.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 0.75 + }, + { + "color": "red", + "value": 1.25 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 25 + }, + "id": 34, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (mastodon) (rate(mastodon_controller_duration_sum[$__rate_interval])) /\nsum by (mastodon) (rate(mastodon_controller_duration_count[$__rate_interval]))", + "instant": true, + "legendFormat": "Average", + "range": false, + "refId": "A" + } + ], + "title": "Puma Avg Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of response codes/sec of HTTP API calls served by the Mastodon web server.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 20, + "x": 4, + "y": 25 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (status) (rate(mastodon_controller_status{controller=~\"(AboutController|AccountsController|ActivityPub.CollectionsController|ActivityPub.InboxesController|ActivityPub.OutboxesController|ActivityPub.RepliesController|Admin.DashboardController|Admin.Settings.AboutController|Admin.Settings.BrandingController|Api.OEmbedController|Api.V1.Accounts.FeaturedTagsController|Api.V1.Accounts.FollowerAccountsController|Api.V1.Accounts.LookupController|Api.V1.Accounts.RelationshipsController|Api.V1.Accounts.SearchController|Api.V1.Accounts.StatusesController|Api.V1.AccountsController|Api.V1.Admin.DimensionsController|Api.V1.Admin.MeasuresController|Api.V1.Admin.RetentionController|Api.V1.Admin.Trends.TagsController|Api.V1.AnnouncementsController|Api.V1.ConversationsController|Api.V1.CustomEmojisController|Api.V1.DirectoriesController|Api.V1.FollowRequestsController|Api.V1.Instances.ExtendedDescriptionsController|Api.V1.InstancesController|Api.V1.ListsController|Api.V1.MarkersController|Api.V1.NotificationsController|Api.V1.Statuses.FavouritedByAccountsController|Api.V1.Statuses.FavouritesController|Api.V1.Statuses.PinsController|Api.V1.Statuses.RebloggedByAccountsController|Api.V1.Statuses.ReblogsController|Api.V1.StatusesController|Api.V1.TagsController|Api.V1.Timelines.HomeController|Api.V1.Timelines.PublicController|Api.V1.Timelines.TagController|Api.V1.Trends.TagsController|Api.V2.InstancesController|Api.V2.MediaController|Api.V2.SearchController|Api.V2.SuggestionsController|Api.Web.SettingsController|ApplicationController|AuthorizeInteractionsController|CustomCssController|FollowerAccountsController|FollowingAccountsController|HomeController|InstanceActorsController|ManifestsController|Settings.Preferences.AppearanceController|Settings.ProfilesController|StatusesController|TagsController|WellKnown.NodeInfoController|WellKnown.WebfingerController)\"}[$__rate_interval]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Puma Controller Return Codes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The total amount of HTTP Queries that Mastodon is serving.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "/sec" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 28 + }, + "id": 33, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sort(sum by (mastodon)(rate(mastodon_controller_duration_count[$__rate_interval])))", + "instant": false, + "legendFormat": "QPS", + "range": true, + "refId": "A" + } + ], + "title": "Puma Queries", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of queries/sec of API calls served by the Mastodon web server.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (controller) (rate(mastodon_controller_duration_count{controller=~\"(AboutController|AccountsController|ActivityPub.CollectionsController|ActivityPub.InboxesController|ActivityPub.OutboxesController|ActivityPub.RepliesController|Admin.DashboardController|Admin.Settings.AboutController|Admin.Settings.BrandingController|Api.OEmbedController|Api.V1.Accounts.FeaturedTagsController|Api.V1.Accounts.FollowerAccountsController|Api.V1.Accounts.LookupController|Api.V1.Accounts.RelationshipsController|Api.V1.Accounts.SearchController|Api.V1.Accounts.StatusesController|Api.V1.AccountsController|Api.V1.Admin.DimensionsController|Api.V1.Admin.MeasuresController|Api.V1.Admin.RetentionController|Api.V1.Admin.Trends.TagsController|Api.V1.AnnouncementsController|Api.V1.ConversationsController|Api.V1.CustomEmojisController|Api.V1.DirectoriesController|Api.V1.FollowRequestsController|Api.V1.Instances.ExtendedDescriptionsController|Api.V1.InstancesController|Api.V1.ListsController|Api.V1.MarkersController|Api.V1.NotificationsController|Api.V1.Statuses.FavouritedByAccountsController|Api.V1.Statuses.FavouritesController|Api.V1.Statuses.PinsController|Api.V1.Statuses.RebloggedByAccountsController|Api.V1.Statuses.ReblogsController|Api.V1.StatusesController|Api.V1.TagsController|Api.V1.Timelines.HomeController|Api.V1.Timelines.PublicController|Api.V1.Timelines.TagController|Api.V1.Trends.TagsController|Api.V2.InstancesController|Api.V2.MediaController|Api.V2.SearchController|Api.V2.SuggestionsController|Api.Web.SettingsController|ApplicationController|AuthorizeInteractionsController|CustomCssController|FollowerAccountsController|FollowingAccountsController|HomeController|InstanceActorsController|ManifestsController|Settings.Preferences.AppearanceController|Settings.ProfilesController|StatusesController|TagsController|WellKnown.NodeInfoController|WellKnown.WebfingerController)\"}[$__rate_interval]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Puma Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Current latency of API calls served by the Mastodon web server.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (controller) (rate(mastodon_controller_duration_sum{controller=~\"(AboutController|AccountsController|ActivityPub.CollectionsController|ActivityPub.InboxesController|ActivityPub.OutboxesController|ActivityPub.RepliesController|Admin.DashboardController|Admin.Settings.AboutController|Admin.Settings.BrandingController|Api.OEmbedController|Api.V1.Accounts.FeaturedTagsController|Api.V1.Accounts.FollowerAccountsController|Api.V1.Accounts.LookupController|Api.V1.Accounts.RelationshipsController|Api.V1.Accounts.SearchController|Api.V1.Accounts.StatusesController|Api.V1.AccountsController|Api.V1.Admin.DimensionsController|Api.V1.Admin.MeasuresController|Api.V1.Admin.RetentionController|Api.V1.Admin.Trends.TagsController|Api.V1.AnnouncementsController|Api.V1.ConversationsController|Api.V1.CustomEmojisController|Api.V1.DirectoriesController|Api.V1.FollowRequestsController|Api.V1.Instances.ExtendedDescriptionsController|Api.V1.InstancesController|Api.V1.ListsController|Api.V1.MarkersController|Api.V1.NotificationsController|Api.V1.Statuses.FavouritedByAccountsController|Api.V1.Statuses.FavouritesController|Api.V1.Statuses.PinsController|Api.V1.Statuses.RebloggedByAccountsController|Api.V1.Statuses.ReblogsController|Api.V1.StatusesController|Api.V1.TagsController|Api.V1.Timelines.HomeController|Api.V1.Timelines.PublicController|Api.V1.Timelines.TagController|Api.V1.Trends.TagsController|Api.V2.InstancesController|Api.V2.MediaController|Api.V2.SearchController|Api.V2.SuggestionsController|Api.Web.SettingsController|ApplicationController|AuthorizeInteractionsController|CustomCssController|FollowerAccountsController|FollowingAccountsController|HomeController|InstanceActorsController|ManifestsController|Settings.Preferences.AppearanceController|Settings.ProfilesController|StatusesController|TagsController|WellKnown.NodeInfoController|WellKnown.WebfingerController)\"}[$__rate_interval])) /\nsum by (controller) (rate(mastodon_controller_duration_count{controller=~\"(AboutController|AccountsController|ActivityPub.CollectionsController|ActivityPub.InboxesController|ActivityPub.OutboxesController|ActivityPub.RepliesController|Admin.DashboardController|Admin.Settings.AboutController|Admin.Settings.BrandingController|Api.OEmbedController|Api.V1.Accounts.FeaturedTagsController|Api.V1.Accounts.FollowerAccountsController|Api.V1.Accounts.LookupController|Api.V1.Accounts.RelationshipsController|Api.V1.Accounts.SearchController|Api.V1.Accounts.StatusesController|Api.V1.AccountsController|Api.V1.Admin.DimensionsController|Api.V1.Admin.MeasuresController|Api.V1.Admin.RetentionController|Api.V1.Admin.Trends.TagsController|Api.V1.AnnouncementsController|Api.V1.ConversationsController|Api.V1.CustomEmojisController|Api.V1.DirectoriesController|Api.V1.FollowRequestsController|Api.V1.Instances.ExtendedDescriptionsController|Api.V1.InstancesController|Api.V1.ListsController|Api.V1.MarkersController|Api.V1.NotificationsController|Api.V1.Statuses.FavouritedByAccountsController|Api.V1.Statuses.FavouritesController|Api.V1.Statuses.PinsController|Api.V1.Statuses.RebloggedByAccountsController|Api.V1.Statuses.ReblogsController|Api.V1.StatusesController|Api.V1.TagsController|Api.V1.Timelines.HomeController|Api.V1.Timelines.PublicController|Api.V1.Timelines.TagController|Api.V1.Trends.TagsController|Api.V2.InstancesController|Api.V2.MediaController|Api.V2.SearchController|Api.V2.SuggestionsController|Api.Web.SettingsController|ApplicationController|AuthorizeInteractionsController|CustomCssController|FollowerAccountsController|FollowingAccountsController|HomeController|InstanceActorsController|ManifestsController|Settings.Preferences.AppearanceController|Settings.ProfilesController|StatusesController|TagsController|WellKnown.NodeInfoController|WellKnown.WebfingerController)\"}[$__rate_interval]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Puma Operations Latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 97, + "panels": [], + "title": "Media Storage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total size of all files in the media storage bucket.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 95, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "json_linode_size", + "legendFormat": "{{hostname}}", + "range": true, + "refId": "A" + } + ], + "title": "Linode Bucket Size", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of files in the media storage bucket.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 93, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "json_linode_objects", + "instant": false, + "interval": "", + "legendFormat": "{{hostname}}", + "range": true, + "refId": "A" + } + ], + "title": "Linode Bucket Objects", + "transformations": [], + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 7, + "panels": [], + "title": "Database", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of SQL statements per second by table.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(mastodon_db_operation_count{table=~\"(account|accounts|announcements|blocks|bookmarks|conversation|conversations|custom|domain|favourites|featured|follows|instances|ip|lists|markers|media|mentions|mutes|notifications|oauth|pg|poll|polls|preview|relays|reports|rules|scheduled|session|settings|site|status|statuses|tag|tags|tombstones|unavailable|user|users|web)\",operation=~\"(delete|insert|select|update)\"}[$__rate_interval])", + "legendFormat": "{{table}}.{{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "Postgres Queries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Latency of database operations by table.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(mastodon_db_operation_sum{table=~\"(account|accounts|announcements|blocks|bookmarks|conversation|conversations|custom|domain|favourites|featured|follows|instances|ip|lists|markers|media|mentions|mutes|notifications|oauth|pg|poll|polls|preview|relays|reports|rules|scheduled|session|settings|site|status|statuses|tag|tags|tombstones|unavailable|user|users|web)\",operation=~\"(delete|insert|select|update)\"}[$__rate_interval]) / \nrate(mastodon_db_operation_count{table=~\"(account|accounts|announcements|blocks|bookmarks|conversation|conversations|custom|domain|favourites|featured|follows|instances|ip|lists|markers|media|mentions|mutes|notifications|oauth|pg|poll|polls|preview|relays|reports|rules|scheduled|session|settings|site|status|statuses|tag|tags|tombstones|unavailable|user|users|web)\",operation=~\"(delete|insert|select|update)\"}[$__rate_interval])", + "legendFormat": "{{table}}.{{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "Postgres Queries Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total amount of SQL operations across Mastodon.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (operation) (rate(mastodon_db_operation_count{operation=~\"(delete|insert|select|update)\"}[$__rate_interval]))", + "legendFormat": "{{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "Postgres Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Running latency of SQL operations across Mastodon.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 54 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (operation) (rate(mastodon_db_operation_sum{operation=~\"(delete|insert|select|update)\"}[$__rate_interval])) / \nsum by (operation) (rate(mastodon_db_operation_count{operation=~\"(delete|insert|select|update)\"}[$__rate_interval]))", + "legendFormat": "{{operation}}", + "range": true, + "refId": "A" + } + ], + "title": "Postgres Operations Latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 104, + "panels": [], + "title": "Search", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of documents indexed by ElasticSearch.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepBefore", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 101, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(elasticsearch_indices_docs)", + "instant": false, + "interval": "", + "legendFormat": "indexed", + "range": true, + "refId": "A" + } + ], + "title": "ElasticSearch Documents", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Disk space used by ElasticSearch index.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepBefore", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 62 + }, + "id": 102, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(elasticsearch_indices_store_size_bytes)", + "legendFormat": "size", + "range": true, + "refId": "A" + } + ], + "title": "ElasticSearch Index Size", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 87, + "panels": [], + "title": "Cache", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Cache operations per second from the Mastodon service.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 70 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (operation)(rate(mastodon_cache_duration_count{operation=~\"(delete|fetch_hit|generate|increment|read_hit|read_miss|read_multi|write)\"}[$__rate_interval]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Cache Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Running latency of the Mastodon cache.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 70 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (operation)(rate(mastodon_cache_duration_sum{operation=~\"(delete|fetch_hit|generate|increment|read_hit|read_miss|read_multi|write)\"}[$__rate_interval])) / \nsum by (operation)(rate(mastodon_cache_duration_count{operation=~\"(delete|fetch_hit|generate|increment|read_hit|read_miss|read_multi|write)\"}[$__rate_interval]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Cache Latency", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 37, + "style": "dark", + "tags": [ + "public" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "nowDelay": "" + }, + "timezone": "America/New_York", + "title": "Mastodon public", + "uid": "IXDZT_dV9", + "version": 266, + "weekStart": "sunday" +} \ No newline at end of file diff --git a/etc/nginx/nginx.conf b/etc/nginx/nginx.conf new file mode 100644 index 0000000..21c1701 --- /dev/null +++ b/etc/nginx/nginx.conf @@ -0,0 +1,50 @@ +user mastodon; # jake: changed from nginx +worker_processes auto; +pid /run/nginx.pid; + +# jake: added +load_module modules/ngx_http_brotli_filter_module.so; +load_module modules/ngx_http_brotli_static_module.so; + +events { + worker_connections 768; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + ## + # Basic Settings + ## + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + # jake: added (prometheus target) + server { + listen 9181; + location /metrics { + stub_status on; + allow 127.0.0.1; + deny all; + } + } + + ## + # Logging Settings + ## + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + ## + # Virtual Host Configs + ## + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} diff --git a/etc/nginx/sites-available/mastodon.conf b/etc/nginx/sites-available/mastodon.conf index 72f03e4..8ba8226 100644 --- a/etc/nginx/sites-available/mastodon.conf +++ b/etc/nginx/sites-available/mastodon.conf @@ -155,6 +155,10 @@ server { add_header Strict-Transport-Security "max-age=63072000; includeSubDomains"; + # jake: added (debugging) + add_header Via $proxy_host; + add_header X-Got-Milk "2%"; + tcp_nodelay on; } @@ -178,9 +182,10 @@ server { proxy_cache_valid 200 7d; proxy_cache_valid 410 24h; proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; - add_header X-Cached $upstream_cache_status; - # jake: added :) + # jake: added (debugging) + add_header Via $proxy_host; + add_header X-Cache-Status $upstream_cache_status; add_header X-Got-Milk "2%"; tcp_nodelay on; diff --git a/etc/prometheus/exporter_configs/json-config.yml b/etc/prometheus/exporter_configs/json-config.yml new file mode 100644 index 0000000..c15a552 --- /dev/null +++ b/etc/prometheus/exporter_configs/json-config.yml @@ -0,0 +1,28 @@ +modules: + linode_bucket: + headers: + # https://cloud.linode.com/profile/tokens + Authorization: "Bearer XXXXXX" + metrics: + - name: json_linode_size + path: "{.size}" + labels: + bucket: "{.label}" + zone: "{.cluster}" + hostname: "{.hostname}" + - name: json_linode_objects + path: "{.objects}" + labels: + bucket: "{.label}" + zone: "{.cluster}" + hostname: "{.hostname}" + + linode_transfer: + headers: + # https://cloud.linode.com/profile/tokens + Authorization: "Bearer XXXXXX" + metrics: + - name: json_linode_transfer_used + path: "{.used}" + - name: json_linode_transfer_quota + path: "{.quota}" diff --git a/etc/prometheus/exporter_configs/statsd-mapping.yml b/etc/prometheus/exporter_configs/statsd-mapping.yml new file mode 100644 index 0000000..1badaa6 --- /dev/null +++ b/etc/prometheus/exporter_configs/statsd-mapping.yml @@ -0,0 +1,95 @@ +## Prometheus Statsd Exporter mapping for Mastodon 4.0+ +## +## Version 1.0, November 2022 +## +## Documentation: https://ipng.ch/s/articles/2022/11/27/mastodon-3.html + +mappings: + ## Web collector + - match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.status\.(.+) + match_type: regex + name: "mastodon_controller_status" + labels: + controller: $1 + action: $2 + format: $3 + status: $4 + mastodon: "web" + - match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.db_time + match_type: regex + name: "mastodon_controller_db_time" + labels: + controller: $1 + action: $2 + format: $3 + mastodon: "web" + - match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.view_time + match_type: regex + name: "mastodon_controller_view_time" + labels: + controller: $1 + action: $2 + format: $3 + mastodon: "web" + - match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.total_duration + match_type: regex + name: "mastodon_controller_duration" + labels: + controller: $1 + action: $2 + format: $3 + mastodon: "web" + + ## Database collector + - match: Mastodon\.production\.db\.tables\.(.+)\.queries\.(.+)\.duration + match_type: regex + name: "mastodon_db_operation" + labels: + table: "$1" + operation: "$2" + mastodon: "db" + + ## Cache collector + - match: Mastodon\.production\.cache\.(.+)\.duration + match_type: regex + name: "mastodon_cache_duration" + labels: + operation: "$1" + mastodon: "cache" + + ## Sidekiq collector + - match: Mastodon\.production\.sidekiq\.(.+)\.processing_time + match_type: regex + name: "mastodon_sidekiq_worker_processing_time" + labels: + worker: "$1" + mastodon: "sidekiq" + - match: Mastodon\.production\.sidekiq\.(.+)\.success + match_type: regex + name: "mastodon_sidekiq_worker_success_total" + labels: + worker: "$1" + mastodon: "sidekiq" + - match: Mastodon\.production\.sidekiq\.(.+)\.failure + match_type: regex + name: "mastodon_sidekiq_worker_failure_total" + labels: + worker: "$1" + mastodon: "sidekiq" + - match: Mastodon\.production\.sidekiq\.queues\.(.+)\.enqueued + match_type: regex + name: "mastodon_sidekiq_queue_enqueued" + labels: + queue: "$1" + mastodon: "sidekiq" + - match: Mastodon\.production\.sidekiq\.queues\.(.+)\.latency + match_type: regex + name: "mastodon_sidekiq_queue_latency" + labels: + queue: "$1" + mastodon: "sidekiq" + - match: Mastodon\.production\.sidekiq\.(.+) + match_type: regex + name: "mastodon_sidekiq_$1" + labels: + mastodon: "sidekiq" diff --git a/etc/prometheus/prometheus.yml b/etc/prometheus/prometheus.yml new file mode 100644 index 0000000..3c14d58 --- /dev/null +++ b/etc/prometheus/prometheus.yml @@ -0,0 +1,80 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] + + - job_name: "node_exporter_metrics" + static_configs: + - targets: ["localhost:9100"] + + - job_name: "redis_exporter_targets" + static_configs: + - targets: ["redis://localhost:6379"] + metrics_path: /scrape + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: localhost:9121 + + - job_name: "redis_exporter_metrics" + static_configs: + - targets: ["localhost:9121"] + + - job_name: "postgres_exporter_metrics" + static_configs: + - targets: ["localhost:9187"] + + - job_name: "nginx_exporter_metrics" + static_configs: + - targets: ["localhost:9113"] + + - job_name: "statsd_exporter_metrics" + static_configs: + - targets: ["localhost:9102"] + + - job_name: "elasticsearch_exporter_metrics" + static_configs: + - targets: ["localhost:9114"] + + - job_name: "json_exporter_metrics" + static_configs: + - targets: ["localhost:9079"] + + - job_name: "json_exporter_targets" + metrics_path: /probe + scrape_interval: 30s + params: + module: [linode_bucket] + static_configs: + - targets: + - https://api.linode.com/v4/object-storage/buckets/us-east-1/jarvis-mastodon + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: localhost:9079 + + - job_name: "json_exporter_targets" + metrics_path: /probe + scrape_interval: 30s + params: + module: [linode_transfer] + static_configs: + - targets: + - https://api.linode.com/v4/account/transfer + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: localhost:9079 \ No newline at end of file diff --git a/notes/prometheus.md b/notes/prometheus.md index 93076e1..d63f0a6 100644 --- a/notes/prometheus.md +++ b/notes/prometheus.md @@ -98,7 +98,7 @@ scrape_configs: metrics_path: /probe scrape_interval: 30s params: - module: [linode] + module: [linode_bucket] static_configs: - targets: - https://api.linode.com/v4/object-storage/buckets/us-east-1/jarvis-mastodon @@ -109,6 +109,22 @@ scrape_configs: target_label: instance - target_label: __address__ replacement: localhost:9079 + + - job_name: "json_exporter_targets" + metrics_path: /probe + scrape_interval: 30s + params: + module: [linode_transfer] + static_configs: + - targets: + - https://api.linode.com/v4/account/transfer + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: localhost:9079 ``` --- @@ -117,7 +133,10 @@ json-config.yml: ```yml modules: - linode: + linode_bucket: + headers: + # https://cloud.linode.com/profile/tokens + Authorization: "Bearer XXXXXX" metrics: - name: json_linode_size path: "{.size}" @@ -125,7 +144,6 @@ modules: bucket: "{.label}" zone: "{.cluster}" hostname: "{.hostname}" - - name: json_linode_objects path: "{.objects}" labels: @@ -133,9 +151,15 @@ modules: zone: "{.cluster}" hostname: "{.hostname}" + linode_transfer: headers: # https://cloud.linode.com/profile/tokens Authorization: "Bearer XXXXXX" + metrics: + - name: json_linode_transfer_used + path: "{.used}" + - name: json_linode_transfer_quota + path: "{.quota}" ``` ---