From 30f958e1c50f5cd2ab2a4063c15987d90b2318d7 Mon Sep 17 00:00:00 2001 From: Jake Jarvis Date: Fri, 16 Dec 2022 09:11:15 -0500 Subject: [PATCH] specifically block Twitterbot from robots.txt --- etc/grafana/public-dashboard.json | 234 +++++++++++++++++++++--- etc/nginx/sites-available/mastodon.conf | 8 +- patches/robots.patch | 10 +- scripts/upgrade.sh | 2 +- 4 files changed, 226 insertions(+), 28 deletions(-) diff --git a/etc/grafana/public-dashboard.json b/etc/grafana/public-dashboard.json index 1fec44f..17b4097 100644 --- a/etc/grafana/public-dashboard.json +++ b/etc/grafana/public-dashboard.json @@ -1767,7 +1767,17 @@ "mode": "off" } }, - "mappings": [], + "mappings": [ + { + "options": { + "NaN": { + "index": 0, + "text": "0" + } + }, + "type": "value" + } + ], "thresholds": { "mode": "absolute", "steps": [ @@ -1806,7 +1816,8 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (worker)(rate(mastodon_sidekiq_worker_success_total{worker=~\"(ActiveJob..QueueAdapters..SidekiqAdapter..JobWrapper|ActivityPub..DeliveryWorker|ActivityPub..DistributionWorker|ActivityPub..FetchRepliesWorker|ActivityPub..LowPriorityDeliveryWorker|ActivityPub..ProcessingWorker|ActivityPub..RawDistributionWorker|ActivityPub..StatusUpdateDistributionWorker|ActivityPub..SynchronizeFeaturedCollectionWorker|ActivityPub..SynchronizeFeaturedTagsCollectionWorker|ActivityPub..UpdateDistributionWorker|Admin..SuspensionWorker|BlockWorker|DeleteMuteWorker|DistributionWorker|DomainBlockWorker|DomainClearMediaWorker|FeedInsertWorker|FetchReplyWorker|LinkCrawlWorker|LocalNotificationWorker|MergeWorker|MuteWorker|PollExpirationNotifyWorker|PushConversationWorker|PushUpdateWorker|RedownloadAvatarWorker|RedownloadHeaderWorker|RedownloadMediaWorker|RefollowWorker|RemoteAccountRefreshWorker|RemovalWorker|Scheduler..AccountsStatusesCleanupScheduler|Scheduler..FollowRecommendationsScheduler|Scheduler..IndexingScheduler|Scheduler..InstanceRefreshScheduler|Scheduler..IpCleanupScheduler|Scheduler..PgheroScheduler|Scheduler..ScheduledStatusesScheduler|Scheduler..SuspendedUserCleanupScheduler|Scheduler..Trends..RefreshScheduler|Scheduler..Trends..ReviewNotificationsScheduler|Scheduler..UserCleanupScheduler|Scheduler..VacuumScheduler|ThreadResolveWorker|UnmergeWorker|VerifyAccountLinksWorker|Web..PushNotificationWorker)\"}[$__rate_interval]))", + "expr": "sum by (worker)(rate(mastodon_sidekiq_worker_success_total[$__rate_interval]))", + "hide": false, "legendFormat": "{{worker}} Success", "range": true, "refId": "A" @@ -1817,7 +1828,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (worker)(rate(mastodon_sidekiq_worker_failure_total{worker=~\"(ActiveJob..QueueAdapters..SidekiqAdapter..JobWrapper|ActivityPub..DeliveryWorker|ActivityPub..DistributionWorker|ActivityPub..FetchRepliesWorker|ActivityPub..LowPriorityDeliveryWorker|ActivityPub..ProcessingWorker|ActivityPub..RawDistributionWorker|ActivityPub..StatusUpdateDistributionWorker|ActivityPub..SynchronizeFeaturedCollectionWorker|ActivityPub..SynchronizeFeaturedTagsCollectionWorker|ActivityPub..UpdateDistributionWorker|Admin..SuspensionWorker|BlockWorker|DeleteMuteWorker|DistributionWorker|DomainBlockWorker|DomainClearMediaWorker|FeedInsertWorker|FetchReplyWorker|LinkCrawlWorker|LocalNotificationWorker|MergeWorker|MuteWorker|PollExpirationNotifyWorker|PushConversationWorker|PushUpdateWorker|RedownloadAvatarWorker|RedownloadHeaderWorker|RedownloadMediaWorker|RefollowWorker|RemoteAccountRefreshWorker|RemovalWorker|Scheduler..AccountsStatusesCleanupScheduler|Scheduler..FollowRecommendationsScheduler|Scheduler..IndexingScheduler|Scheduler..InstanceRefreshScheduler|Scheduler..IpCleanupScheduler|Scheduler..PgheroScheduler|Scheduler..ScheduledStatusesScheduler|Scheduler..SuspendedUserCleanupScheduler|Scheduler..Trends..RefreshScheduler|Scheduler..Trends..ReviewNotificationsScheduler|Scheduler..UserCleanupScheduler|Scheduler..VacuumScheduler|ThreadResolveWorker|UnmergeWorker|VerifyAccountLinksWorker|Web..PushNotificationWorker)\"}[$__rate_interval]))", + "expr": "sum by (worker)(rate(mastodon_sidekiq_worker_failure_total[$__rate_interval]))", "hide": false, "legendFormat": "{{worker}} Failure", "range": true, @@ -1825,6 +1836,15 @@ } ], "title": "Sidekiq Workers", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + } + ], "type": "timeseries" }, { @@ -1869,7 +1889,17 @@ } }, "decimals": 1, - "mappings": [], + "mappings": [ + { + "options": { + "NaN": { + "index": 0, + "text": "0" + } + }, + "type": "value" + } + ], "thresholds": { "mode": "absolute", "steps": [ @@ -1909,13 +1939,22 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (worker)(rate(mastodon_sidekiq_worker_processing_time_sum{worker=~\"(ActiveJob..QueueAdapters..SidekiqAdapter..JobWrapper|ActivityPub..DeliveryWorker|ActivityPub..DistributionWorker|ActivityPub..FetchRepliesWorker|ActivityPub..LowPriorityDeliveryWorker|ActivityPub..ProcessingWorker|ActivityPub..RawDistributionWorker|ActivityPub..StatusUpdateDistributionWorker|ActivityPub..SynchronizeFeaturedCollectionWorker|ActivityPub..SynchronizeFeaturedTagsCollectionWorker|ActivityPub..UpdateDistributionWorker|Admin..SuspensionWorker|BlockWorker|DeleteMuteWorker|DistributionWorker|DomainBlockWorker|DomainClearMediaWorker|FeedInsertWorker|FetchReplyWorker|LinkCrawlWorker|LocalNotificationWorker|MergeWorker|MuteWorker|PollExpirationNotifyWorker|PushConversationWorker|PushUpdateWorker|RedownloadAvatarWorker|RedownloadHeaderWorker|RedownloadMediaWorker|RefollowWorker|RemoteAccountRefreshWorker|RemovalWorker|Scheduler..AccountsStatusesCleanupScheduler|Scheduler..FollowRecommendationsScheduler|Scheduler..IndexingScheduler|Scheduler..InstanceRefreshScheduler|Scheduler..IpCleanupScheduler|Scheduler..PgheroScheduler|Scheduler..ScheduledStatusesScheduler|Scheduler..SuspendedUserCleanupScheduler|Scheduler..Trends..RefreshScheduler|Scheduler..Trends..ReviewNotificationsScheduler|Scheduler..UserCleanupScheduler|Scheduler..VacuumScheduler|ThreadResolveWorker|UnmergeWorker|VerifyAccountLinksWorker|Web..PushNotificationWorker)\"}[$__rate_interval]))\n/ sum by (worker)(rate(mastodon_sidekiq_worker_processing_time_count{worker=~\"(ActiveJob..QueueAdapters..SidekiqAdapter..JobWrapper|ActivityPub..DeliveryWorker|ActivityPub..DistributionWorker|ActivityPub..FetchRepliesWorker|ActivityPub..LowPriorityDeliveryWorker|ActivityPub..ProcessingWorker|ActivityPub..RawDistributionWorker|ActivityPub..StatusUpdateDistributionWorker|ActivityPub..SynchronizeFeaturedCollectionWorker|ActivityPub..SynchronizeFeaturedTagsCollectionWorker|ActivityPub..UpdateDistributionWorker|Admin..SuspensionWorker|BlockWorker|DeleteMuteWorker|DistributionWorker|DomainBlockWorker|DomainClearMediaWorker|FeedInsertWorker|FetchReplyWorker|LinkCrawlWorker|LocalNotificationWorker|MergeWorker|MuteWorker|PollExpirationNotifyWorker|PushConversationWorker|PushUpdateWorker|RedownloadAvatarWorker|RedownloadHeaderWorker|RedownloadMediaWorker|RefollowWorker|RemoteAccountRefreshWorker|RemovalWorker|Scheduler..AccountsStatusesCleanupScheduler|Scheduler..FollowRecommendationsScheduler|Scheduler..IndexingScheduler|Scheduler..InstanceRefreshScheduler|Scheduler..IpCleanupScheduler|Scheduler..PgheroScheduler|Scheduler..ScheduledStatusesScheduler|Scheduler..SuspendedUserCleanupScheduler|Scheduler..Trends..RefreshScheduler|Scheduler..Trends..ReviewNotificationsScheduler|Scheduler..UserCleanupScheduler|Scheduler..VacuumScheduler|ThreadResolveWorker|UnmergeWorker|VerifyAccountLinksWorker|Web..PushNotificationWorker)\"}[$__rate_interval]))\n", + "expr": "sum by (worker)(rate(mastodon_sidekiq_worker_processing_time_sum[$__rate_interval]))\n/ sum by (worker)(rate(mastodon_sidekiq_worker_processing_time_count[$__rate_interval]))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Sidekiq Workers Latency", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + } + ], "type": "timeseries" }, { @@ -2150,13 +2189,28 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (status) (rate(mastodon_controller_status{controller=~\"(AboutController|AccountsController|ActivityPub.CollectionsController|ActivityPub.InboxesController|ActivityPub.OutboxesController|ActivityPub.RepliesController|Admin.DashboardController|Admin.Settings.AboutController|Admin.Settings.BrandingController|Api.OEmbedController|Api.V1.Accounts.FeaturedTagsController|Api.V1.Accounts.FollowerAccountsController|Api.V1.Accounts.LookupController|Api.V1.Accounts.RelationshipsController|Api.V1.Accounts.SearchController|Api.V1.Accounts.StatusesController|Api.V1.AccountsController|Api.V1.Admin.DimensionsController|Api.V1.Admin.MeasuresController|Api.V1.Admin.RetentionController|Api.V1.Admin.Trends.TagsController|Api.V1.AnnouncementsController|Api.V1.ConversationsController|Api.V1.CustomEmojisController|Api.V1.DirectoriesController|Api.V1.FollowRequestsController|Api.V1.Instances.ExtendedDescriptionsController|Api.V1.InstancesController|Api.V1.ListsController|Api.V1.MarkersController|Api.V1.NotificationsController|Api.V1.Statuses.FavouritedByAccountsController|Api.V1.Statuses.FavouritesController|Api.V1.Statuses.PinsController|Api.V1.Statuses.RebloggedByAccountsController|Api.V1.Statuses.ReblogsController|Api.V1.StatusesController|Api.V1.TagsController|Api.V1.Timelines.HomeController|Api.V1.Timelines.PublicController|Api.V1.Timelines.TagController|Api.V1.Trends.TagsController|Api.V2.InstancesController|Api.V2.MediaController|Api.V2.SearchController|Api.V2.SuggestionsController|Api.Web.SettingsController|ApplicationController|AuthorizeInteractionsController|CustomCssController|FollowerAccountsController|FollowingAccountsController|HomeController|InstanceActorsController|ManifestsController|Settings.Preferences.AppearanceController|Settings.ProfilesController|StatusesController|TagsController|WellKnown.NodeInfoController|WellKnown.WebfingerController)\"}[$__rate_interval]))", + "expr": "sum by (status) (rate(mastodon_controller_status[$__rate_interval]))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Puma Controller Return Codes", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "status" + } + }, + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + } + ], "type": "timeseries" }, { @@ -2304,14 +2358,29 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (controller) (rate(mastodon_controller_duration_count{controller=~\"(AboutController|AccountsController|ActivityPub.CollectionsController|ActivityPub.InboxesController|ActivityPub.OutboxesController|ActivityPub.RepliesController|Admin.AccountsController|Admin.AnnouncementsController|Admin.CustomEmojisController|Admin.DashboardController|Admin.DomainBlocksController|Admin.EmailDomainBlocksController|Admin.ExportDomainBlocksController|Admin.FollowRecommendationsController|Admin.InstancesController|Admin.InvitesController|Admin.RelaysController|Admin.ReportsController|Admin.RolesController|Admin.RulesController|Admin.Settings.AboutController|Admin.Settings.AppearanceController|Admin.Settings.BrandingController|Admin.Settings.ContentRetentionController|Admin.Settings.DiscoveryController|Admin.Settings.OtherController|Admin.Settings.RegistrationsController|Admin.Trends.StatusesController|Api.OEmbedController|Api.V1.Accounts.CredentialsController|Api.V1.Accounts.FeaturedTagsController|Api.V1.Accounts.FollowerAccountsController|Api.V1.Accounts.FollowingAccountsController|Api.V1.Accounts.IdentityProofsController|Api.V1.Accounts.ListsController|Api.V1.Accounts.LookupController|Api.V1.Accounts.RelationshipsController|Api.V1.Accounts.SearchController|Api.V1.Accounts.StatusesController|Api.V1.AccountsController|Api.V1.Admin.DimensionsController|Api.V1.Admin.MeasuresController|Api.V1.Admin.RetentionController|Api.V1.Admin.Trends.TagsController|Api.V1.AnnouncementsController|Api.V1.Apps.CredentialsController|Api.V1.BlocksController|Api.V1.BookmarksController|Api.V1.ConversationsController|Api.V1.CustomEmojisController|Api.V1.DirectoriesController|Api.V1.FavouritesController|Api.V1.FiltersController|Api.V1.FollowRequestsController|Api.V1.Instances.ActivityController|Api.V1.Instances.DomainBlocksController|Api.V1.Instances.ExtendedDescriptionsController|Api.V1.Instances.PeersController|Api.V1.Instances.PrivacyPoliciesController|Api.V1.Instances.RulesController|Api.V1.InstancesController|Api.V1.ListsController|Api.V1.MarkersController|Api.V1.MediaController|Api.V1.MutesController|Api.V1.NotificationsController|Api.V1.Polls.VotesController|Api.V1.PollsController|Api.V1.PreferencesController|Api.V1.Push.SubscriptionsController|Api.V1.Statuses.BookmarksController|Api.V1.Statuses.FavouritedByAccountsController|Api.V1.Statuses.FavouritesController|Api.V1.Statuses.PinsController|Api.V1.Statuses.RebloggedByAccountsController|Api.V1.Statuses.ReblogsController|Api.V1.Statuses.SourcesController|Api.V1.StatusesController|Api.V1.TagsController|Api.V1.Timelines.HomeController|Api.V1.Timelines.PublicController|Api.V1.Timelines.TagController|Api.V1.Trends.LinksController|Api.V1.Trends.StatusesController|Api.V1.Trends.TagsController|Api.V2.InstancesController|Api.V2.MediaController|Api.V2.SearchController|Api.V2.SuggestionsController|Api.Web.SettingsController|ApplicationController|Auth.ConfirmationsController|Auth.PasswordsController|Auth.RegistrationsController|Auth.SessionsController|AuthorizeInteractionsController|CustomCssController|FiltersController|FollowerAccountsController|FollowingAccountsController|HealthController|HomeController|InstanceActorsController|ManifestsController|MediaProxyController|Oauth.TokensController|Settings.FlavoursController|Settings.Preferences.AppearanceController|Settings.Preferences.NotificationsController|Settings.ProfilesController|StatusesController|TagsController|WellKnown.HostMetaController|WellKnown.NodeInfoController|WellKnown.WebfingerController)\"}[$__rate_interval]))", + "expr": "sum by (controller) (rate(mastodon_controller_duration_count[$__rate_interval]))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Puma Operations", - "transformations": [], + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "columns", + "valueLabel": "controller" + } + }, + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + } + ], "type": "timeseries" }, { @@ -2395,13 +2464,28 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (controller) (rate(mastodon_controller_duration_sum{controller=~\"(AboutController|AccountsController|ActivityPub.CollectionsController|ActivityPub.InboxesController|ActivityPub.OutboxesController|ActivityPub.RepliesController|Admin.AccountsController|Admin.AnnouncementsController|Admin.CustomEmojisController|Admin.DashboardController|Admin.DomainBlocksController|Admin.EmailDomainBlocksController|Admin.ExportDomainBlocksController|Admin.FollowRecommendationsController|Admin.InstancesController|Admin.InvitesController|Admin.RelaysController|Admin.ReportsController|Admin.RolesController|Admin.RulesController|Admin.Settings.AboutController|Admin.Settings.AppearanceController|Admin.Settings.BrandingController|Admin.Settings.ContentRetentionController|Admin.Settings.DiscoveryController|Admin.Settings.OtherController|Admin.Settings.RegistrationsController|Admin.Trends.StatusesController|Api.OEmbedController|Api.V1.Accounts.CredentialsController|Api.V1.Accounts.FeaturedTagsController|Api.V1.Accounts.FollowerAccountsController|Api.V1.Accounts.FollowingAccountsController|Api.V1.Accounts.IdentityProofsController|Api.V1.Accounts.ListsController|Api.V1.Accounts.LookupController|Api.V1.Accounts.RelationshipsController|Api.V1.Accounts.SearchController|Api.V1.Accounts.StatusesController|Api.V1.AccountsController|Api.V1.Admin.DimensionsController|Api.V1.Admin.MeasuresController|Api.V1.Admin.RetentionController|Api.V1.Admin.Trends.TagsController|Api.V1.AnnouncementsController|Api.V1.Apps.CredentialsController|Api.V1.BlocksController|Api.V1.BookmarksController|Api.V1.ConversationsController|Api.V1.CustomEmojisController|Api.V1.DirectoriesController|Api.V1.FavouritesController|Api.V1.FiltersController|Api.V1.FollowRequestsController|Api.V1.Instances.ActivityController|Api.V1.Instances.DomainBlocksController|Api.V1.Instances.ExtendedDescriptionsController|Api.V1.Instances.PeersController|Api.V1.Instances.PrivacyPoliciesController|Api.V1.Instances.RulesController|Api.V1.InstancesController|Api.V1.ListsController|Api.V1.MarkersController|Api.V1.MediaController|Api.V1.MutesController|Api.V1.NotificationsController|Api.V1.Polls.VotesController|Api.V1.PollsController|Api.V1.PreferencesController|Api.V1.Push.SubscriptionsController|Api.V1.Statuses.BookmarksController|Api.V1.Statuses.FavouritedByAccountsController|Api.V1.Statuses.FavouritesController|Api.V1.Statuses.PinsController|Api.V1.Statuses.RebloggedByAccountsController|Api.V1.Statuses.ReblogsController|Api.V1.Statuses.SourcesController|Api.V1.StatusesController|Api.V1.TagsController|Api.V1.Timelines.HomeController|Api.V1.Timelines.PublicController|Api.V1.Timelines.TagController|Api.V1.Trends.LinksController|Api.V1.Trends.StatusesController|Api.V1.Trends.TagsController|Api.V2.InstancesController|Api.V2.MediaController|Api.V2.SearchController|Api.V2.SuggestionsController|Api.Web.SettingsController|ApplicationController|Auth.ConfirmationsController|Auth.PasswordsController|Auth.RegistrationsController|Auth.SessionsController|AuthorizeInteractionsController|CustomCssController|FiltersController|FollowerAccountsController|FollowingAccountsController|HealthController|HomeController|InstanceActorsController|ManifestsController|MediaProxyController|Oauth.TokensController|Settings.FlavoursController|Settings.Preferences.AppearanceController|Settings.Preferences.NotificationsController|Settings.ProfilesController|StatusesController|TagsController|WellKnown.HostMetaController|WellKnown.NodeInfoController|WellKnown.WebfingerController)\"}[$__rate_interval])) /\nsum by (controller) (rate(mastodon_controller_duration_count{controller=~\"(AboutController|AccountsController|ActivityPub.CollectionsController|ActivityPub.InboxesController|ActivityPub.OutboxesController|ActivityPub.RepliesController|Admin.AccountsController|Admin.AnnouncementsController|Admin.CustomEmojisController|Admin.DashboardController|Admin.DomainBlocksController|Admin.EmailDomainBlocksController|Admin.ExportDomainBlocksController|Admin.FollowRecommendationsController|Admin.InstancesController|Admin.InvitesController|Admin.RelaysController|Admin.ReportsController|Admin.RolesController|Admin.RulesController|Admin.Settings.AboutController|Admin.Settings.AppearanceController|Admin.Settings.BrandingController|Admin.Settings.ContentRetentionController|Admin.Settings.DiscoveryController|Admin.Settings.OtherController|Admin.Settings.RegistrationsController|Admin.Trends.StatusesController|Api.OEmbedController|Api.V1.Accounts.CredentialsController|Api.V1.Accounts.FeaturedTagsController|Api.V1.Accounts.FollowerAccountsController|Api.V1.Accounts.FollowingAccountsController|Api.V1.Accounts.IdentityProofsController|Api.V1.Accounts.ListsController|Api.V1.Accounts.LookupController|Api.V1.Accounts.RelationshipsController|Api.V1.Accounts.SearchController|Api.V1.Accounts.StatusesController|Api.V1.AccountsController|Api.V1.Admin.DimensionsController|Api.V1.Admin.MeasuresController|Api.V1.Admin.RetentionController|Api.V1.Admin.Trends.TagsController|Api.V1.AnnouncementsController|Api.V1.Apps.CredentialsController|Api.V1.BlocksController|Api.V1.BookmarksController|Api.V1.ConversationsController|Api.V1.CustomEmojisController|Api.V1.DirectoriesController|Api.V1.FavouritesController|Api.V1.FiltersController|Api.V1.FollowRequestsController|Api.V1.Instances.ActivityController|Api.V1.Instances.DomainBlocksController|Api.V1.Instances.ExtendedDescriptionsController|Api.V1.Instances.PeersController|Api.V1.Instances.PrivacyPoliciesController|Api.V1.Instances.RulesController|Api.V1.InstancesController|Api.V1.ListsController|Api.V1.MarkersController|Api.V1.MediaController|Api.V1.MutesController|Api.V1.NotificationsController|Api.V1.Polls.VotesController|Api.V1.PollsController|Api.V1.PreferencesController|Api.V1.Push.SubscriptionsController|Api.V1.Statuses.BookmarksController|Api.V1.Statuses.FavouritedByAccountsController|Api.V1.Statuses.FavouritesController|Api.V1.Statuses.PinsController|Api.V1.Statuses.RebloggedByAccountsController|Api.V1.Statuses.ReblogsController|Api.V1.Statuses.SourcesController|Api.V1.StatusesController|Api.V1.TagsController|Api.V1.Timelines.HomeController|Api.V1.Timelines.PublicController|Api.V1.Timelines.TagController|Api.V1.Trends.LinksController|Api.V1.Trends.StatusesController|Api.V1.Trends.TagsController|Api.V2.InstancesController|Api.V2.MediaController|Api.V2.SearchController|Api.V2.SuggestionsController|Api.Web.SettingsController|ApplicationController|Auth.ConfirmationsController|Auth.PasswordsController|Auth.RegistrationsController|Auth.SessionsController|AuthorizeInteractionsController|CustomCssController|FiltersController|FollowerAccountsController|FollowingAccountsController|HealthController|HomeController|InstanceActorsController|ManifestsController|MediaProxyController|Oauth.TokensController|Settings.FlavoursController|Settings.Preferences.AppearanceController|Settings.Preferences.NotificationsController|Settings.ProfilesController|StatusesController|TagsController|WellKnown.HostMetaController|WellKnown.NodeInfoController|WellKnown.WebfingerController)\"}[$__rate_interval]))", + "expr": "sum by (controller) (rate(mastodon_controller_duration_sum[$__rate_interval]))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Puma Operations Latency", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "controller" + } + }, + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + } + ], "type": "timeseries" }, { @@ -2450,7 +2534,7 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": false, + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -2541,7 +2625,7 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": false, + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -2695,13 +2779,22 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(mastodon_db_operation_count{table=~\"(account|accounts|admin|announcement|announcements|appeals|ar|backups|blocks|bookmarks|conversation|conversations|counties|custom|devices|domain|email|favourites|featured|follow|follows|imports|instances|invites|ip|list|lists|login|markers|media|mentions|mutes|notifications|oauth|pg|pghero|poll|polls|preview|relays|report|reports|rules|scheduled|schema|session|settings|site|status|statuses|tag|tags|tombstones|unavailable|user|users|web)\",operation=~\"(delete|insert|select|update)\"}[$__rate_interval])", + "expr": "rate(mastodon_db_operation_count[$__rate_interval])", "legendFormat": "{{table}}.{{operation}}", "range": true, "refId": "A" } ], "title": "Postgres Queries", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + } + ], "type": "timeseries" }, { @@ -2745,7 +2838,17 @@ "mode": "off" } }, - "mappings": [], + "mappings": [ + { + "options": { + "NaN": { + "index": 0, + "text": "0" + } + }, + "type": "value" + } + ], "thresholds": { "mode": "absolute", "steps": [ @@ -2784,13 +2887,22 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(mastodon_db_operation_sum{table=~\"(account|accounts|admin|announcement|announcements|appeals|ar|backups|blocks|bookmarks|conversation|conversations|counties|custom|devices|domain|email|favourites|featured|follow|follows|imports|instances|invites|ip|list|lists|login|markers|media|mentions|mutes|notifications|oauth|pg|pghero|poll|polls|preview|relays|report|reports|rules|scheduled|schema|session|settings|site|status|statuses|tag|tags|tombstones|unavailable|user|users|web)\",operation=~\"(delete|insert|select|update)\"}[$__rate_interval]) / \nrate(mastodon_db_operation_count{table=~\"(account|accounts|admin|announcement|announcements|appeals|ar|backups|blocks|bookmarks|conversation|conversations|counties|custom|devices|domain|email|favourites|featured|follow|follows|imports|instances|invites|ip|list|lists|login|markers|media|mentions|mutes|notifications|oauth|pg|pghero|poll|polls|preview|relays|report|reports|rules|scheduled|schema|session|settings|site|status|statuses|tag|tags|tombstones|unavailable|user|users|web)\",operation=~\"(delete|insert|select|update)\"}[$__rate_interval])", + "expr": "rate(mastodon_db_operation_sum[$__rate_interval]) / \nrate(mastodon_db_operation_count[$__rate_interval])", "legendFormat": "{{table}}.{{operation}}", "range": true, "refId": "A" } ], "title": "Postgres Queries Latency", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + } + ], "type": "timeseries" }, { @@ -2873,13 +2985,28 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (operation) (rate(mastodon_db_operation_count{operation=~\"(delete|insert|select|update)\"}[$__rate_interval]))", + "expr": "sum by (operation) (rate(mastodon_db_operation_count[$__rate_interval]))", "legendFormat": "{{operation}}", "range": true, "refId": "A" } ], "title": "Postgres Operations", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "operation" + } + }, + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + } + ], "type": "timeseries" }, { @@ -2927,7 +3054,17 @@ } }, "decimals": 1, - "mappings": [], + "mappings": [ + { + "options": { + "NaN": { + "index": 0, + "text": "0" + } + }, + "type": "value" + } + ], "thresholds": { "mode": "absolute", "steps": [ @@ -2966,13 +3103,28 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (operation) (rate(mastodon_db_operation_sum{operation=~\"(delete|insert|select|update)\"}[$__rate_interval])) / \nsum by (operation) (rate(mastodon_db_operation_count{operation=~\"(delete|insert|select|update)\"}[$__rate_interval]))", + "expr": "sum by (operation) (rate(mastodon_db_operation_sum[$__rate_interval])) / \nsum by (operation) (rate(mastodon_db_operation_count[$__rate_interval]))", "legendFormat": "{{operation}}", "range": true, "refId": "A" } ], "title": "Postgres Operations Latency", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "operation" + } + }, + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + } + ], "type": "timeseries" }, { @@ -3267,13 +3419,28 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (operation)(rate(mastodon_cache_duration_count{operation=~\"(delete|delete_matched|delete_multi|fetch_hit|generate|increment|read_hit|read_miss|read_multi|write)\"}[$__rate_interval]))", + "expr": "sum by (operation)(rate(mastodon_cache_duration_count[$__rate_interval]))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Cache Operations", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "operation" + } + }, + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + } + ], "type": "timeseries" }, { @@ -3318,7 +3485,17 @@ } }, "decimals": 1, - "mappings": [], + "mappings": [ + { + "options": { + "NaN": { + "index": 0, + "text": "0" + } + }, + "type": "value" + } + ], "thresholds": { "mode": "absolute", "steps": [ @@ -3357,13 +3534,28 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (operation)(rate(mastodon_cache_duration_sum{operation=~\"(delete|delete_matched|delete_multi|fetch_hit|generate|increment|read_hit|read_miss|read_multi|write)\"}[$__rate_interval])) / \nsum by (operation)(rate(mastodon_cache_duration_count{operation=~\"(delete|delete_matched|delete_multi|fetch_hit|generate|increment|read_hit|read_miss|read_multi|write)\"}[$__rate_interval]))", + "expr": "sum by (operation)(rate(mastodon_cache_duration_sum[$__rate_interval])) / \nsum by (operation)(rate(mastodon_cache_duration_count[$__rate_interval]))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Cache Latency", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "operation" + } + }, + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + } + ], "type": "timeseries" } ], @@ -3386,6 +3578,6 @@ "timezone": "America/New_York", "title": "Mastodon public", "uid": "IXDZT_dV9", - "version": 285, + "version": 299, "weekStart": "sunday" } \ No newline at end of file diff --git a/etc/nginx/sites-available/mastodon.conf b/etc/nginx/sites-available/mastodon.conf index 521f294..924a5a9 100644 --- a/etc/nginx/sites-available/mastodon.conf +++ b/etc/nginx/sites-available/mastodon.conf @@ -44,24 +44,24 @@ server { gzip_comp_level 6; gzip_buffers 16 8k; gzip_http_version 1.1; - gzip_types application/atom+xml application/javascript application/json application/rss+xml + gzip_min_length 256; + gzip_types application/atom+xml application/javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype application/x-font-ttf application/x-javascript application/xhtml+xml application/xml font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml; - gzip_min_length 256; # https://github.com/google/ngx_brotli#sample-configuration # https://github.com/jakejarvis/mastodon-utils/wiki/Brotli-compression-for-nginx brotli on; brotli_comp_level 4; brotli_static on; - brotli_types application/atom+xml application/javascript application/json application/rss+xml + brotli_min_length 256; + brotli_types application/atom+xml application/javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype application/x-font-ttf application/x-javascript application/xhtml+xml application/xml font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml; - brotli_min_length 256; # add shortcut to public Grafana dashboard location ~ ^/dashboard/?$ { diff --git a/patches/robots.patch b/patches/robots.patch index b1c9439..2c69096 100644 --- a/patches/robots.patch +++ b/patches/robots.patch @@ -1,11 +1,17 @@ diff --git a/public/robots.txt b/public/robots.txt -index 771bf2160..1f53798bb 100644 +index 771bf2160b..af16ddb0ce 100644 --- a/public/robots.txt +++ b/public/robots.txt -@@ -1,5 +1,2 @@ +@@ -1,5 +1,8 @@ -# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file - ++# block everything except About page User-agent: * -Disallow: /media_proxy/ -Disallow: /interact/ ++Allow: /about ++Disallow: / ++ ++# sorry, Elon ++User-agent: Twitterbot +Disallow: / diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index 3e4ec09..3989e7e 100755 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -34,7 +34,7 @@ fi # create blank custom.css (this overrides any CSS set in the admin panel, but if that's not being used, then # this quickly saves a request to the backend) -sudo -u mastodon "$APP_ROOT/public/custom.css" +sudo -u mastodon touch "$APP_ROOT/public/custom.css" # set new ruby version RUBY_VERSION="$(sudo -u mastodon cat $APP_ROOT/.ruby-version)"