block bots

app/controllers/application_controller.rb

@@ -1,4 +1,6 @@
 class ApplicationController < ActionController::Base
+  include BotBlocker
+
   # Changes to the importmap will invalidate the etag for HTML responses
   stale_when_importmap_changes
 

app/controllers/concerns/bot_blocker.rb

@@ -0,0 +1,56 @@
+module BotBlocker
+  extend ActiveSupport::Concern
+
+  included do
+    before_action :block_bots
+  end
+
+  private
+
+  def block_bots
+    return unless bot_request?
+
+    render plain: "Bot access is not allowed", status: :forbidden
+  end
+
+  def bot_request?
+    user_agent = request.user_agent.to_s.downcase
+
+    # List of known bot user agents
+    bot_patterns = [
+      'gptbot',           # OpenAI GPTBot
+      'chatgpt',          # ChatGPT
+      'claude-web',       # Anthropic Claude
+      'bingbot',          # Microsoft Bing
+      'googlebot',        # Google
+      'baiduspider',      # Baidu
+      'yandexbot',        # Yandex
+      'duckduckbot',      # DuckDuckGo
+      'slurp',            # Yahoo
+      'facebookexternalhit', # Facebook
+      'twitterbot',       # Twitter
+      'linkedinbot',      # LinkedIn
+      'whatsapp',         # WhatsApp
+      'telegrambot',      # Telegram
+      'slackbot',         # Slack
+      'discordbot',       # Discord
+      'applebot',         # Apple
+      'ia_archiver',      # Alexa/Internet Archive
+      'petalbot',         # Huawei
+      'seznambot',        # Seznam
+      'ahrefsbot',        # Ahrefs
+      'semrushbot',       # SEMrush
+      'mj12bot',          # Majestic
+      'dotbot',           # OpenSiteExplorer
+      'rogerbot',         # Moz
+      'exabot',           # Exalead
+      'facebot',          # Facebook
+      'spider',           # Generic spiders
+      'crawler',          # Generic crawlers
+      'scraper',          # Generic scrapers
+      'bot',              # Generic bots (last resort)
+    ]
+
+    bot_patterns.any? { |pattern| user_agent.include?(pattern) }
+  end
+end