From 5f90025493c756fe06936cf2021e6d387d258d97 Mon Sep 17 00:00:00 2001
From: Lixfel <lixfel@steamwar.de>
Date: Tue, 26 Nov 2024 15:31:44 +0100
Subject: [PATCH] Give AI bots the boot

Mix of

- https://blog.cloudflare.com/declaring-your-aindependence-block-ai-bots-scrapers-and-crawlers-with-a-single-click/
- https://www.google.com/robots.txt
---
 astro.config.mjs | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/astro.config.mjs b/astro.config.mjs
index b1e0ba8..edf7e0a 100644
--- a/astro.config.mjs
+++ b/astro.config.mjs
@@ -34,9 +34,27 @@ export default defineConfig({
             },
         }),
         robotsTxt({
-            policy: [{
-                userAgent: "*", disallow: ["/admin", "/login", "/dashboard", "/de", "/_astro"],
-            }],
+            policy: [
+                { userAgent: "*", disallow: ["/admin", "/login", "/dashboard", "/de", "/_astro"] },
+                { userAgent: "anthropic-ai", disallow: "/" },
+                { userAgent: "Applebot-Extended", disallow: "/" },
+                { userAgent: "Bytespider", disallow: "/" },
+                { userAgent: "CCBot", disallow: "/" },
+                { userAgent: "ChatGPT-User", disallow: "/" },
+                { userAgent: "ClaudeBot", disallow: "/" },
+                { userAgent: "Claude-Web", disallow: "/" },
+                { userAgent: "cohere-ai", disallow: "/" },
+                { userAgent: "Diffbot", disallow: "/" },
+                { userAgent: "FacebookBot", disallow: "/" },
+                { userAgent: "GPTBot", disallow: "/" },
+                { userAgent: "ImagesiftBot", disallow: "/" },
+                { userAgent: "Meta-ExternalAgent", disallow: "/" },
+                { userAgent: "Meta-ExternalFetcher", disallow: "/" },
+                { userAgent: "omgili", disallow: "/" },
+                { userAgent: "OmigliBot", disallow: "/" },
+                { userAgent: "PerplexityBot", disallow: "/" },
+                { userAgent: "Timpibot", disallow: "/" }
+            ],
         }),
         mdx(),
     ],