astro.config.mjs

@@ -34,9 +34,27 @@ export default defineConfig({
             },
         }),
         robotsTxt({
-            policy: [{
+            policy: [
-                userAgent: "*", disallow: ["/admin", "/login", "/dashboard", "/de", "/_astro"],
+                { userAgent: "*", disallow: ["/admin", "/login", "/dashboard", "/de", "/_astro"] },
-            }],
+                { userAgent: "anthropic-ai", disallow: "/" },
+                { userAgent: "Applebot-Extended", disallow: "/" },
+                { userAgent: "Bytespider", disallow: "/" },
+                { userAgent: "CCBot", disallow: "/" },
+                { userAgent: "ChatGPT-User", disallow: "/" },
+                { userAgent: "ClaudeBot", disallow: "/" },
+                { userAgent: "Claude-Web", disallow: "/" },
+                { userAgent: "cohere-ai", disallow: "/" },
+                { userAgent: "Diffbot", disallow: "/" },
+                { userAgent: "FacebookBot", disallow: "/" },
+                { userAgent: "GPTBot", disallow: "/" },
+                { userAgent: "ImagesiftBot", disallow: "/" },
+                { userAgent: "Meta-ExternalAgent", disallow: "/" },
+                { userAgent: "Meta-ExternalFetcher", disallow: "/" },
+                { userAgent: "omgili", disallow: "/" },
+                { userAgent: "OmigliBot", disallow: "/" },
+                { userAgent: "PerplexityBot", disallow: "/" },
+                { userAgent: "Timpibot", disallow: "/" }
+            ],
         }),
         mdx(),
     ],