Update robots.txt

avipars · Sep 8, 2024 · 8f3428e · 8f3428e
1 parent cbe27de
commit 8f3428e
Showing 1 changed file with 36 additions and 2 deletions.
diff --git a/robots.txt b/robots.txt
@@ -1,6 +1,40 @@
+User-agent: PetalBot
+Disallow: /
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: PerplexityBot
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+‍User-agent: Claude-Web
+Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
+User-agent: FacebookBot
+Disallow: /
+
+User-agent: PiplBot
+Disallow: /
+
 User-agent: *
-Allow: /
 Disallow: /404.html
 Disallow: /redirects.json
+Allow: /
+
 
-Sitemap: https://cs.aviparshan.com/sitemap.xml
+Sitemap: https://cs.aviparshan.com/sitemap.xml