Merge pull request #16 from nisbet-hubbard/patch-1

Remove potential false positives
ai-robots-txt · Jun 20, 2024 · 4b8499d · 4b8499d
2 parents 429336d + 56c2285
commit 4b8499d
Showing 1 changed file with 0 additions and 6 deletions.
diff --git a/robots.txt b/robots.txt
@@ -2,8 +2,6 @@ User-agent: AdsBot-Google
 User-agent: Amazonbot
 User-agent: anthropic-ai
 User-agent: Applebot-Extended
-User-agent: AwarioRssBot
-User-agent: AwarioSmartBot
 User-agent: Bytespider
 User-agent: CCBot
 User-agent: ChatGPT-User
@@ -19,15 +17,11 @@ User-agent: GoogleOther
 User-agent: GPTBot
 User-agent: img2dataset
 User-agent: ImagesiftBot
-User-agent: magpie-crawler
-User-agent: Meltwater
 User-agent: omgili
 User-agent: omgilibot
 User-agent: peer39_crawler
 User-agent: peer39_crawler/1.0
 User-agent: PerplexityBot
-User-agent: PiplBot
-User-agent: scoop.it
 User-agent: Seekr
 User-agent: YouBot
 Disallow: /