robots.txt

User-agent: *
Disallow: /bin/
Disallow: /cgi-bin/
Disallow: /assets/
Disallow: /magus/auth/
Disallow: /ftp/MidnightBSD/distfiles/*.asc
Disallow: /ftp/MidnightBSD/distfiles/*.sig
Disallow: /ftp/
Disallow: /rspamd/
Disallow: /fgmunin/
Disallow: /roundcube/
Disallow: /opengrok/
Crawl-delay: 5
Sitemap: http://www.midnightbsd.org/sitemap.xml
Sitemap: http://www.midnightbsd.org/sitemap1.xml
Sitemap: http://www.midnightbsd.org/sitemap2.xml
Sitemap: http://www.midnightbsd.org/sitemap3.xml
Sitemap: http://www.midnightbsd.org/sitemap4.xml
Sitemap: http://www.midnightbsd.org/sitemap5.xml
Sitemap: http://www.midnightbsd.org/sitemap6.xml
Sitemap: http://www.midnightbsd.org/sitemap7.xml
Sitemap: http://www.midnightbsd.org/sitemap8.xml
Sitemap: http://www.midnightbsd.org/sitemap9.xml
Host: www.midnightbsd.org
User-agent: AhrefsBot 
Disallow: /

User-agent: The Knowledge AI
Disallow: /

User-agent: dotbot
Disallow: /magus/

User-agent: AhrefsBot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: Amazonbot
Disallow: /magus/
Disallow: /ftp/