# #################################################################### # # AI bots # # @see https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/refs/heads/main/robots.txt # #################################################################### # User-agent: AI2Bot # User-agent: Ai2Bot-Dolma # User-agent: Amazonbot # User-agent: anthropic-ai # User-agent: Applebot # User-agent: Applebot-Extended # User-agent: Bytespider # User-agent: CCBot # User-agent: ChatGPT-User # User-agent: Claude-Web # User-agent: ClaudeBot # User-agent: cohere-ai # User-agent: cohere-training-data-crawler # User-agent: Diffbot # User-agent: DuckAssistBot # User-agent: FacebookBot # User-agent: FriendlyCrawler # User-agent: Google-Extended # User-agent: GoogleOther # User-agent: GoogleOther-Image # User-agent: GoogleOther-Video # User-agent: GPTBot # User-agent: iaskspider/2.0 # User-agent: ICC-Crawler # User-agent: ImagesiftBot # User-agent: img2dataset # User-agent: ISSCyberRiskCrawler # User-agent: Kangaroo Bot # User-agent: Meta-ExternalAgent # User-agent: Meta-ExternalFetcher # User-agent: OAI-SearchBot # User-agent: omgili # User-agent: omgilibot # User-agent: PanguBot # User-agent: PerplexityBot # User-agent: PetalBot # User-agent: Scrapy # User-agent: SemrushBot # User-agent: Sidetrade indexer bot # User-agent: Timpibot # User-agent: VelenPublicWebCrawler # User-agent: Webzio-Extended # User-agent: YouBot # Disallow: / # # # #################################################################### # # Crawler bots # # @see https://github.com/leoloso/gatowebsites/pull/426 # #################################################################### # User-agent: BLEXBot # Disallow: /