# Monitoring and Uptime bots # Generally considered harmless, these bots are intended to simply check links still work and to alert a website owner if their website goes offline. User-agent: ahrefsbot Allow: / User-agent: BetterStackBot Allow: / User-agent: cloudflare Allow: / User-agent: cron-job.org Allow: / User-agent: Pingdom.com_bot Allow: / User-agent: uptimerobot Allow: / User-agent: urlresolver Allow: / User-agent: w3c_validator Allow: / User-agent: wordpress Allow: / User-agent: wpbot Allow: / User-agent: wprocketbot Allow: / # Security and Vulnerability Scanners User-agent: BitSightBot Allow: / User-agent: censys.io Allow: / User-agent: shodan Allow: / # Search Engine Bots # These bots allow websites to show up in search results, or allow a minimal amount of information to be shared with a host site when a link is shared there. User-agent: adsbot-google Allow: / User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / User-agent: BingPreview Allow: / User-agent: Discordbot Allow: / User-agent: DuckDuckBot Allow: / User-agent: embedly Allow: / User-agent: facebookexternalhit Allow: / User-agent: quora Allow: / # Disallow List User-agent: a6-indexer Disallow: / Crawl-delay: 86400 User-agent: Addsearchbot Disallow: / Crawl-delay: 86400 User-agent: AdIdxBot Disallow: / Crawl-delay: 86400 User-agent: ADmantX Disallow: / Crawl-delay: 86400 User-agent: AhrefsBot Disallow: / Crawl-delay: 86400 User-agent: ai-crawler Disallow: / Crawl-delay: 86400 User-agent: AI2Bot Disallow: / Crawl-delay: 86400 User-agent: Ai2Bot-Dolma Disallow: / Crawl-delay: 86400 User-agent: aiHitBot Disallow: / Crawl-delay: 86400 User-agent: AlphaAI Disallow: / Crawl-delay: 86400 User-agent: Alphabot Disallow: / Crawl-delay: 86400 User-agent: Amazonbot Disallow: / Crawl-delay: 86400 User-agent: Andibot Disallow: / Crawl-delay: 86400 User-agent: anthropic-ai Disallow: / Crawl-delay: 86400 User-agent: aolbuild Disallow: / Crawl-delay: 86400 User-agent: AppleBot Disallow: / Crawl-delay: 86400 User-agent: Applebot-Extended Disallow: / Crawl-delay: 86400 User-agent: Awario Disallow: / Crawl-delay: 86400 User-agent: AwarioSmartBot Disallow: / Crawl-delay: 86400 User-agent: Baiduspider Disallow: / Crawl-delay: 86400 User-agent: bedrockbot Disallow: / Crawl-delay: 86400 User-agent: BigSur.ai Disallow: / Crawl-delay: 86400 User-agent: BLEXBot Disallow: / Crawl-delay: 86400 User-agent: Brightbot Disallow: / Crawl-delay: 86400 User-agent: Brightbot 1.0 Disallow: / Crawl-delay: 86400 User-agent: butterfly Disallow: / Crawl-delay: 86400 User-agent: Buzzbot Disallow: / Crawl-delay: 86400 User-agent: Bytespider Disallow: / Crawl-delay: 86400 User-agent: CCBot Disallow: / Crawl-delay: 86400 User-agent: ChatGPT Disallow: / Crawl-delay: 86400 User-agent: ChatGPT-User Disallow: / Crawl-delay: 86400 User-agent: claritybot Disallow: / Crawl-delay: 86400 User-agent: Claude-SearchBot Disallow: / Crawl-delay: 86400 User-agent: Claude-User Disallow: / Crawl-delay: 86400 User-agent: Claude-Web Disallow: / Crawl-delay: 86400 User-agent: ClaudeBot Disallow: / Crawl-delay: 86400 User-agent: cohere-ai Disallow: / Crawl-delay: 86400 User-agent: cohere-training-data-crawler Disallow: / Crawl-delay: 86400 User-agent: CommonCrawl Disallow: / Crawl-delay: 86400 User-agent: Cotoyogi Disallow: / Crawl-delay: 86400 User-agent: Crawlspace Disallow: / Crawl-delay: 86400 User-agent: Datenbank Crawler Disallow: / Crawl-delay: 86400 User-agent: DeepSeekBot Disallow: / Crawl-delay: 86400 User-agent: Devin Disallow: / Crawl-delay: 86400 User-agent: Diffbot Disallow: / Crawl-delay: 86400 User-agent: DOC Disallow: / Crawl-delay: 86400 User-agent: DomainCrawler Disallow: / Crawl-delay: 86400 User-agent: dotbot Disallow: / Crawl-delay: 86400 User-agent: Download Ninja Disallow: / Crawl-delay: 86400 User-agent: DuckAssistBot Disallow: / Crawl-delay: 86400 User-agent: Echobot Bot Disallow: / Crawl-delay: 86400 User-agent: EchoboxBot Disallow: / Crawl-delay: 86400 User-agent: FacebookBot Disallow: / Crawl-delay: 86400 User-agent: facebot Disallow: / Crawl-delay: 86400 User-agent: Factset_spyderbot Disallow: / Crawl-delay: 86400 User-agent: Fetch Disallow: / Crawl-delay: 86400 User-agent: FirecrawlAgent Disallow: / Crawl-delay: 86400 User-agent: Friendly Crawler Disallow: / Crawl-delay: 86400 User-agent: FriendlyCrawler Disallow: / Crawl-delay: 86400 User-agent: Gemini Disallow: / Crawl-delay: 86400 User-agent: Gemini-Deep-Research Disallow: / Crawl-delay: 86400 User-agent: Google Read Aloud Disallow: / Crawl-delay: 86400 User-agent: Google-CloudVertexBot Disallow: / Crawl-delay: 86400 User-agent: Google-Extended Disallow: / Crawl-delay: 86400 User-agent: Google-InspectionTool Disallow: / Crawl-delay: 86400 User-agent: Google-speakr Disallow: / Crawl-delay: 86400 User-agent: GoogleAgent-Mariner Disallow: / Crawl-delay: 86400 User-agent: Googlebot Disallow: / Crawl-delay: 86400 User-agent: Googlebot Desktop Disallow: / Crawl-delay: 86400 User-agent: Googlebot Smartphone Disallow: / Crawl-delay: 86400 User-agent: Googlebot-Image Disallow: / Crawl-delay: 86400 User-agent: Googlebot-News Disallow: / Crawl-delay: 86400 User-agent: Googlebot-Video Disallow: / Crawl-delay: 86400 User-agent: GoogleOther Disallow: / Crawl-delay: 86400 User-agent: GoogleOther-Image Disallow: / Crawl-delay: 86400 User-agent: GoogleOther-Video Disallow: / Crawl-delay: 86400 User-agent: GPTBot Disallow: / Crawl-delay: 86400 User-agent: grub-client Disallow: / Crawl-delay: 86400 User-agent: HTTrack Disallow: / Crawl-delay: 86400 User-agent: huggingface Disallow: / Crawl-delay: 86400 User-agent: ia_archiver Disallow: / Crawl-delay: 86400 User-agent: iaskspider/2.0 Disallow: / Crawl-delay: 86400 User-agent: ICC-Crawler Disallow: / Crawl-delay: 86400 User-agent: ImagesiftBot Disallow: / Crawl-delay: 86400 User-agent: img2dataset Disallow: / Crawl-delay: 86400 User-agent: ISSCyberRiskCrawler Disallow: / Crawl-delay: 86400 User-agent: IsraBot Disallow: / Crawl-delay: 86400 User-agent: k2spider Disallow: / Crawl-delay: 86400 User-agent: Kangaroo Bot Disallow: / Crawl-delay: 86400 User-agent: larbin Disallow: / Crawl-delay: 86400 User-agent: libwww Disallow: / Crawl-delay: 86400 User-agent: linkedinbot Disallow: / Crawl-delay: 86400 User-agent: linko Disallow: / Crawl-delay: 86400 User-agent: magpie-crawler Disallow: / Crawl-delay: 86400 User-agent: Mediapartners-Google* Disallow: / Crawl-delay: 86400 User-agent: Meltwater Disallow: / Crawl-delay: 86400 User-agent: meta-externalads Disallow: / Crawl-delay: 86400 User-agent: meta-externalagent Disallow: / Crawl-delay: 86400 User-agent: meta-externalfetcher Disallow: / Crawl-delay: 86400 User-agent: meta-webindexer Disallow: / Crawl-delay: 86400 User-agent: MicrosoftPreview Disallow: / Crawl-delay: 86400 User-agent: Microsoft.URL.Control Disallow: / Crawl-delay: 86400 User-agent: MistralAI-User Disallow: / Crawl-delay: 86400 User-agent: MJ12bot Disallow: / Crawl-delay: 86400 User-agent: MSIECrawler Disallow: / Crawl-delay: 86400 User-agent: msnbot Disallow: / Crawl-delay: 86400 User-agent: MyCentralAIScraperBot Disallow: / Crawl-delay: 86400 User-agent: netcraftsurvey Disallow: / Crawl-delay: 86400 User-agent: netEstate Imprint Crawler Disallow: / Crawl-delay: 86400 User-agent: news-please Disallow: / Crawl-delay: 86400 User-agent: NewsNow Disallow: / Crawl-delay: 86400 User-agent: NovaAct Disallow: / Crawl-delay: 86400 User-agent: NPBot Disallow: / Crawl-delay: 86400 User-agent: Nutch Disallow: / Crawl-delay: 86400 User-agent: OAI-SearchBot Disallow: / Crawl-delay: 86400 User-agent: Offline Explorer Disallow: / Crawl-delay: 86400 User-agent: omgili Disallow: / Crawl-delay: 86400 User-agent: omgilibot Disallow: / Crawl-delay: 86400 User-agent: Orthogaffe Disallow: / Crawl-delay: 86400 User-agent: OpenAI Disallow: / Crawl-delay: 86400 User-agent: Operator Disallow: / Crawl-delay: 86400 User-agent: outbrain Disallow: / Crawl-delay: 86400 User-agent: PanguBot Disallow: / Crawl-delay: 86400 User-agent: Panscient Disallow: / Crawl-delay: 86400 User-agent: panscient.com Disallow: / Crawl-delay: 86400 User-agent: Perplexity-ai Disallow: / Crawl-delay: 86400 User-agent: Perplexity-User Disallow: / Crawl-delay: 86400 User-agent: PerplexityBot Disallow: / Crawl-delay: 86400 User-agent: PetalBot Disallow: / Crawl-delay: 86400 User-agent: PhindBot Disallow: / Crawl-delay: 86400 User-agent: pinterest Disallow: / Crawl-delay: 86400 User-agent: Pinterestbot Disallow: / Crawl-delay: 86400 User-agent: PiplBot Disallow: / Crawl-delay: 86400 User-agent: Poseidon Research Crawler Disallow: / Crawl-delay: 86400 User-agent: QualifiedBot Disallow: / Crawl-delay: 86400 User-agent: QuillBot Disallow: / Crawl-delay: 86400 User-agent: quillbot.com Disallow: / Crawl-delay: 86400 User-agent: rogerbot Disallow: / Crawl-delay: 86400 User-agent: SBIntuitionsBot Disallow: / Crawl-delay: 86400 User-agent: scoop.it Disallow: / Crawl-delay: 86400 User-agent: Scrapy Disallow: / Crawl-delay: 86400 User-agent: Screaming Frog SEO Spider Disallow: / Crawl-delay: 86400 User-agent: Seekr Disallow: / Crawl-delay: 86400 User-agent: SemrushBot Disallow: / Crawl-delay: 86400 User-agent: SemrushBot-OCOB Disallow: / Crawl-delay: 86400 User-agent: SemrushBot-SA Disallow: / Crawl-delay: 86400 User-agent: SemrushBot-SI Disallow: / Crawl-delay: 86400 User-agent: SemrushBot-SWA Disallow: / Crawl-delay: 86400 User-agent: SentiBot Disallow: / Crawl-delay: 86400 User-agent: SeznamBot Disallow: / Crawl-delay: 86400 User-agent: showyoubot Disallow: / Crawl-delay: 86400 User-agent: Sidetrade indexer bot Disallow: / Crawl-delay: 86400 User-agent: SiteSnagger Disallow: / Crawl-delay: 86400 User-agent: slackbot Disallow: / Crawl-delay: 86400 User-agent: Slurp Disallow: / Crawl-delay: 86400 User-agent: sogou Disallow: / Crawl-delay: 86400 User-agent: Storebot-Google Disallow: / Crawl-delay: 86400 User-agent: SummalyBot Disallow: / Crawl-delay: 86400 User-agent: TelegramBot Disallow: / Crawl-delay: 86400 User-agent: Teleport Disallow: / Crawl-delay: 86400 User-agent: TeleportPro Disallow: / Crawl-delay: 86400 User-agent: teoma Disallow: / Crawl-delay: 86400 User-agent: TerraCotta Disallow: / Crawl-delay: 86400 User-agent: Thinkbot Disallow: / Crawl-delay: 86400 User-agent: TikTokSpider Disallow: / Crawl-delay: 86400 User-agent: Timpibot Disallow: / Crawl-delay: 86400 User-agent: TurnitinBot Disallow: / Crawl-delay: 86400 User-agent: tweetmemebot Disallow: / Crawl-delay: 86400 User-agent: Twitterbot Disallow: / Crawl-delay: 86400 User-agent: UbiCrawler Disallow: / Crawl-delay: 86400 User-agent: viberbot Disallow: / Crawl-delay: 86400 User-agent: VelenPublicWebCrawler Disallow: / Crawl-delay: 86400 User-agent: vkshare Disallow: / Crawl-delay: 86400 User-agent: WARDBot Disallow: / Crawl-delay: 86400 User-agent: WebCopier Disallow: / Crawl-delay: 86400 User-agent: WebReaper Disallow: / Crawl-delay: 86400 User-agent: WebStripper Disallow: / Crawl-delay: 86400 User-agent: Webzio Disallow: / Crawl-delay: 86400 User-agent: Webzio-Extended Disallow: / Crawl-delay: 86400 User-agent: WebZIP Disallow: / Crawl-delay: 86400 User-agent: wget Disallow: / Crawl-delay: 86400 User-agent: Xenu Disallow: / Crawl-delay: 86400 User-agent: YaK Disallow: / Crawl-delay: 86400 User-agent: YandexAdditional Disallow: / Crawl-delay: 86400 User-agent: YandexAdditionalBot Disallow: / Crawl-delay: 86400 User-agent: Yandex Disallow: / Crawl-delay: 86400 User-agent: YandexBot Disallow: / Crawl-delay: 86400 User-agent: Yeti Disallow: / Crawl-delay: 86400 User-agent: YouBot Disallow: / Crawl-delay: 86400 User-agent: Zao Disallow: / Crawl-delay: 86400 User-agent: Zealbot Disallow: / Crawl-delay: 86400 User-agent: ZoominfoBot Disallow: / Crawl-delay: 86400 User-agent: ZumBot Disallow: / Crawl-delay: 86400 User-agent: ZyBORG Disallow: / Crawl-delay: 86400