User-agent: * Disallow: /thcp/ Disallow: /apps/dev/* Disallow: /resumes/ Disallow: /resumes2/ Disallow: /dev/ Disallow: /pro-tools/* Disallow: /preparse/* Disallow: /submit-new/* Disallow: /include/ Disallow: /cron/* Disallow: /editing/ Disallow: /?URL=http:// Disallow: /?URL=* Disallow: /embed-templates/ Disallow: /embed-templates/* Disallow: /pdfView-pro/ Disallow: /editing/* Disallow: /old-design/* Disallow: /trendreports/download2 Disallow: /include/* Disallow: /trending # Story blocks for selected bots User-agent: archive.org_bot Disallow: /*.html User-agent: ia_archiver Disallow: /*.html User-agent: ia_archiver-web.archive.org Disallow: /*.html # Complete blocks of uninteresting bots User-agent: A6-Indexer Disallow: / User-agent: AddSearchBot Disallow: / User-agent: aiHitBot Disallow: / User-agent: Andibot Disallow: / User-agent: AspiegelBot Disallow: / User-agent: Barkrowler Disallow: / User-agent: bigsur.ai Disallow: / User-agent: BLP_bbot Disallow: / User-agent: Brightbot 1.0 Disallow: / User-agent: Buck Disallow: / User-agent: Bytespider Disallow: / User-agent: CCBot Disallow: / User-agent: ChatGLM-Spider Disallow: / User-agent: coccocbot-web Disallow: / User-agent: cohere-ai Disallow: / User-agent: cohere-training-data-crawler Disallow: / User-agent: Cotoyogi Disallow: / User-agent: crawler4j Disallow: / User-agent: Crawlspace Disallow: / User-agent: Datenbank Crawler Disallow: / User-agent: Diffbot Disallow: / User-agent: EasouSpider Disallow: / User-agent: Echobot Bot Disallow: / User-agent: EchoboxBot Disallow: / User-agent: ecoResearch Disallow: / User-agent: FirecrawlAgent Disallow: / User-agent: Grok Disallow: / User-agent: GrokBot Disallow: / User-agent: GrokBot/1.0 Disallow: / User-agent: Grok-DeepSearch/1.0 Disallow: / User-agent: Genieo Disallow: / User-agent: https://hada.news Disallow: / User-agent: iaskspider/2.0 Disallow: / User-agent: ICC-Crawler Disallow: / User-agent: img2dataset Disallow: / User-agent: JenkersBot Disallow: / User-agent: Jetslide Disallow: / User-agent: Kangaroo Bot Disallow: / User-agent: KunatoCrawler Disallow: / User-agent: LivelapBot Disallow: / User-agent: magpie-crawler Disallow: / User-agent: MauiBot Disallow: / User-agent: MAZBot Disallow: / User-agent: MistralAI-User Disallow: / User-agent: MistralAI-User/1.0 Disallow: / User-agent: Mojeek Disallow: / User-agent: MojeekBot Disallow: / User-agent: MyCentralAIScraperBot Disallow: / User-agent: news-please Disallow: / User-agent: NewsNow Disallow: / User-agent: NovaAct Disallow: / User-agent: Omgilibot Disallow: / User-agent: omgili Disallow: / User-agent: PetalBot Disallow: / User-agent: PanguBot Disallow: / User-agent: Poseidon Research Crawler Disallow: / User-agent: Primalbot Disallow: / User-agent: QuillBot Disallow: / User-agent: quillbot.com Disallow: / User-agent: R6_CommentReader Disallow: / User-agent: SBIntuitionsBot Disallow: / User-agent: Scrapy Disallow: / User-agent: SeekportBot Disallow: / User-agent: Seekr Disallow: / User-agent: SeekrBot Disallow: / User-agent: SeznamBot Disallow: / User-agent: SeznamHomepageCrawler Disallow: / User-agent: Sidetrade indexer bot Disallow: / User-agent: Spinn3r Disallow: / User-agent: TaraGroup Intelligent Bot Disallow: / User-agent: Thinkbot Disallow: / User-agent: Timpibot Disallow: / User-agent: VelenPublicWebCrawler Disallow: / User-agent: Verity Disallow: / User-agent: Verity/1.1 Disallow: / User-agent: ViennaTinyBot Disallow: / User-agent: WebVac Disallow: / User-agent: Webzio-Extended Disallow: / User-agent: WebZip Disallow: / User-agent: WRTNBot Disallow: / User-agent: xAI Disallow: / User-agent: xAI-Grok/1.0 Disallow: / User-agent: yacy Disallow: / User-agent: yacybot Disallow: / User-agent: YouBot Disallow: / Sitemap: https://www.trendhunter.com/sitemapindex.xml