User-agent: * Disallow: /shared/ Disallow: /errors/ Disallow: /cgi-bin/ Disallow: /search/ Disallow: /Library/ Disallow: /Legal/ Disallow: /sales/cart.php Disallow: /sales/step2.php Disallow: /forum/ Disallow: /baselines/ Disallow: /includes/ Disallow: /cache/ User-Agent: bingbot Crawl-delay: 10 User-agent: msnbot Crawl-delay: 20 User-agent: BLEXBot Crawl-delay: 20 User-agent: grapeshot Crawl-Delay: 20 User-agent: proximic Crawl-Delay: 20 User-agent: Linespider Crawl-Delay: 20 User-agent: CriteoBot/0.1 Crawl-Delay: 20 User-agent: TTD-Content Crawl-Delay: 20 User-agent: Neevabot Crawl-Delay: 20 User-agent: YandexBot Crawl-Delay: 20 User-agent: GPTBot Crawl-Delay: 20 User-agent: MJ12bot Disallow: / User-agent: PetalBot Disallow: / User-Agent: AhrefsBot Disallow: / User-agent: dotbot Disallow: / User-agent: SeznamBot Disallow: / User-agent: Gigabot Disallow: / User-agent: ZumBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: BDCbot Disallow: / User-agent: SEOkicks Disallow: / User-agent: SeekportBot Disallow: / User-agent: PickyBot Disallow: / User-agent: rogerbot Disallow: /