# Sitemap location Sitemap: https://coveringscanada.ca/sitemap_index.xml # Amazon Bots - Aggressive throttling User-agent: Amazonbot Crawl-delay: 30 Disallow: /wp-admin/ Disallow: /wp-includes/ Disallow: /wp-content/plugins/ Disallow: /wp-content/themes/ Disallow: /wp-content/uploads/ Disallow: /?s= Disallow: /search/ Disallow: */feed/ Disallow: */trackback/ Disallow: */xmlrpc.php User-agent: Amazon-Route53-Health-Check-Service Disallow: / # Other aggressive bots you might want to throttle User-agent: AhrefsBot Crawl-delay: 20 Disallow: / User-agent: SemrushBot Crawl-delay: 20 Disallow: / User-agent: MJ12bot Crawl-delay: 30 Disallow: / User-agent: DotBot Crawl-delay: 30 Disallow: / User-agent: BLEXBot Disallow: / User-agent: PetalBot Crawl-delay: 30 Disallow: / # Good bots - Allow with reasonable crawl delay User-agent: Googlebot Crawl-delay: 2 Disallow: /wp-admin/ Disallow: /wp-includes/ Allow: /wp-admin/admin-ajax.php User-agent: Bingbot Crawl-delay: 2 Disallow: /wp-admin/ Disallow: /wp-includes/ # Default for all other bots User-agent: * Crawl-delay: 10 Disallow: /wp-admin/ Disallow: /wp-includes/ Disallow: /wp-content/plugins/ Disallow: /wp-content/themes/ Disallow: /?s= Disallow: /search/ Disallow: */feed/ Disallow: */trackback/ Disallow: */xmlrpc.php Disallow: /cgi-bin/ Disallow: /calendar-3/action~posterboard/ Disallow: /calendar-3/action~agenda/ Disallow: /calendar-3/action~oneday/ Disallow: /calendar-3/action~month/ Disallow: /calendar-3/action~week/ Disallow: /calendar-3/action~stream/ Allow: /wp-content/uploads/