diff --git a/robots.txt.html b/robots.txt.html index 22e0d5e63eb65fb46cc4b89153a99cd00032f315..5688997dd6bceffac34fd4ab2e39ec33841cf9dc 100644 --- a/robots.txt.html +++ b/robots.txt.html @@ -18,6 +18,9 @@ Allow: /sites/[(#URL_SITE_SPIP|replace{'www\.',''}|replace{'ww2\.',''}|replac Sitemap: #URL_SITE_SPIP/sitemap.xml User-agent: bingbot -Crawl-delay: 1 +Crawl-delay: 29 + +User-Agent: MJ12bot +Crawl-Delay: 31 ]