User-agent: Mediapartners-Google* Disallow: /cgi-bin/ Disallow: /inc/ Disallow: /scripts/ Disallow: /mt-static/ Disallow: /source/ User-agent: * Crawl-delay: 5 # Wait 5 seconds between successive requests Request-rate: 1/5 # maximum rate is one page every 5 seconds Visit-time: 0600-1000 # only visit between 06:00 and 10:00 UTC