Sitemap: http://www.crossenagency.com/sitemap.xml
User-agent: * # this is wildcard for all bots
Disallow: /scotmail/ # directories to not allow bots in ex /downloads/
Disallow: /worksite/
Disallow: /stats/
Disallow: /cgi-bin/
Request-rate: 1/5 # maximum rate is one page every 5 seconds
Sitemap: http://www.crossenagency.com/sitemap.xml # site map