User-Agent: check_http
Allow:  /

User-agent: *
Disallow: /
Allow: /sites/shared/
Allow: /sites/*/includes/*.css
Allow: /sites/*/includes/*.js
Allow: /assets/
Allow: /sites/*/?ajax=
Allow: /sites/*/?category=