# A crawler obeys only the most specific group that names it, so every group # carries the full Disallow list. Googlebot and Bingbot get their own group # only for the shorter crawl delay. User-agent: Googlebot User-agent: Bingbot Allow: / Disallow: /admin/ Disallow: /api/ Disallow: /go/ Disallow: /casinos/? Crawl-delay: 1 User-agent: * Allow: / # Block admin and API from indexing Disallow: /admin/ Disallow: /api/ # Block redirect URLs from indexing (these are for affiliate tracking) Disallow: /go/ # Filtered directory views (?category=, ?feature=, ?new=, ?region=, ?search=) # are the same casinos in a different cut; crawling every combination ties # up the web workers. The bare /casinos/ stays crawlable. Disallow: /casinos/? Crawl-delay: 2 # The demo permalink serves known crawlers a noindexed link-preview page # (users/demo_preview.html) instead of logging them in, and it must stay # crawlable so social scrapers can build the card. Do NOT disallow /demo/. # Allow sitemap Sitemap: https://www.sweepscointracker.com/sitemap.xml