User-agent: *
Allow: /
Allow: /crawl
Allow: /static-crawl

# Block the control page for testing
Disallow: /no-crawl

# Block API endpoint for testing
# Disallow: /api/data

# Block test image for testing
Disallow: /test-image.svg

# Optional: Allow specific search engines to test different behaviors
# User-agent: Googlebot
# Disallow: /no-crawl
# Disallow: /api/data
# Disallow: /test-image.svg

Sitemap: http://localhost:3000/sitemap.xml