User-agent: *
Allow: /
Allow: /search
Allow: /sitemap.xml
Allow: /latest
Allow: /trending
Disallow: /admin/
Disallow: /private/
Disallow: /api/

# SEO Directives
Crawl-delay: 1
Clean-param: ref&source

# Sitemaps
Sitemap: https://perumperindo.bohay.workers.dev/sitemap.xml

# Search Engine Specific
User-agent: Googlebot
Allow: /
Allow: /search
Crawl-delay: 0.5

User-agent: Bingbot
Allow: /
Allow: /search
Crawl-delay: 0.5

User-agent: YandexBot
Allow: /
Allow: /search

User-agent: Baiduspider
Allow: /
Allow: /search
Crawl-delay: 1

# IndexNow Verification
User-agent: IndexNow
Allow: /