# Robots.txt for Passport Pals Travel Agency Directory
# SEO Optimized for Search Engine Crawling

# Allow all search engines to crawl
User-agent: *
Allow: /

# Main Pages - High Priority for Crawling
Allow: /
Allow: /index.html
Allow: /agencies.html
Allow: /services.html
Allow: /about.html
Allow: /contact.html

# Static Assets - Allow Crawling for Better Understanding
Allow: /css/
Allow: /js/
Allow: /img/

# Disallow Crawling of Sensitive Files
Disallow: /Travel Agency Data GH.csv
Disallow: /.gitignore
Disallow: /LICENSE
Disallow: /.git/
Disallow: /node_modules/
Disallow: /.env
Disallow: /admin/
Disallow: /private/

# Specific Bot Directives for Better Crawling
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

# Crawler Delay to Reduce Server Load
User-agent: *
Crawl-delay: 1
Request-rate: 1/1s

# Sitemap Locations for Search Engines
Sitemap: https://www.passportpals.com/sitemap.xml
Sitemap: https://www.passportpals.com/sitemap-news.xml

# Comment: Update domain.com with your actual domain before deployment