# Robots.txt for Aladdin Bioinformatics Foundation
# https://www.aladdinbf.org/robots.txt

# Allow all crawlers
User-agent: *
Allow: /

# Sitemap location
Sitemap: https://www.aladdinbf.org/sitemap.xml

# Disallow sensitive areas
Disallow: /api/
Disallow: /_next/
Disallow: /static/
Disallow: /admin/
Disallow: /private/
Disallow: /internal/

# Allow important content areas
Allow: /wastewater-amr/
Allow: /assets/
Allow: /images/
Allow: /videos/
Allow: /events/
Allow: /research/
Allow: /publications/

# Crawl delay for respectful crawling
Crawl-delay: 1

# Specific rules for different bots
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 2

# Block AI training bots
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

# Block archive bots
User-agent: ia_archiver
Disallow: /

User-agent: archive.org_bot
Disallow: /