# ============================================
# AI Training & AI Search Crawlers
# ============================================

# OpenAI AI Crawlers
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

# Anthropic AI Crawlers
User-agent: ClaudeBot
Allow: /

# Google AI Crawlers
User-agent: Google-Extended
Allow: /

# Perplexity AI Crawlers
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# X/Twitter AI Crawler
User-agent: GrokBot
Allow: /

# Apple AI Crawler
User-agent: Applebot-Extended
Allow: /

# Common Crawl (used by AI training)
User-agent: CCBot
Allow: /

# ============================================
# Search Engine Crawlers
# ============================================

# Google
User-agent: Googlebot
Allow: /

# Microsoft/Bing
User-agent: bingbot
Allow: /

# Yahoo
User-agent: Yahoo! Slurp
Allow: /

# Facebook (for social sharing)
User-agent: FacebookBot
Allow: /

# ============================================
# Default Rules for All Crawlers
# ============================================

User-agent: *

# ============================================
# DISALLOW Rules (must come before Allow rules)
# ============================================

# Block all JSON files and JSON API responses
Disallow: /*.json$
Disallow: /*.json?*

# Block admin and private areas
Disallow: /admin/
Disallow: /api/draft-mode/
Disallow: /api/auth/
Disallow: /api/revalidate/
Disallow: /api/google-indexing/
Disallow: /api/sendEmail/
Disallow: /api/categories/
Disallow: /api/tags/
Disallow: /api/navigation
Disallow: /api/company

# Block all other API routes (general catch-all)
Disallow: /api/

# ============================================
# ALLOW Rules (specific resources)
# ============================================

# Allow specific API endpoints that serve public content (sitemaps)
Allow: /api/sitemap
Allow: /api/sitemap-image
Allow: /api/sitemap-news
Allow: /sitemap-static.xml

# Explicitly allow critical resources for rendering
Allow: /_next/static/
Allow: /_next/image/
Allow: /fonts/
Allow: /images/
Allow: /*.js$
Allow: /*.css$
Allow: /*.woff$
Allow: /*.woff2$
Allow: /*.ttf$
Allow: /*.eot$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.ico$

# Allow all pages by default (must come after Disallow rules)
Allow: /

# ============================================
# Sitemaps
# ============================================

Sitemap: https://www.globalbankingandfinance.com/sitemap-index.xml
Sitemap: https://www.globalbankingandfinance.com/sitemap-static.xml
Sitemap: https://www.globalbankingandfinance.com/sitemap-image.xml
Sitemap: https://www.globalbankingandfinance.com/sitemap-news.xml