User-agent: *
Allow: /
Disallow: /private/
Disallow: /admin/
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /*?*
Disallow: /*.php$
Disallow: /search/
Disallow: /tag/

# AI/Agent-Specific Directives
# Common AI crawler user agents (add as they become known)

User-agent: GPTBot
Allow: /

Disallow: /private/

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: Omgili
Allow: /

User-agent: Applebot-Extended
Allow: /

# Coursed API endpoints for AI agents

User-agent: *

Allow: /api/courses.json


# Book API endpoints for AI agents

User-agent: *

Allow: /api/books.json

Allow: /api/books/related.json

Allow: /api/books/stats.json

Allow: /api/books/*.json$

Disallow: /api/admin/

Disallow: /api/private/


User-agent: 
Sitemap: https://ev0lv.com/sitemap_index.xml


# Block resource-heavy or non-content pages from general crawlers
User-agent: *
Disallow: /assets/js/
Disallow: /assets/css/
Disallow: /node_modules/
Disallow: /vendor/

# Crawl-delay for aggressive crawlers (prevents server overload)
Crawl-delay: 2

# Sitemap location (CRITICAL - ensure this file exists!)
Sitemap: https://ev0lv.com/sitemap.xml
Sitemap: https://ev0lv.com/sitemap_index.xml
Sitemap: https://ev0lv.com/post-sitemap.xml
Sitemap: https://ev0lv.com/page-sitemap.xml
Sitemap: https://ev0lv.com/course-sitemap.xml
Sitemap: https://ev0lv.com/book-sitemap.xml