# https://www.robotstxt.org/robotstxt.html
User-agent: *
Allow: /

# Block access to admin and private areas
Disallow: /admin/
Disallow: /private/
Disallow: /account/
Disallow: /api/
Disallow: /*.json$
Disallow: /node_modules/
Disallow: /src/

# Allow index of important pages
Allow: /bootcamps/
Allow: /internships/
Allow: /curriculum/
Allow: /mentorship/
Allow: /live-projects/
Allow: /about/
Allow: /contact/
Allow: /enroll/
Allow: /terms/
Allow: /privacy/

# Allow important files
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /llms.txt
Allow: /favicon.ico
Allow: /manifest.json

# Crawl delay (in seconds) for bots
Crawl-delay: 1

# Sitemap location
Sitemap: https://maintedge.com/sitemap.xml

# AI and LLM specific guidance
# For AI crawlers and language models
Allow: /llms.txt