# Harley Street Institute - Robots.txt
# https://www.harleystreetinstitute.com

User-agent: *
Allow: /

# Sitemap locations
Sitemap: https://www.harleystreetinstitute.com/sitemap.xml
Sitemap: https://www.harleystreetinstitute.com/sitemap-courses.xml
Sitemap: https://www.harleystreetinstitute.com/sitemap-pages.xml
Sitemap: https://www.harleystreetinstitute.com/sitemap-journal.xml
Sitemap: https://www.harleystreetinstitute.com/sitemap-aesthetic-talk.xml

# Aesthetic Talk — explicitly allow chapter pages
Allow: /aesthetic-talk/

# Journal — explicitly allow article HTML, PDFs and JATS exports
Allow: /journal/
Allow: /functions/v1/journal-article-pdf
Allow: /functions/v1/journal-issue-cover-pdf
Allow: /functions/v1/journal-article-jats
Allow: /functions/v1/journal-oai-pmh

# Disallow admin and protected areas
Disallow: /admin/
Disallow: /auth/
Disallow: /app/
Disallow: /media-manager/
Disallow: /page-management/
Disallow: /seo-evaluator/
Disallow: /seo-dashboard/
Disallow: /blog-management/
Disallow: /debug-blog-add/
Disallow: /user-management/
Disallow: /initialize-content/
Disallow: /journal/legacy-home/
Disallow: /polynucleotides-online-assessment/
Disallow: /coming-soon/
Disallow: /booking-success/
Disallow: /booking-cancelled/
Disallow: /contact-thank-you/
Disallow: /endolaser-models/

# Scratch / prototype routes (not for indexing; route cleanup pending)
Disallow: /a2/
Disallow: /a3/
Disallow: /a4/
Disallow: /a5/
Disallow: /ab/
Disallow: /f2/
Disallow: /home6/
Disallow: /j2/
Disallow: /j3/
Disallow: /newhome1/
Disallow: /newhome2/
Disallow: /nt/
Disallow: /nt1/
Disallow: /test/

# Disallow all URLs with query parameters (prevents duplicate content)
Disallow: /*?*

# Disallow tag, category, and author pages (low SEO value, duplicate content)
Disallow: /tag/
Disallow: /tags/
Disallow: /category/
Disallow: /categories/
Disallow: /author/
Disallow: /blog/author/
Disallow: /blog/tag/
Disallow: /blog/category/

# Note: Non-trailing slash URLs redirect to trailing slash versions via 301
# Canonical URLs always use trailing slashes (e.g., /course/ not /course)
# This is enforced via:
# 1. netlify.toml configuration
# 2. _redirects file
# 3. SEO component canonical tags
# 4. Client-side redirect in SEO component

# Set crawl delay to be respectful
Crawl-delay: 1