# ═══════════════════════════════════════════════════════════════════════════════
# ROCSITE.COM, robots.txt
# The institutional memory of the company. Independent validation is the bar.
# ═══════════════════════════════════════════════════════════════════════════════

# ─── DEFAULT: ALL BOTS ────────────────────────────────────────────────────────
User-agent: *

# Allow current pages
Allow: /
Allow: /about-adam
Allow: /rocsite-ich
Allow: /rocsite-clinical-ai
Allow: /ai-governor
Allow: /sepsis-falsification
Allow: /research
Allow: /contact
Allow: /investors
Allow: /privacy
Allow: /terms
Allow: /legal
Allow: /llms.txt
Allow: /llms-full.txt

# Block old real estate pages
Disallow: /about-us
Disallow: /our-services
Disallow: /agents
Disallow: /join-rocsite
Disallow: /contact-and-support
Disallow: /advanced-home-property-search
Disallow: /privacy-policy
Disallow: /properties/
Disallow: /*-north-carolina

# Block WordPress/CMS remnants
Disallow: /wp-admin
Disallow: /wp-content
Disallow: /wp-login.php
Disallow: /wp-includes
Disallow: /feed
Disallow: /xmlrpc.php

# Block internal API and data paths
Disallow: /api/
Disallow: /data/
Disallow: /static/js/
Disallow: /static/css/

# Block backup / dev / source-leak surfaces
Disallow: /*.bak$
Disallow: /*.bak.*
Disallow: /*.bak-*
Disallow: /*.old$
Disallow: /*.orig$
Disallow: /*.swp$
Disallow: /*.tmp$
Disallow: /CLAUDE.md
Disallow: /*.md$
Disallow: /htaccess.txt
Disallow: /error_log
Disallow: /track-install.txt

# ─── AI / LLM CRAWLERS, WELCOME ─────────────────────────────────────────────
# Explicitly allow AI crawlers to access the site and the LLM content files.

User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /properties/

User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /properties/

User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /properties/

User-agent: Anthropic
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: Claude-Web
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /properties/

User-agent: Bytespider
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /api/
Disallow: /properties/

User-agent: cohere-ai
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: Meta-ExternalAgent
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

# ─── SITEMAPS ─────────────────────────────────────────────────────────────────
Sitemap: https://rocsite.com/sitemap.xml