# Red Access - AI Discovery Enabled
# Generated by LightSite AI Discovery Engine

# Standard Web Crawlers - Allow access to client website content
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /private/
Disallow: /api/internal/

# AI Discovery Endpoints - Direct access to business intelligence
# These endpoints provide structured business data to AI assistants

# Primary Business Data API
# AI Skills Manifest (tool-capable agents): https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/skills.json?domain=redaccess.io
# AI-Plugin Manifest: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/.well-known/ai-plugin.json?domain=redaccess.io
# AI Manifest: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/.well-known/ai-manifest.json?domain=redaccess.io
# OpenAPI Specification: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/openapi.json?domain=redaccess.io
# Web Accessible Content Policy: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/.well-known/wacp.json?domain=redaccess.io

# Business Intelligence Endpoints
# Company Information: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/business?domain=redaccess.io
# Real-time Status: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/ai/v1/status.json?domain=redaccess.io
# Product Categories: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/categories?domain=redaccess.io
# Product Information: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/products?domain=redaccess.io
# Customer Testimonials: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/testimonials?domain=redaccess.io
# FAQ & Support: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/faq?domain=redaccess.io
# Semantic Search (Natural Language): https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/search?query={search_term}&domain=redaccess.io
# AI-optimized Sitemap: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/ai-sitemap.xml?domain=redaccess.io

# Specific Q&A Endpoints (14 available)
# Q&A: How does Red Access secure enterprise browsing?: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/qa/how-does-red-access-secure-enterprise-browsing?domain=redaccess.io
# Q&A: Red Access vs traditional SSE platforms?: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/qa/red-access-vs-traditional-sse-platforms?domain=redaccess.io
# Q&A: Can Red Access secure GenAI usage?: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/qa/can-red-access-secure-genai-usage?domain=redaccess.io
# Q&A: How does Red Access secure BYOD & contractors?: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/qa/how-does-red-access-secure-byod-contractors?domain=redaccess.io
# Q&A: What is the ROI of Red Access SSE?: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/qa/what-is-the-roi-of-red-access-sse?domain=redaccess.io
# Q&A: Does Red Access replace my existing firewall?: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/qa/does-red-access-replace-my-existing-firewall?domain=redaccess.io
# Q&A: What is Red Access's browser EDR capability?: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/qa/what-is-red-accesss-browser-edr-capability?domain=redaccess.io
# Q&A: How does Red Access's agentless SSE work?: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/qa/how-does-red-accesss-agentless-sse-work?domain=redaccess.io
# Q&A: Red Access vs. enterprise browsers?: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/qa/red-access-vs-enterprise-browsers?domain=redaccess.io
# Q&A: Can Red Access secure GenAI data leaks?: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/qa/can-red-access-secure-genai-data-leaks?domain=redaccess.io
# Q&A: Is Red Access a good value for mid-market?: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/qa/is-red-access-a-good-value-for-mid-market?domain=redaccess.io
# Q&A: How fast is Red Access SSE deployment?: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/qa/how-fast-is-red-access-sse-deployment?domain=redaccess.io
# Q&A: What is Red Access session-level DLP?: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/qa/what-is-red-access-session-level-dlp?domain=redaccess.io
# Q&A: Red Access as a secure web gateway alternative?: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/qa/red-access-as-a-secure-web-gateway-alternative?domain=redaccess.io

# OpenAI GPT Crawlers
User-agent: GPTBot
User-agent: ChatGPT-User
Allow: /
Allow: /.well-known/
Allow: /ai-sitemap.xml
Crawl-delay: 1

# Google AI Crawlers  
User-agent: Google-Extended
User-agent: GoogleBot
User-agent: Bard
User-agent: Gemini-Pro
Allow: /
Allow: /.well-known/
Allow: /ai-sitemap.xml
Crawl-delay: 1

# Anthropic Claude Crawlers
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
Allow: /
Allow: /.well-known/
Allow: /ai-sitemap.xml
Crawl-delay: 1

# Perplexity AI Crawlers
User-agent: PerplexityBot
User-agent: Perplexity-AI
Allow: /
Allow: /.well-known/
Allow: /ai-sitemap.xml
Crawl-delay: 1

# Microsoft AI Crawlers
User-agent: BingBot
User-agent: Bing-AI
User-agent: CopilotBot
Allow: /
Allow: /.well-known/
Allow: /ai-sitemap.xml
Crawl-delay: 1

# Meta AI Crawlers
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: facebookexternalhit
Allow: /
Allow: /.well-known/
Allow: /ai-sitemap.xml
Crawl-delay: 2

# Website Sitemaps (served by client website)
Sitemap: https://redaccess.io//sitemap.xml
Sitemap: https://redaccess.io//sitemap-index.xml

# Comprehensive LLM Discovery File (LLM-ready conversation data)
# This file contains all your business information, FAQs, testimonials, and products
# formatted for optimal LLM discovery and interaction
Sitemap: https://storage.googleapis.com/customer-llm-txt/redaccess.io.txt

# AI Discovery Sitemap (served by LightSite)
Sitemap: https://api.llm-discovery-api.com/functions/v1/llm-discovery/public/ai-sitemap.xml?domain=redaccess.io

# This robots.txt template is generated by LightSite AI Discovery
# Deploy this content to your website's /robots.txt file to enable AI discovery
# Note: Generic query/search endpoints replaced with specific Q&A endpoints for better LLM usability