# 99.co Robots.txt - Updated for AI/LLM Access
# Last updated: July 2025

# =============================================================================
# BLOCK SPECIFIC PROBLEMATIC BOTS
# =============================================================================
User-agent: trovitBot
Disallow: /

# =============================================================================
# TRADITIONAL SEARCH ENGINE BOTS WITH CRAWL DELAY
# =============================================================================
User-agent: rogerbot
Crawl-delay: 3

User-agent: dotbot
Crawl-delay: 3

User-agent: AhrefsBot
User-agent: AhrefsSiteAudit
Crawl-delay: 5

# =============================================================================
# AI/LLM BOTS - SPECIAL ACCESS FOR ENHANCED CONTENT
# =============================================================================
# Meta/Facebook AI Bots
User-agent: Meta-ExternalFetcher
User-agent: meta-externalagent/1.1
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: facebookexternalhit

# Anthropic (Claude) AI Bots
User-agent: ClaudeBot
User-agent: claude-web
User-agent: anthropic-ai

# OpenAI (ChatGPT/GPT) Bots
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: OpenAI-SearchBot

# Perplexity AI Bots
User-agent: PerplexityBot
User-agent: Perplexity-User

# Google AI Bots (Bard/Gemini)
User-agent: Google-Extended
User-agent: GoogleOther

# Microsoft AI Bots (Copilot/Bing)
User-agent: BingBot
User-agent: CCBot

# Other Major AI Platforms
User-agent: cohere-ai
User-agent: YouBot
User-agent: Bytespider
User-agent: Operator
User-agent: BraveBot
User-agent: AI2Bot
User-agent: Applebot-Extended

# Allow AI bots to access markdown versions and llms.txt files in 99ID
Allow: /
Allow: /*.md$
Allow: /id/llms-full.txt
Allow: /id/llms.txt

# Block AI bots from sensitive areas (same as regular bots)
Disallow: /api
Disallow: /cdn-cgi/
Disallow: /id/account/
Disallow: /id/ajax/
Disallow: /id/ajax2/
Disallow: /id/cart/
Disallow: /id/assets/
Disallow: /id/images/
Disallow: /id/komunitas/ajax/
Disallow: /id/ad/
Disallow: /id/administration/
Disallow: /id/api/
Disallow: /id/logout
Disallow: /id/signup
Disallow: /id/ezy/
Disallow: /id/tracker.gif
Disallow: /id/guide/analytics
Disallow: /id/property/report-ad
Disallow: /id/komunitas/forums/create-topic
Disallow: /id/property/quick-nav-load
Disallow: /id/search/histogram
Disallow: /id/p/
Disallow: /id/*/ajax-social-share
Disallow: /id/*/ajax-favorite-button
Disallow: /id/*/ajax-text-placeholder
Disallow: /id/property/ajax-map
Disallow: /id/property/ajax-similar-property-list
Disallow: /id/property/insert-visited-property
Disallow: /id/property/send-multiple-enquiry-history
Disallow: /id/properti/*/print.pdf
Disallow: /id/property/ajax-nearby-agents
Disallow: /id/cari/
Disallow: /id/amp/

# =============================================================================
# DEFAULT RULES FOR ALL OTHER BOTS
# =============================================================================
User-agent: *

# Sitemaps (Original 99.co sitemaps preserved)
Sitemap: https://www.99.co/singapore/sitemap.xml
Sitemap: https://www.99.co/id/sitemap-v1/sitemap.xml

# Original 99.co ID restrictions
Disallow: /api
Disallow: /cdn-cgi/
Disallow: /id/account/
Disallow: /id/ajax/
Disallow: /id/ajax2/
Disallow: /id/cart/
Disallow: /id/assets/
Disallow: /id/images/
Disallow: /id/komunitas/ajax/
Disallow: /id/ad/
Disallow: /id/administration/
Disallow: /id/api/
Disallow: /id/logout
Disallow: /id/signup
Disallow: /id/ezy/
Disallow: /id/tracker.gif
Disallow: /id/guide/analytics
Disallow: /id/property/report-ad
Disallow: /id/komunitas/forums/create-topic
Disallow: /id/property/quick-nav-load
Disallow: /id/search/histogram
Disallow: /id/p/
Disallow: /id/*/pdf
Disallow: /id/*/ajax-social-share
Disallow: /id/*?pageSize=
Disallow: /id/*?listingType
Disallow: /id/*/ajax-favorite-button
Disallow: /id/*/ajax-text-placeholder
Disallow: /id/property/ajax-map
Disallow: /id/property/ajax-similar-property-list
Disallow: /id/property/insert-visited-property
Disallow: /id/property/send-multiple-enquiry-history
Disallow: /id/*?tipe_properti=
Disallow: /id/*?urut=*
Disallow: /id/properti/*/print.pdf
Disallow: /id/property/ajax-nearby-agents
Disallow: /id/cari/
Disallow: /id/amp/

# 99.co Singapore restrictions
Disallow: /singapore/rent/mrt-stations
Disallow: /singapore/sale/mrt-stations
Disallow: /singapore/s/rent/mrt-stations
Disallow: /singapore/s/sale/mrt-stations
Disallow: /singapore/rent/schools
Disallow: /singapore/sale/schools
Disallow: /singapore/s/rent/schools
Disallow: /singapore/s/sale/schools
Disallow: /singapore/rent/travel-time
Disallow: /singapore/sale/travel-time
Disallow: /singapore/s/rent/travel-time
Disallow: /singapore/s/sale/travel-time
Disallow: /singapore/new-launches/marina-one-residence
Disallow: /singapore/s/

# Block regular bots from accessing AI-specific content 99ID
Disallow: /*.md$
Disallow: /llms.txt
Disallow: /llms-full.txt
Disallow: /id/llms.txt
Disallow: /id/llms-full.txt