# 99.co Robots.txt - Updated for AI/LLM Access # Last updated: July 2025 # ============================================================================= # BLOCK SPECIFIC PROBLEMATIC BOTS # ============================================================================= User-agent: trovitBot Disallow: / # ============================================================================= # TRADITIONAL SEARCH ENGINE BOTS WITH CRAWL DELAY # ============================================================================= User-agent: rogerbot Crawl-delay: 3 User-agent: dotbot Crawl-delay: 3 User-agent: AhrefsBot User-agent: AhrefsSiteAudit Crawl-delay: 5 # ============================================================================= # AI/LLM BOTS - SPECIAL ACCESS FOR ENHANCED CONTENT # ============================================================================= # Meta/Facebook AI Bots User-agent: Meta-ExternalFetcher User-agent: meta-externalagent/1.1 User-agent: Meta-ExternalAgent User-agent: FacebookBot User-agent: facebookexternalhit # Anthropic (Claude) AI Bots User-agent: ClaudeBot User-agent: claude-web User-agent: anthropic-ai # OpenAI (ChatGPT/GPT) Bots User-agent: GPTBot User-agent: OAI-SearchBot User-agent: ChatGPT-User User-agent: OpenAI-SearchBot # Perplexity AI Bots User-agent: PerplexityBot User-agent: Perplexity-User # Google AI Bots (Bard/Gemini) User-agent: Google-Extended User-agent: GoogleOther # Microsoft AI Bots (Copilot/Bing) User-agent: BingBot User-agent: CCBot # Other Major AI Platforms User-agent: cohere-ai User-agent: YouBot User-agent: Bytespider User-agent: Operator User-agent: BraveBot User-agent: AI2Bot User-agent: Applebot-Extended # Allow AI bots to access markdown versions and llms.txt files in 99ID Allow: / Allow: /*.md$ Allow: /id/llms-full.txt Allow: /id/llms.txt # Block AI bots from sensitive areas (same as regular bots) Disallow: /api Disallow: /cdn-cgi/ Disallow: /id/account/ Disallow: /id/ajax/ Disallow: /id/ajax2/ Disallow: /id/cart/ Disallow: /id/assets/ Disallow: /id/images/ Disallow: /id/komunitas/ajax/ Disallow: /id/ad/ Disallow: /id/administration/ Disallow: /id/api/ Disallow: /id/logout Disallow: /id/signup Disallow: /id/ezy/ Disallow: /id/tracker.gif Disallow: /id/guide/analytics Disallow: /id/property/report-ad Disallow: /id/komunitas/forums/create-topic Disallow: /id/property/quick-nav-load Disallow: /id/search/histogram Disallow: /id/p/ Disallow: /id/*/ajax-social-share Disallow: /id/*/ajax-favorite-button Disallow: /id/*/ajax-text-placeholder Disallow: /id/property/ajax-map Disallow: /id/property/ajax-similar-property-list Disallow: /id/property/insert-visited-property Disallow: /id/property/send-multiple-enquiry-history Disallow: /id/properti/*/print.pdf Disallow: /id/property/ajax-nearby-agents Disallow: /id/cari/ Disallow: /id/amp/ # ============================================================================= # DEFAULT RULES FOR ALL OTHER BOTS # ============================================================================= User-agent: * # Sitemaps (Original 99.co sitemaps preserved) Sitemap: https://www.99.co/singapore/sitemap.xml Sitemap: https://www.99.co/id/sitemap-v1/sitemap.xml # Original 99.co ID restrictions Disallow: /api Disallow: /cdn-cgi/ Disallow: /id/account/ Disallow: /id/ajax/ Disallow: /id/ajax2/ Disallow: /id/cart/ Disallow: /id/assets/ Disallow: /id/images/ Disallow: /id/komunitas/ajax/ Disallow: /id/ad/ Disallow: /id/administration/ Disallow: /id/api/ Disallow: /id/logout Disallow: /id/signup Disallow: /id/ezy/ Disallow: /id/tracker.gif Disallow: /id/guide/analytics Disallow: /id/property/report-ad Disallow: /id/komunitas/forums/create-topic Disallow: /id/property/quick-nav-load Disallow: /id/search/histogram Disallow: /id/p/ Disallow: /id/*/pdf Disallow: /id/*/ajax-social-share Disallow: /id/*?pageSize= Disallow: /id/*?listingType Disallow: /id/*/ajax-favorite-button Disallow: /id/*/ajax-text-placeholder Disallow: /id/property/ajax-map Disallow: /id/property/ajax-similar-property-list Disallow: /id/property/insert-visited-property Disallow: /id/property/send-multiple-enquiry-history Disallow: /id/*?tipe_properti= Disallow: /id/*?urut=* Disallow: /id/properti/*/print.pdf Disallow: /id/property/ajax-nearby-agents Disallow: /id/cari/ Disallow: /id/amp/ # 99.co Singapore restrictions Disallow: /singapore/rent/mrt-stations Disallow: /singapore/sale/mrt-stations Disallow: /singapore/s/rent/mrt-stations Disallow: /singapore/s/sale/mrt-stations Disallow: /singapore/rent/schools Disallow: /singapore/sale/schools Disallow: /singapore/s/rent/schools Disallow: /singapore/s/sale/schools Disallow: /singapore/rent/travel-time Disallow: /singapore/sale/travel-time Disallow: /singapore/s/rent/travel-time Disallow: /singapore/s/sale/travel-time Disallow: /singapore/new-launches/marina-one-residence Disallow: /singapore/s/ # Block regular bots from accessing AI-specific content 99ID Disallow: /*.md$ Disallow: /llms.txt Disallow: /llms-full.txt Disallow: /id/llms.txt Disallow: /id/llms-full.txt