# HelpGeo robots.txt
# Site: https://helpgeo.cn
# Purpose: Allow mainstream search engines and AI/LLM crawlers (self-demo for a GEO site).

# ---------- Generic search engines ----------
User-agent: *
Allow: /

# Admin / functional endpoints: do not index.
Disallow: /admin/
Disallow: /api/
Disallow: /includes/
Disallow: /install.php
Disallow: /captcha_img.php
Disallow: /login.php
Disallow: /logout.php
Disallow: /my.php
Disallow: /search.php
Disallow: /site_content.php
Disallow: /site_content_edit.php
# V4.1: /uploads/ no longer blocked — og:image, content images and provider logos
# live there; blocking it hurt image search and multimodal AI crawlers.

# Runtime / ops endpoints: not content, keep out of the index.
Disallow: /healthz.php
Disallow: /storage/
Disallow: /cron_report.php
Disallow: /cron_monitor.php
Disallow: /cron_backup.php
Disallow: /sitemap.xml.php

# Legacy / backup artifacts that may exist on the server (defensive).
Disallow: /helpgeo/
Disallow: /qa_seed/
Disallow: /helpgeocn_*

# ---------- AI / LLM crawlers: welcome to crawl all public content ----------
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: cohere-ai
User-agent: PerplexityBot
User-agent: Bytespider
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /includes/
Disallow: /install.php
Disallow: /helpgeo/
Disallow: /qa_seed/
Disallow: /helpgeocn_*

# ---------- Sitemap & AI discovery ----------
Sitemap: https://helpgeo.cn/sitemap.xml
# Machine-readable indexes for AI/LLM platforms (fetch directly):
#   https://helpgeo.cn/llms.txt       - site overview + canonical links
#   https://helpgeo.cn/llms-full.txt  - full-text index
#   https://helpgeo.cn/ai.txt         - AI usage / attribution policy