# Default: everything public is crawlable.
# Content-Signal (https://contentsignals.org/): search, AI answers and AI
# training are all allowed, matching the explicit AI-crawler groups below
# (GPTBot, Google-Extended and Applebot-Extended are training crawlers).
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /hq/

# AI assistants and answer engines are explicitly welcome — being cited in
# ChatGPT/Claude/Perplexity answers is distribution. Same rules as above.
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /hq/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /hq/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /hq/

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /hq/

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /hq/

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /hq/

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

# Machine-readable site overview for LLMs:
# https://newsstocks.live/llms.txt

Sitemap: https://newsstocks.live/sitemap.xml
Sitemap: https://newsstocks.live/news-sitemap.xml