# robots.txt for ZenBot Scanner
# https://www.zenbotscanner.com / https://www.zenscans.com

# ---- Default policy for all crawlers ------------------------------
User-agent: *

# Allow the marketing / app landing pages — these should be indexed.
# (Anything not under a Disallow below is fair game.)

# Disallow internal endpoints. Zero SEO value, all scraping risk.
Disallow: /api/
# The login link is on every page, so Google indexed its bare URL (443 impressions at position
# 2.3 under the brand, Search Console 2026-10-10): a Disallow stops crawling, not indexing. It
# has to be crawlable for Google to see the "noindex" the server sends on it.
Allow: /api/auth/login
Disallow: /statehub
Disallow: /_blazor
Disallow: /_framework/
Disallow: /charting_library/
Disallow: /authentication/

# User-specific share URLs (custom scans). Disallow by default for
# privacy and because results are time-sensitive (a scan from 6 months
# ago has different output today). Remove this line if you ever want
# share-links to be search-indexable.
Disallow: /custom/

# Per-user windows and screens: chart pop-outs (one URL per symbol, unbounded) and the account
# page. Empty app shells with nothing to index. (/longs, /shorts, /multi stay crawlable: they
# are the home page's nav links and show as Google sitelinks.)
Disallow: /popout/
Disallow: /account

# Live scan routes. Those with a static /scans/ page stay crawlable (they carry the links people
# share; the app canonicalises them to the page). Those without one are listed below by
# StaticScanPagesService, which publishes this file with the marker replaced.
# Generated by StaticScanPagesService: live scan routes with no crawlable /scans/ page.
Disallow: /Attack-YD-Low$
Disallow: /Basic-Materials$
Disallow: /Cons-Disc.$
Disallow: /Cons-Staples$
Disallow: /Daily-Pullback_Long$
Disallow: /Daily-Pullback_Short$
Disallow: /Energy$
Disallow: /Finance$
Disallow: /Health-Care$
Disallow: /Industrials$
Disallow: /Most-Popular_Long$
Disallow: /Most-Popular_Short$
Disallow: /Real-Estate$
Disallow: /Technology$
Disallow: /Telecomm$
Disallow: /Utilities$
Disallow: /Your-Flags_Long$
Disallow: /Your-Flags_Short$

# Internal HTML fragments embedded by the app — not standalone pages.
# They answer 200 with a bare body (no doctype/head/title), so an indexed
# one is a broken-looking orphan. The app routes (/turbo, /install, ...)
# are the real destinations.
Disallow: /landing.html
Disallow: /turbo.html
Disallow: /install.html
Disallow: /support.html
Disallow: /how-to-trade-body.html
Disallow: /ai-stock-scanner-body.html

# Page SHELLS. These are near-duplicates of the assembled pages ("/" and
# "/how-to-trade") and were directly fetchable — classic duplicate
# content competing with the real url.
Disallow: /landing-shell.html
Disallow: /how-to-trade-shell.html


# ---- AI / LLM crawlers ----
# Allowed by default — explicit decision (2026-04-30) to let LLM
# training and citation traffic through. The disallows above still
# apply to them (they shouldn't be hitting /api/ either), but the
# public marketing/app pages are fair game. If you ever change your
# mind, add per-bot blocks here for GPTBot, ClaudeBot, anthropic-ai,
# CCBot, Google-Extended, PerplexityBot, Bytespider, etc.

# ---- Aggressive SEO scrapers --------------------------------------
# Not search engines — these scrape the web to build their own
# analytics products on top of yours. They consume bandwidth without
# providing referral value. The DataFeedController rate-limiter
# catches some via UA matching; robots.txt is cheap layer-7 backup
# for the cooperative subset.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# ---- Sitemap ------------------------------------------------------
Sitemap: https://www.zenbotscanner.com/sitemap.xml
# Static scan pages (generated every 15 min into live-wwwroot/scans/).
Sitemap: https://www.zenbotscanner.com/scans/sitemap.xml