# https://www.robotstxt.org/robotstxt.html
User-agent: *
Allow: /
Disallow: /api/
Disallow: /ws/
Disallow: /login
Disallow: /signup
Disallow: /register
Disallow: /reset-password
Disallow: /forgot-password
Disallow: /verify-email
Disallow: /dashboard
Disallow: /wallet
Disallow: /settings
Disallow: /profile
Disallow: /transactions
Disallow: /family
Disallow: /subscription
Disallow: /referral
Disallow: /redeem
Disallow: /notifications
Disallow: /verification
Disallow: /delete-account
Disallow: /delete-confirm
Disallow: /dashboard-demo
Disallow: /login-demo
Disallow: /dev-clear-sw
Disallow: /system-status
Disallow: /turnstile-debug
# The Bitcoin whitepaper PDF is duplicated across thousands of sites, so Google
# won't index our copy (reports it under "Crawled - currently not indexed").
# Disallow it so crawl attention isn't spent on a page that can never index.
Disallow: /bitcoin-whitepaper.pdf
Allow: /terms
Allow: /privacy
Allow: /support
Allow: /joinus
Allow: /blog
Allow: /explorer
Allow: /sats-back
Allow: /family-corner
Allow: /refer-a-friend
Allow: /courses
Allow: /glossary
Allow: /convert
Allow: /llms.txt
Allow: /llms-full.txt

# AI assistant context (machine-readable summaries of the site):
#   Summary index: https://www.btcbitbybit.com/llms.txt
#   Full content:  https://www.btcbitbybit.com/llms-full.txt

# Sitemap locations
Sitemap: https://www.btcbitbybit.com/sitemap.xml
Sitemap: https://www.btcbitbybit.com/blog/sitemap.xml

# Specific bot rules
User-agent: Googlebot
Disallow: /api/
Disallow: /ws/
Disallow: /bitcoin-whitepaper.pdf

User-agent: Bingbot
Disallow: /api/
Disallow: /ws/
Disallow: /bitcoin-whitepaper.pdf

# AI crawlers (AEO / generative-engine optimisation)
# Explicitly allowed so they can index our content for ChatGPT, Claude,
# Perplexity, Google Gemini, and others. Same private-area exclusions
# apply as the default group.
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /ws/
Disallow: /dashboard
Disallow: /wallet
Disallow: /settings
Disallow: /profile
Disallow: /family
Disallow: /subscription
Disallow: /referral
Disallow: /redeem
Disallow: /notifications

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /ws/
Disallow: /dashboard
Disallow: /wallet
Disallow: /settings
Disallow: /profile
Disallow: /family
Disallow: /subscription
Disallow: /referral
Disallow: /redeem
Disallow: /notifications

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /ws/
Disallow: /dashboard
Disallow: /wallet
Disallow: /settings
Disallow: /profile
Disallow: /family
Disallow: /subscription
Disallow: /referral
Disallow: /redeem
Disallow: /notifications

User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /ws/

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /ws/
Disallow: /dashboard
Disallow: /wallet
Disallow: /settings
Disallow: /profile
Disallow: /family
Disallow: /subscription
Disallow: /referral
Disallow: /redeem
Disallow: /notifications

User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /ws/
Disallow: /dashboard
Disallow: /wallet
Disallow: /settings
Disallow: /profile
Disallow: /family
Disallow: /subscription
Disallow: /referral
Disallow: /redeem
Disallow: /notifications

# Block bad bots (SEO scrapers — no value to us, only adds load)
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /