# Public crawl rules — search engines and AI answer engines stay allowed.
# Training-only / bulk-scrape agents are blocked below.
# robots.txt is advisory; enforcement can be added via Cloudflare if needed.
#
# Text and data mining rights are expressly reserved under Art. 4(3) of
# Directive (EU) 2019/790. Machine-readable reservation:
#   https://www.nylundsboathouse.com/.well-known/tdmrep.json
# Human-readable terms:
#   https://www.nylundsboathouse.com/terms

User-agent: *
Allow: /

# Internal / non-index surfaces
Disallow: /admin
Disallow: /api
Disallow: /auth
Disallow: /boats/carousel

# Same surfaces under locale prefixes — the catch-all url pattern makes
# /sv/admin, /de/api/*, /fi/veneet/carousel etc. resolve to the same routes.
Disallow: /*/admin
Disallow: /*/api
Disallow: /*/auth
Disallow: /*/carousel

# --- AI training / bulk scrapers (not search indexing) ---
# OpenAI model training (OAI-SearchBot / ChatGPT-User remain under * → allowed)
User-agent: GPTBot
Disallow: /

# Google Gemini / Vertex AI training (Googlebot search unaffected)
User-agent: Google-Extended
Disallow: /

# Apple Intelligence / Apple foundation models (Applebot search unaffected)
User-agent: Applebot-Extended
Disallow: /

# Common Crawl — source for many training datasets
User-agent: CCBot
Disallow: /

# Anthropic training (Claude retrieval bots may still use other UAs under *)
User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Meta AI training
User-agent: FacebookBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# ByteDance
User-agent: Bytespider
Disallow: /

# Amazon / Alexa training-style crawl
User-agent: Amazonbot
Disallow: /

# Cohere
User-agent: cohere-ai
Disallow: /

# Diffbot
User-agent: Diffbot
Disallow: /

# Image scrapers often used for training sets
User-agent: ImagesiftBot
Disallow: /

Sitemap: https://www.nylundsboathouse.com/sitemap.xml