# Essential search engines - unrestricted access
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: AdsBot-Google
User-agent: bingbot
User-agent: msnbot
User-agent: Slurp
Allow: /
Crawl-delay: 0
Disallow: /admin
Disallow: /account/
Disallow: /cart

# Beneficial crawlers - moderate access
User-agent: Applebot
User-agent: Amazonbot
User-agent: facebookexternalhit
User-agent: Twitterbot
User-agent: LinkedInBot
Allow: /
Crawl-delay: 2
Disallow: /admin
Disallow: /account/
Disallow: /cart

# Facebook aggressive crawler - heavily rate limited
User-agent: meta-externalagent
Crawl-delay: 60
Request-rate: 1/60
Allow: /
Disallow: /admin
Disallow: /account/
Disallow: /cart

# SEO tools - aggressive crawl delay
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: DotBot
User-agent: MJ12bot
User-agent: Screaming Frog SEO Spider
Crawl-delay: 30
Request-rate: 1/30
Disallow: /admin
Disallow: /account/
Disallow: /cart

# AI bots - allowed for AEO/GEO training and user-facing citations
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: CCBot
User-agent: cohere-ai
Allow: /
Crawl-delay: 5
Disallow: /admin
Disallow: /account/
Disallow: /cart

# International search engines - moderate access
User-agent: Baiduspider
User-agent: Baiduspider-image
User-agent: Yandex
User-agent: YandexBot
User-agent: Sogou web spider
User-agent: SeznamBot
Crawl-delay: 20
Request-rate: 1/20
Disallow: /admin
Disallow: /account/
Disallow: /cart

# Scrapers and generic tools - blocked
User-agent: scrapy
User-agent: python-requests
User-agent: curl
User-agent: wget
User-agent: Apache-HttpClient
User-agent: Go-http-client
Disallow: /

# Honeypot paths to identify bad bots
User-agent: *
Disallow: /wp-admin
Disallow: /wp-login.php
Disallow: /.env
Disallow: /admin.php
Disallow: /config.php
Disallow: /.git/

# Default rules for all other agents
User-agent: *
Allow: /
Crawl-delay: 10
Disallow: /admin
Disallow: /account/
Disallow: /cart

# Allow crawling of all public pages
Allow: /shop
Allow: /blog
Allow: /tutorials
Allow: /guides
Allow: /travel
Allow: /team
Allow: /contact
Allow: /brands

# Sitemap locations
Sitemap: https://olefloridaflyshop.com/sitemap.xml
Sitemap: https://olefloridaflyshop.com/sitemap-pages.xml
Sitemap: https://olefloridaflyshop.com/sitemap-blogs.xml
Sitemap: https://olefloridaflyshop.com/sitemap-tutorials.xml
Sitemap: https://olefloridaflyshop.com/sitemap-products.xml
Sitemap: https://olefloridaflyshop.com/sitemap-collections.xml