# Configuración global para todos los robots
User-agent: *
Allow: /
Crawl-delay: 10

# Directorios y páginas restringidas
Disallow: /admin/
Disallow: /chat/
Disallow: /professions/
Disallow: /subscriptions/
Disallow: /wallet/
Disallow: /cart/
Disallow: /services/
Disallow: /notifications/
Disallow: /privacy-policy/
Disallow: /use-conditions/

# Extensiones y patrones adicionales a restringir
Disallow: /*.php$
Disallow: /*?*
Disallow: /*/temp/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /includes/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/*/readme.txt
Disallow: /*.sql$
Disallow: /*.log$
Disallow: /*.bak$
Disallow: /*.json$
Disallow: /*.xml$
Disallow: /search/
Disallow: /*?s=*

# Configuración específica para bots principales
User-agent: Googlebot
Crawl-delay: 5
Allow: /*.xml$

User-agent: Bingbot
Crawl-delay: 7
Allow: /*.xml$

# Disable OpenAI's crawler
User-agent: GPTBot
Disallow: /

# Disable Amazon's crawler
User-agent: Amazonbot
Disallow: /

# Configuración para bots agresivos
User-agent: AhrefsBot
Crawl-delay: 30

# User-agent: SemrushBot
# Crawl-delay: 30

User-agent: MJ12bot
Crawl-delay: 30

User-agent: Baiduspider
Crawl-delay: 40
Disallow: /

User-agent: Meta-ExternalAgent
Crawl-delay: 40
Disallow: /

User-agent: DataForSeoBot
Crawl-delay: 40
Disallow: /

User-agent: ClaudeBot
Crawl-delay: 40
Disallow: /

User-agent: SemrushBot
Crawl-delay: 40
Disallow: /


# Mapas del sitio
Sitemap: https://www.febelink.com/sitemap.xml

# Fin del archivo