###############################################
# robots.txt - Permitir indexação e leitura por LLMs e buscadores
# Última atualização: 05/2026
###############################################

# Aplicar regras gerais a todos os robôs
User-agent: *
Allow: /                     # Permite acesso geral ao site
Disallow: /mapa-fst/         # Bloqueia painel internos
Disallow: /xml
Disallow: /Xml/
Disallow: /404/
Disallow: /*.pdf$
Disallow: /_arquivosestaticos/
Disallow: /content/itau/resultado-de-busca-generico
Disallow: /private/
Disallow: /auth/
Disallow: /login/
Disallow: /minhaconta/
Disallow: /api/internal/
Disallow: /admin/            

# Exceções — liberar arquivos que são necessários para o funcionamento visual do site
Allow: /investimentos/
Allow: /api/public/
Allow: /api/hub-content/
Allow: /api/llm-access/
Allow: /static/images/
Allow: /static/assets/


###############################################
# Permissões específicas para LLMs (modelos de IA)
# Esses agentes pertencem a sistemas de IA generativa e buscadores com extensão LLM
###############################################

# OpenAI / ChatGPT
User-agent: OAI-SearchBot
Allow: /
Allow: /investimentos/
Disallow: /login/
Disallow: /private/
Disallow: /api/internal/

# Google / Gemini (para acesso ampliado de IA)
User-agent: Google-Extended
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /
Allow: /investimentos/
Disallow: /auth/
Disallow: /api/internal/

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /investimentos/
Disallow: /auth/
Disallow: /api/internal/

# Microsoft Bing / Copilot
User-agent: BingBot
Allow: /

# Apple / Applebot e Applebot-Extended (para Siri, Spotlight, Apple Intelligence)
User-agent: Applebot
Allow: /
User-agent: Applebot-Extended
Allow: /

# Meta (Facebook / Instagram)
User-agent: facebookexternalhit
Allow: /
User-agent: meta-externalagent
Allow: /

# LinkedIn
User-agent: LinkedInBot
Allow: /

# Amazon (Alexa, AWS, Amazon Search)
User-agent: Amazonbot
Allow: /

# Common Crawl (fornece muitos conjuntos de dados usados em IA)
User-agent: CCBot
Allow: /


Crawl-delay: 10

###############################################
# Sitemap — referência oficial de URLs do site
###############################################

Sitemap: https://itau.com.br/investimentos/sitemap.xml
Sitemap: https://itau.com.br/investimentos/sitemap-home.xml
Sitemap: https://itau.com.br/investimentos/tipos-de-investimentos/sitemap.xml
Sitemap: https://itau.com.br/investimentos/analises/sitemap.xml
Sitemap: https://itau.com.br/investimentos/carteiras-recomendadas/sitemap.xml
Sitemap: https://itau.com.br/investimentos/noticias/sitemap.xml
Sitemap: https://itau.com.br/investimentos/blog/sitemap.xml
Sitemap: https://itau.com.br/investimentos/impactos-de-mercado/sitemap.xml
Sitemap: https://itau.com.br/investimentos/servicos/sitemap.xml

###############################################

# Itau Unibanco
Sitemap: https://www.itau.com.br/sitemap.xml

###############################################
# Recomendações adicionais (opcionais)
# - Adicione no <head> das páginas:
#   <meta name="robots" content="index, follow">
#   <meta name="ai-access" content="allow">
# - Esses metadados ajudam novos crawlers de IA a identificar permissão explícita.
###############################################