NeuralCrawl

Grupo Bimbo / robots.txt snapshot

← back to grupobimbo.com · fetched 2026-06-27T17:18:59Z (1mo ago) · HTTP 200 · 4200 bytes · sha256 c37586e7a5d2b4f8 · raw

final URL: https://www.grupobimbo.com/robots.txt

1#
2# robots.txt
3#
4# This file is to prevent the crawling and indexing of certain parts
5# of your site by web crawlers and spiders run by sites like Yahoo!
6# and Google. By telling these "robots" where not to go on your site,
7# you save bandwidth and server resources.
8#
9# This file will be ignored unless it is at the root of your host:
10# Used: http://example.com/robots.txt
11# Ignored: http://example.com/site/robots.txt
12#
13# For more information about the robots.txt standard, see:
14# http://www.robotstxt.org/robotstxt.html
15
16# ============================================================
17# AI / LLM crawlers - explicit allowlist (Auditoria WEB-AEO 9-jun-2026)
18# Each AI bot is allowed to crawl public content but blocked from
19# private/admin paths, mirroring the User-agent: * rules below.
20# ============================================================
21User-agent: GPTBot
22Allow: /
23Disallow: /core/
24Disallow: /profiles/
25Disallow: /admin/
26Disallow: /comment/reply/
27Disallow: /filter/tips
28Disallow: /node/add/
29Disallow: /search/
30Disallow: /user/register/
31Disallow: /user/password/
32Disallow: /user/login/
33Disallow: /user/logout/
34
35User-agent: OAI-SearchBot
36Allow: /
37Disallow: /core/
38Disallow: /profiles/
39Disallow: /admin/
40Disallow: /comment/reply/
41Disallow: /filter/tips
42Disallow: /node/add/
43Disallow: /search/
44Disallow: /user/register/
45Disallow: /user/password/
46Disallow: /user/login/
47Disallow: /user/logout/
48
49User-agent: ChatGPT-User
50Allow: /
51Disallow: /core/
52Disallow: /profiles/
53Disallow: /admin/
54Disallow: /comment/reply/
55Disallow: /filter/tips
56Disallow: /node/add/
57Disallow: /search/
58Disallow: /user/register/
59Disallow: /user/password/
60Disallow: /user/login/
61Disallow: /user/logout/
62
63User-agent: ClaudeBot
64Allow: /
65Disallow: /core/
66Disallow: /profiles/
67Disallow: /admin/
68Disallow: /comment/reply/
69Disallow: /filter/tips
70Disallow: /node/add/
71Disallow: /search/
72Disallow: /user/register/
73Disallow: /user/password/
74Disallow: /user/login/
75Disallow: /user/logout/
76
77User-agent: Claude-Web
78Allow: /
79Disallow: /core/
80Disallow: /profiles/
81Disallow: /admin/
82Disallow: /comment/reply/
83Disallow: /filter/tips
84Disallow: /node/add/
85Disallow: /search/
86Disallow: /user/register/
87Disallow: /user/password/
88Disallow: /user/login/
89Disallow: /user/logout/
90
91User-agent: PerplexityBot
92Allow: /
93Disallow: /core/
94Disallow: /profiles/
95Disallow: /admin/
96Disallow: /comment/reply/
97Disallow: /filter/tips
98Disallow: /node/add/
99Disallow: /search/
100Disallow: /user/register/
101Disallow: /user/password/
102Disallow: /user/login/
103Disallow: /user/logout/
104
105User-agent: Google-Extended
106Allow: /
107Disallow: /core/
108Disallow: /profiles/
109Disallow: /admin/
110Disallow: /comment/reply/
111Disallow: /filter/tips
112Disallow: /node/add/
113Disallow: /search/
114Disallow: /user/register/
115Disallow: /user/password/
116Disallow: /user/login/
117Disallow: /user/logout/
118
119User-agent: CCBot
120Allow: /
121Disallow: /core/
122Disallow: /profiles/
123Disallow: /admin/
124Disallow: /comment/reply/
125Disallow: /filter/tips
126Disallow: /node/add/
127Disallow: /search/
128Disallow: /user/register/
129Disallow: /user/password/
130Disallow: /user/login/
131Disallow: /user/logout/
132
133User-agent: *
134# CSS, JS, Images
135Allow: /core/*.css$
136Allow: /core/*.css?
137Allow: /core/*.js$
138Allow: /core/*.js?
139Allow: /core/*.gif
140Allow: /core/*.jpg
141Allow: /core/*.jpeg
142Allow: /core/*.png
143Allow: /core/*.svg
144Allow: /profiles/*.css$
145Allow: /profiles/*.css?
146Allow: /profiles/*.js$
147Allow: /profiles/*.js?
148Allow: /profiles/*.gif
149Allow: /profiles/*.jpg
150Allow: /profiles/*.jpeg
151Allow: /profiles/*.png
152Allow: /profiles/*.svg
153# Directories
154Disallow: /core/
155Disallow: /profiles/
156# Files
157Disallow: /README.txt
158Disallow: /web.config
159# Paths (clean URLs)
160Disallow: /admin/
161Disallow: /comment/reply/
162Disallow: /filter/tips
163Disallow: /node/add/
164Disallow: /search/
165Disallow: /user/register/
166Disallow: /user/password/
167Disallow: /user/login/
168Disallow: /user/logout/
169# Paths (no clean URLs)
170Disallow: /index.php/admin/
171Disallow: /index.php/comment/reply/
172Disallow: /index.php/filter/tips
173Disallow: /index.php/node/add/
174Disallow: /index.php/search/
175Disallow: /index.php/user/password/
176Disallow: /index.php/user/register/
177Disallow: /index.php/user/login/
178Disallow: /index.php/user/logout/
179
180# Sitemap
181Sitemap: https://www.grupobimbo.com/sitemap.xml