NeuralCrawl

Weather / robots.txt snapshot

← back to weather.com · fetched 2026-07-28T22:23:38Z (8d ago) · HTTP 200 · 4940 bytes · sha256 277b44f1cd907fe6 · raw

final URL: https://weather.com/robots.txt

1#
2# /robots.txt
3#
4
5#
6# Last updated by noel.madali 07/27/2026
7#
8
9# Explicit Allows
10
11User-agent: ChatGPT-User
12Allow: /
13
14User-agent: AmazonAdBot
15Allow: /
16
17# Disallowed for PhantomJS
18
19User-agent: *
20# Crawl-delay: 10
21
22User-agent: CriteoBot/0.1
23Disallow:
24Crawl-delay: .02
25
26# Directories
27Disallow: /includes/
28Disallow: /life/
29Disallow: /misc/
30Disallow: /modules/
31Disallow: /profiles/
32Disallow: /scripts/
33Disallow: /themes/
34Disallow: /api/payload
35Allow: /api/payload/v1/sitemaps
36Disallow: /taxonomy/
37# Files
38Disallow: /appspromo
39Disallow: /CHANGELOG.txt
40Disallow: /cron.php
41Disallow: /INSTALL.mysql.txt
42Disallow: /INSTALL.pgsql.txt
43Disallow: /INSTALL.sqlite.txt
44Disallow: /install.php
45Disallow: /INSTALL.txt
46Disallow: /LICENSE.txt
47Disallow: /MAINTAINERS.txt
48Disallow: /update.php
49Disallow: /UPGRADE.txt
50Disallow: /xmlrpc.php
51# Paths (clean URLs)
52Disallow: /migration/
53Disallow: /admin/
54Disallow: /comment/reply/
55Disallow: /filter/tips/
56Disallow: /node/add/
57Disallow: /search/
58Disallow: /user/register/
59Disallow: /user/password/
60Disallow: /user/login/
61Disallow: /user/logout/
62Disallow: /g00/
63Disallow: /g01/
64Disallow: /g02/
65Disallow: /g03/
66Disallow: /g04/
67Disallow: /g05/
68Disallow: /g06/
69Disallow: /g07/
70Disallow: /g08/
71Disallow: /g09/
72Disallow: /g10/
73Disallow: /g11/
74Disallow: /g12/
75Disallow: /g13/
76Disallow: /g14/
77Disallow: /g15/
78Disallow: /g16/
79Disallow: /g17/
80Disallow: /g18/
81Disallow: /g19/
82Disallow: /g20/
83Disallow: /g21/
84Disallow: /g22/
85Disallow: /g23/
86Disallow: /g24/
87Disallow: /g25/
88Disallow: /g26/
89Disallow: /g27/
90Disallow: /g28/
91Disallow: /g29/
92Disallow: /g30/
93Disallow: /g31/
94Disallow: /g32/
95Disallow: /g33/
96Disallow: /g34/
97Disallow: /g35/
98Disallow: /g36/
99Disallow: /g37/
100Disallow: /g38/
101Disallow: /g39/
102Disallow: /g40/
103Disallow: /g41/
104Disallow: /g42/
105Disallow: /g43/
106Disallow: /g44/
107Disallow: /g45/
108Disallow: /g46/
109Disallow: /g47/
110Disallow: /g48/
111Disallow: /g49/
112Disallow: /g50/
113Disallow: /g51/
114Disallow: /g52/
115Disallow: /g53/
116Disallow: /g54/
117Disallow: /g55/
118Disallow: /g56/
119Disallow: /g57/
120Disallow: /g58/
121Disallow: /g59/
122Disallow: /g60/
123Disallow: /g61/
124Disallow: /g62/
125Disallow: /g63/
126Disallow: /g64/
127Disallow: /g65/
128Disallow: /g66/
129Disallow: /g67/
130Disallow: /g68/
131Disallow: /g69/
132Disallow: /g70/
133Disallow: /g71/
134Disallow: /g72/
135Disallow: /g73/
136Disallow: /g74/
137Disallow: /g75/
138Disallow: /g76/
139Disallow: /g77/
140Disallow: /g78/
141Disallow: /g79/
142Disallow: /g80/
143Disallow: /g81/
144Disallow: /g82/
145Disallow: /g83/
146Disallow: /g84/
147Disallow: /g85/
148Disallow: /g86/
149Disallow: /g87/
150Disallow: /g88/
151Disallow: /g89/
152Disallow: /g90/
153Disallow: /g91/
154Disallow: /g92/
155Disallow: /g93/
156Disallow: /g94/
157Disallow: /g95/
158Disallow: /g96/
159Disallow: /g97/
160Disallow: /g98/
161Disallow: /g99/
162# Paths (no clean URLs)
163Disallow: /*?apiKey=
164Disallow: /*&apiKey=
165Disallow: /?q=admin/
166Disallow: /?q=comment/reply/
167Disallow: /?q=filter/tips/
168Disallow: /?q=node/add/
169Disallow: /?q=search/
170Disallow: /?q=user/password/
171Disallow: /?q=user/register/
172Disallow: /?q=user/login/
173Disallow: /?q=user/logout/
174
175Disallow: /sponsored
176Disallow: /ugc
177Disallow: /sponsored-content
178
179Disallow: /api/v1/p/redux-dal
180# International weather language URL paths (404s)
181Disallow: /tiempo/
182Disallow: /clima/
183Disallow: /temps/
184Disallow: /vreme/
185Disallow: /idojaras/
186Disallow: /pogoda/
187Disallow: /wetter/
188Disallow: /vejret/
189Disallow: /pocasi/
190Disallow: /cuaca/
191Disallow: /weer/
192
193#
194# Block Bots
195#
196
197User-agent: Amazonbot
198Disallow: /
199
200User-agent: anthropic-ai
201Disallow: /
202
203User-agent: Applebot-Extended
204Disallow: /
205
206User-agent: AwarioRssBot
207User-agent: AwarioSmartBot
208Disallow: /
209
210User-agent: Bytespider
211Disallow: /
212
213User-agent: CCBot
214Disallow: /
215
216User-agent: ClaudeBot
217Disallow: /
218
219User-agent: Claude-Web
220Disallow: /
221
222User-agent: cohere-ai
223Disallow: /
224
225User-agent: DataForSeoBot
226Disallow: /
227
228User-agent: Diffbot
229Disallow: /
230
231User-agent: FacebookBot
232Disallow: /
233
234User-agent: FriendlyCrawler
235Disallow: /
236
237User-agent: Google-Extended
238Disallow: /
239
240User-agent: GPTBot
241Disallow: /
242
243User-agent: ImagesiftBot
244Disallow: /
245
246User-agent: magpie-crawler
247Disallow: /
248
249User-agent: Meta-ExternalAgent
250User-agent: meta-externalagent
251Disallow: /
252
253User-agent: NewsNow
254Disallow: /
255
256User-agent: news-please
257Disallow: /
258
259User-agent: omgili
260Disallow: /
261
262User-agent: omgilibot
263Disallow: /
264
265User-agent: peer39_crawler
266User-agent: peer39_crawler/1.0
267Disallow: /
268
269User-agent: PerplexityBot
270Disallow: /
271
272User-agent: Quora-Bot
273Disallow: /
274
275User-agent: Scrapy
276Disallow: /
277
278User-agent: TurnitinBot
279Disallow: /
280
281#
282# Sitemaps
283#
284
285Sitemap: https://weather.com/en-US/sitemaps/sitemap.xml
286Sitemap: https://weather.com/pt-PT/sitemaps/sitemap.xml
287Sitemap: https://weather.com/de-DE/sitemaps/sitemap.xml
288Sitemap: https://weather.com/fr-FR/sitemaps/sitemap.xml
289Sitemap: https://weather.com/es-US/sitemaps/sitemap.xml
290Sitemap: https://weather.com/es-ES/sitemaps/sitemap.xml
291Sitemap: https://weather.com/en-IN/sitemaps/sitemap.xml
292Sitemap: https://weather.com/en-GB/sitemaps/sitemap.xml
293Sitemap: https://weather.com/en-CA/sitemaps/sitemap.xml