NeuralCrawl

Weather / robots.txt snapshot

← back to weather.com · fetched 2026-06-26T13:23:15Z (3h ago) · HTTP 200 · 5006 bytes · sha256 e02328a337962a37 · raw

final URL: https://weather.com/robots.txt

1#
2# /robots.txt
3#
4
5#
6# Last updated by nickhstr 03/16/2026
7#
8
9# Explicit Allows
10
11User-agent: ChatGPT-User
12Allow: /
13
14User-agent: AmazonAdBot
15Allow: /
16
17# Disallowed for PhantomJS
18
19User-agent: *
20# Crawl-delay: 10
21
22User-agent: CriteoBot/0.1
23Disallow:
24Crawl-delay: .02
25
26# Directories
27Disallow: /includes/
28Disallow: /life/
29Disallow: /misc/
30Disallow: /modules/
31Disallow: /profiles/
32Disallow: /scripts/
33Disallow: /themes/
34Disallow: /api/payload
35Allow: /api/payload/v1/sitemaps
36Disallow: /taxonomy/
37# Files
38Disallow: /appspromo
39Disallow: /CHANGELOG.txt
40Disallow: /cron.php
41Disallow: /INSTALL.mysql.txt
42Disallow: /INSTALL.pgsql.txt
43Disallow: /INSTALL.sqlite.txt
44Disallow: /install.php
45Disallow: /INSTALL.txt
46Disallow: /LICENSE.txt
47Disallow: /MAINTAINERS.txt
48Disallow: /update.php
49Disallow: /UPGRADE.txt
50Disallow: /xmlrpc.php
51# Paths (clean URLs)
52Disallow: /migration/
53Disallow: /admin/
54Disallow: /comment/reply/
55Disallow: /filter/tips/
56Disallow: /node/add/
57Disallow: /search/
58Disallow: /user/register/
59Disallow: /user/password/
60Disallow: /user/login/
61Disallow: /user/logout/
62Disallow: /g00/
63Disallow: /g01/
64Disallow: /g02/
65Disallow: /g03/
66Disallow: /g04/
67Disallow: /g05/
68Disallow: /g06/
69Disallow: /g07/
70Disallow: /g08/
71Disallow: /g09/
72Disallow: /g10/
73Disallow: /g11/
74Disallow: /g12/
75Disallow: /g13/
76Disallow: /g14/
77Disallow: /g15/
78Disallow: /g16/
79Disallow: /g17/
80Disallow: /g18/
81Disallow: /g19/
82Disallow: /g20/
83Disallow: /g21/
84Disallow: /g22/
85Disallow: /g23/
86Disallow: /g24/
87Disallow: /g25/
88Disallow: /g26/
89Disallow: /g27/
90Disallow: /g28/
91Disallow: /g29/
92Disallow: /g30/
93Disallow: /g31/
94Disallow: /g32/
95Disallow: /g33/
96Disallow: /g34/
97Disallow: /g35/
98Disallow: /g36/
99Disallow: /g37/
100Disallow: /g38/
101Disallow: /g39/
102Disallow: /g40/
103Disallow: /g41/
104Disallow: /g42/
105Disallow: /g43/
106Disallow: /g44/
107Disallow: /g45/
108Disallow: /g46/
109Disallow: /g47/
110Disallow: /g48/
111Disallow: /g49/
112Disallow: /g50/
113Disallow: /g51/
114Disallow: /g52/
115Disallow: /g53/
116Disallow: /g54/
117Disallow: /g55/
118Disallow: /g56/
119Disallow: /g57/
120Disallow: /g58/
121Disallow: /g59/
122Disallow: /g60/
123Disallow: /g61/
124Disallow: /g62/
125Disallow: /g63/
126Disallow: /g64/
127Disallow: /g65/
128Disallow: /g66/
129Disallow: /g67/
130Disallow: /g68/
131Disallow: /g69/
132Disallow: /g70/
133Disallow: /g71/
134Disallow: /g72/
135Disallow: /g73/
136Disallow: /g74/
137Disallow: /g75/
138Disallow: /g76/
139Disallow: /g77/
140Disallow: /g78/
141Disallow: /g79/
142Disallow: /g80/
143Disallow: /g81/
144Disallow: /g82/
145Disallow: /g83/
146Disallow: /g84/
147Disallow: /g85/
148Disallow: /g86/
149Disallow: /g87/
150Disallow: /g88/
151Disallow: /g89/
152Disallow: /g90/
153Disallow: /g91/
154Disallow: /g92/
155Disallow: /g93/
156Disallow: /g94/
157Disallow: /g95/
158Disallow: /g96/
159Disallow: /g97/
160Disallow: /g98/
161Disallow: /g99/
162# Paths (no clean URLs)
163Disallow: /*?apiKey=
164Disallow: /*&apiKey=
165Disallow: /?q=admin/
166Disallow: /?q=comment/reply/
167Disallow: /?q=filter/tips/
168Disallow: /?q=node/add/
169Disallow: /?q=search/
170Disallow: /?q=user/password/
171Disallow: /?q=user/register/
172Disallow: /?q=user/login/
173Disallow: /?q=user/logout/
174Disallow: /*?traffic_source
175Disallow: /*?base
176Disallow: /*?phenomena
177
178Disallow: /sponsored
179Disallow: /ugc
180Disallow: /sponsored-content
181
182Disallow: /api/v1/p/redux-dal
183# International weather language URL paths (404s)
184Disallow: /tiempo/
185Disallow: /clima/
186Disallow: /temps/
187Disallow: /vreme/
188Disallow: /idojaras/
189Disallow: /pogoda/
190Disallow: /wetter/
191Disallow: /vejret/
192Disallow: /pocasi/
193Disallow: /cuaca/
194Disallow: /weer/
195
196#
197# Block Bots
198#
199
200User-agent: Amazonbot
201Disallow: /
202
203User-agent: anthropic-ai
204Disallow: /
205
206User-agent: Applebot-Extended
207Disallow: /
208
209User-agent: AwarioRssBot
210User-agent: AwarioSmartBot
211Disallow: /
212
213User-agent: Bytespider
214Disallow: /
215
216User-agent: CCBot
217Disallow: /
218
219User-agent: ClaudeBot
220Disallow: /
221
222User-agent: Claude-Web
223Disallow: /
224
225User-agent: cohere-ai
226Disallow: /
227
228User-agent: DataForSeoBot
229Disallow: /
230
231User-agent: Diffbot
232Disallow: /
233
234User-agent: FacebookBot
235Disallow: /
236
237User-agent: FriendlyCrawler
238Disallow: /
239
240User-agent: Google-Extended
241Disallow: /
242
243User-agent: GPTBot
244Disallow: /
245
246User-agent: ImagesiftBot
247Disallow: /
248
249User-agent: magpie-crawler
250Disallow: /
251
252User-agent: Meta-ExternalAgent
253User-agent: meta-externalagent
254Disallow: /
255
256User-agent: NewsNow
257Disallow: /
258
259User-agent: news-please
260Disallow: /
261
262User-agent: omgili
263Disallow: /
264
265User-agent: omgilibot
266Disallow: /
267
268User-agent: peer39_crawler
269User-agent: peer39_crawler/1.0
270Disallow: /
271
272User-agent: PerplexityBot
273Disallow: /
274
275User-agent: Quora-Bot
276Disallow: /
277
278User-agent: Scrapy
279Disallow: /
280
281User-agent: TurnitinBot
282Disallow: /
283
284#
285# Sitemaps
286#
287
288Sitemap: https://weather.com/en-US/sitemaps/sitemap.xml
289Sitemap: https://weather.com/pt-PT/sitemaps/sitemap.xml
290Sitemap: https://weather.com/de-DE/sitemaps/sitemap.xml
291Sitemap: https://weather.com/fr-FR/sitemaps/sitemap.xml
292Sitemap: https://weather.com/es-US/sitemaps/sitemap.xml
293Sitemap: https://weather.com/es-ES/sitemaps/sitemap.xml
294Sitemap: https://weather.com/en-IN/sitemaps/sitemap.xml
295Sitemap: https://weather.com/en-GB/sitemaps/sitemap.xml
296Sitemap: https://weather.com/en-CA/sitemaps/sitemap.xml