NeuralCrawl

Weather / robots.txt snapshot

← back to weather.com · fetched 2026-09-16T08:45:16Z (3d ago) · HTTP 200 · 3539 bytes · sha256 28adb9aed15bb249 · raw

final URL: https://weather.com/robots.txt

1#
2# /robots.txt
3#
4
5#
6# Last updated by nick.hester 09/08/2026
7#
8
9#
10# AI / AEO bot policy (CW-9994)
11# We explicitly allow AI bots that fetch pages on behalf of a user's direct
12# request (answer engines like ChatGPT-User, Brightbot) so our content can
13# be cited in AI-generated answers. Bulk AI-training/indexing crawlers
14# remain blocked below under "Block Bots" (e.g. YouBot, Google-CloudVertexBot).
15#
16
17# Explicit Allows
18
19User-agent: ChatGPT-User
20Allow: /
21
22User-agent: AmazonAdBot
23Allow: /
24
25User-agent: Brightbot 1.0
26Allow: /
27
28User-agent: CriteoBot/0.1
29Allow: /
30
31# Disallowed for PhantomJS
32
33User-agent: *
34Crawl-delay: 10
35# Directories
36Disallow: /includes/
37Disallow: /life/
38Disallow: /misc/
39Disallow: /modules/
40Disallow: /profiles/
41Disallow: /scripts/
42Disallow: /themes/
43Disallow: /api/payload
44Allow: /api/payload/v1/sitemaps
45Disallow: /taxonomy/
46# Files
47Disallow: /appspromo
48Disallow: /CHANGELOG.txt
49Disallow: /cron.php
50Disallow: /INSTALL.mysql.txt
51Disallow: /INSTALL.pgsql.txt
52Disallow: /INSTALL.sqlite.txt
53Disallow: /install.php
54Disallow: /INSTALL.txt
55Disallow: /LICENSE.txt
56Disallow: /MAINTAINERS.txt
57Disallow: /update.php
58Disallow: /UPGRADE.txt
59Disallow: /xmlrpc.php
60# Paths (clean URLs)
61Disallow: /migration/
62Disallow: /admin/
63Disallow: /comment/reply/
64Disallow: /filter/tips/
65Disallow: /node/add/
66Disallow: /search/
67Disallow: /user/register/
68Disallow: /user/password/
69Disallow: /user/login/
70Disallow: /user/logout/
71# Paths (no clean URLs)
72Disallow: /*?apiKey=
73Disallow: /*&apiKey=
74Disallow: /?q=admin/
75Disallow: /?q=comment/reply/
76Disallow: /?q=filter/tips/
77Disallow: /?q=node/add/
78Disallow: /?q=search/
79Disallow: /?q=user/password/
80Disallow: /?q=user/register/
81Disallow: /?q=user/login/
82Disallow: /?q=user/logout/
83
84Disallow: /sponsored
85Disallow: /ugc
86Disallow: /sponsored-content
87
88Disallow: /api/v1/p/redux-dal
89# International weather language URL paths (404s)
90Disallow: /tiempo/
91Disallow: /clima/
92Disallow: /temps/
93Disallow: /vreme/
94Disallow: /idojaras/
95Disallow: /pogoda/
96Disallow: /wetter/
97Disallow: /vejret/
98Disallow: /pocasi/
99Disallow: /cuaca/
100Disallow: /weer/
101
102#
103# Block Bots
104#
105
106User-agent: Amazonbot
107Disallow: /
108
109User-agent: AwarioRssBot
110User-agent: AwarioSmartBot
111Disallow: /
112
113User-agent: Bytespider
114Disallow: /
115
116User-agent: CCBot
117Disallow: /
118
119User-agent: DataForSeoBot
120Disallow: /
121
122User-agent: Diffbot
123Disallow: /
124
125User-agent: FacebookBot
126Disallow: /
127
128User-agent: FriendlyCrawler
129Disallow: /
130
131User-agent: Google-CloudVertexBot
132Disallow: /
133
134User-agent: ImagesiftBot
135Disallow: /
136
137User-agent: magpie-crawler
138Disallow: /
139
140User-agent: NewsNow
141Disallow: /
142
143User-agent: news-please
144Disallow: /
145
146User-agent: omgili
147Disallow: /
148
149User-agent: omgilibot
150Disallow: /
151
152User-agent: peer39_crawler
153User-agent: peer39_crawler/1.0
154Disallow: /
155
156User-agent: Quora-Bot
157Disallow: /
158
159User-agent: Scrapy
160Disallow: /
161
162User-agent: TurnitinBot
163Disallow: /
164
165User-agent: YouBot
166Disallow: /
167
168#
169# Sitemaps
170#
171
172Sitemap: https://weather.com/en-US/sitemaps/sitemap.xml
173Sitemap: https://weather.com/pt-PT/sitemaps/sitemap.xml
174Sitemap: https://weather.com/de-DE/sitemaps/sitemap.xml
175Sitemap: https://weather.com/fr-FR/sitemaps/sitemap.xml
176Sitemap: https://weather.com/es-US/sitemaps/sitemap.xml
177Sitemap: https://weather.com/es-ES/sitemaps/sitemap.xml
178Sitemap: https://weather.com/en-IN/sitemaps/sitemap.xml
179Sitemap: https://weather.com/en-GB/sitemaps/sitemap.xml
180Sitemap: https://weather.com/en-CA/sitemaps/sitemap.xml
181Sitemap: https://weather.com/api/payload/v1/sitemaps/en-US/sitemap.xml
182Sitemap: https://weather.com/api/location/sitemaps/en-US/sitemap.xml