Weather / robots.txt
← back to weather.com · change detected 3d ago (2026-09-16T08:45:16Z) · modified +20 −133
What changed
- Added rules for user-agent ‘Brightbot 1.0’
- Blocked user-agent ‘Google-CloudVertexBot’ entirely
- 🤖 AIBlocked YouBot (You.com) entirely
- 🤖 AIUnblocked anthropic-ai (Anthropic) - rules removed
- 🤖 AIUnblocked Applebot-Extended (Apple) - rules removed
- 🤖 AIUnblocked Claude-Web (Anthropic) - rules removed
- 🤖 AIUnblocked ClaudeBot (Anthropic) - rules removed
- 🤖 AIUnblocked cohere-ai (Cohere) - rules removed
- 🤖 AIUnblocked Google-Extended (Google) - rules removed
- 🤖 AIUnblocked GPTBot (OpenAI) - rules removed
- 🤖 AIUnblocked meta-externalagent (Meta) - rules removed
- 🤖 AIUnblocked PerplexityBot (Perplexity) - rules removed
- Added Allow: /api/payload/v1/sitemaps for *
- Added Disallow: /*&apiKey= for *
- Added Disallow: /*?apiKey= for *
- Added Disallow: /?q=admin/ for *
- Added Disallow: /?q=comment/reply/ for *
- Added Disallow: /?q=filter/tips/ for *
- Added Disallow: /?q=node/add/ for *
- Added Disallow: /?q=search/ for *
- Added Disallow: /?q=user/login/ for *
- Added Disallow: /?q=user/logout/ for *
- Added Disallow: /?q=user/password/ for *
- Added Disallow: /?q=user/register/ for *
- Added Disallow: /CHANGELOG.txt for *
- Added Disallow: /INSTALL.mysql.txt for *
- Added Disallow: /INSTALL.pgsql.txt for *
- Added Disallow: /INSTALL.sqlite.txt for *
- Added Disallow: /INSTALL.txt for *
- Added Disallow: /LICENSE.txt for *
- Added Disallow: /MAINTAINERS.txt for *
- Added Disallow: /UPGRADE.txt for *
- Added Disallow: /admin/ for *
- Added Disallow: /api/payload for *
- Added Disallow: /api/v1/p/redux-dal for *
- Added Disallow: /appspromo for *
- Added Disallow: /clima/ for *
- Added Disallow: /comment/reply/ for *
- Added Disallow: /cron.php for *
- Added Disallow: /cuaca/ for *
- Added Disallow: /filter/tips/ for *
- Added Disallow: /idojaras/ for *
- Added Disallow: /includes/ for *
- Added Disallow: /install.php for *
- Added Disallow: /life/ for *
- Added Disallow: /migration/ for *
- Added Disallow: /misc/ for *
- Added Disallow: /modules/ for *
- Added Disallow: /node/add/ for *
- Added Disallow: /pocasi/ for *
- ... and 184 more change(s)
Diff old (2026-08-21T12:17:12Z) → new (2026-09-16T08:45:16Z)
| @@ -3,7 +3,15 @@ | ||
| 3 | 3 | # |
| 4 | 4 | |
| 5 | 5 | # |
| 6 | - # Last updated by noel.madali 08/20/2026 | |
| 6 | + # Last updated by nick.hester 09/08/2026 | |
| 7 | + # | |
| 8 | + | |
| 9 | + # | |
| 10 | + # AI / AEO bot policy (CW-9994) | |
| 11 | + # We explicitly allow AI bots that fetch pages on behalf of a user's direct | |
| 12 | + # request (answer engines like ChatGPT-User, Brightbot) so our content can | |
| 13 | + # be cited in AI-generated answers. Bulk AI-training/indexing crawlers | |
| 14 | + # remain blocked below under "Block Bots" (e.g. YouBot, Google-CloudVertexBot). | |
| 7 | 15 | # |
| 8 | 16 | |
| 9 | 17 | # Explicit Allows |
| @@ -14,15 +22,16 @@ | ||
| 14 | 22 | User-agent: AmazonAdBot |
| 15 | 23 | Allow: / |
| 16 | 24 | |
| 25 | + User-agent: Brightbot 1.0 | |
| 26 | + Allow: / | |
| 27 | + | |
| 28 | + User-agent: CriteoBot/0.1 | |
| 29 | + Allow: / | |
| 30 | + | |
| 17 | 31 | # Disallowed for PhantomJS |
| 18 | 32 | |
| 19 | 33 | User-agent: * |
| 20 | - # Crawl-delay: 10 | |
| 21 | - | |
| 22 | - User-agent: CriteoBot/0.1 | |
| 23 | - Disallow: | |
| 24 | - Crawl-delay: .02 | |
| 25 | - | |
| 34 | + Crawl-delay: 10 | |
| 26 | 35 | # Directories |
| 27 | 36 | Disallow: /includes/ |
| 28 | 37 | Disallow: /life/ |
| @@ -59,106 +68,6 @@ | ||
| 59 | 68 | Disallow: /user/password/ |
| 60 | 69 | Disallow: /user/login/ |
| 61 | 70 | Disallow: /user/logout/ |
| 62 | - Disallow: /g00/ | |
| 63 | - Disallow: /g01/ | |
| 64 | - Disallow: /g02/ | |
| 65 | - Disallow: /g03/ | |
| 66 | - Disallow: /g04/ | |
| 67 | - Disallow: /g05/ | |
| 68 | - Disallow: /g06/ | |
| 69 | - Disallow: /g07/ | |
| 70 | - Disallow: /g08/ | |
| 71 | - Disallow: /g09/ | |
| 72 | - Disallow: /g10/ | |
| 73 | - Disallow: /g11/ | |
| 74 | - Disallow: /g12/ | |
| 75 | - Disallow: /g13/ | |
| 76 | - Disallow: /g14/ | |
| 77 | - Disallow: /g15/ | |
| 78 | - Disallow: /g16/ | |
| 79 | - Disallow: /g17/ | |
| 80 | - Disallow: /g18/ | |
| 81 | - Disallow: /g19/ | |
| 82 | - Disallow: /g20/ | |
| 83 | - Disallow: /g21/ | |
| 84 | - Disallow: /g22/ | |
| 85 | - Disallow: /g23/ | |
| 86 | - Disallow: /g24/ | |
| 87 | - Disallow: /g25/ | |
| 88 | - Disallow: /g26/ | |
| 89 | - Disallow: /g27/ | |
| 90 | - Disallow: /g28/ | |
| 91 | - Disallow: /g29/ | |
| 92 | - Disallow: /g30/ | |
| 93 | - Disallow: /g31/ | |
| 94 | - Disallow: /g32/ | |
| 95 | - Disallow: /g33/ | |
| 96 | - Disallow: /g34/ | |
| 97 | - Disallow: /g35/ | |
| 98 | - Disallow: /g36/ | |
| 99 | - Disallow: /g37/ | |
| 100 | - Disallow: /g38/ | |
| 101 | - Disallow: /g39/ | |
| 102 | - Disallow: /g40/ | |
| 103 | - Disallow: /g41/ | |
| 104 | - Disallow: /g42/ | |
| 105 | - Disallow: /g43/ | |
| 106 | - Disallow: /g44/ | |
| 107 | - Disallow: /g45/ | |
| 108 | - Disallow: /g46/ | |
| 109 | - Disallow: /g47/ | |
| 110 | - Disallow: /g48/ | |
| 111 | - Disallow: /g49/ | |
| 112 | - Disallow: /g50/ | |
| 113 | - Disallow: /g51/ | |
| 114 | - Disallow: /g52/ | |
| 115 | - Disallow: /g53/ | |
| 116 | - Disallow: /g54/ | |
| 117 | - Disallow: /g55/ | |
| 118 | - Disallow: /g56/ | |
| 119 | - Disallow: /g57/ | |
| 120 | - Disallow: /g58/ | |
| 121 | - Disallow: /g59/ | |
| 122 | - Disallow: /g60/ | |
| 123 | - Disallow: /g61/ | |
| 124 | - Disallow: /g62/ | |
| 125 | - Disallow: /g63/ | |
| 126 | - Disallow: /g64/ | |
| 127 | - Disallow: /g65/ | |
| 128 | - Disallow: /g66/ | |
| 129 | - Disallow: /g67/ | |
| 130 | - Disallow: /g68/ | |
| 131 | - Disallow: /g69/ | |
| 132 | - Disallow: /g70/ | |
| 133 | - Disallow: /g71/ | |
| 134 | - Disallow: /g72/ | |
| 135 | - Disallow: /g73/ | |
| 136 | - Disallow: /g74/ | |
| 137 | - Disallow: /g75/ | |
| 138 | - Disallow: /g76/ | |
| 139 | - Disallow: /g77/ | |
| 140 | - Disallow: /g78/ | |
| 141 | - Disallow: /g79/ | |
| 142 | - Disallow: /g80/ | |
| 143 | - Disallow: /g81/ | |
| 144 | - Disallow: /g82/ | |
| 145 | - Disallow: /g83/ | |
| 146 | - Disallow: /g84/ | |
| 147 | - Disallow: /g85/ | |
| 148 | - Disallow: /g86/ | |
| 149 | - Disallow: /g87/ | |
| 150 | - Disallow: /g88/ | |
| 151 | - Disallow: /g89/ | |
| 152 | - Disallow: /g90/ | |
| 153 | - Disallow: /g91/ | |
| 154 | - Disallow: /g92/ | |
| 155 | - Disallow: /g93/ | |
| 156 | - Disallow: /g94/ | |
| 157 | - Disallow: /g95/ | |
| 158 | - Disallow: /g96/ | |
| 159 | - Disallow: /g97/ | |
| 160 | - Disallow: /g98/ | |
| 161 | - Disallow: /g99/ | |
| 162 | 71 | # Paths (no clean URLs) |
| 163 | 72 | Disallow: /*?apiKey= |
| 164 | 73 | Disallow: /*&apiKey= |
| @@ -197,12 +106,6 @@ | ||
| 197 | 106 | User-agent: Amazonbot |
| 198 | 107 | Disallow: / |
| 199 | 108 | |
| 200 | - User-agent: anthropic-ai | |
| 201 | - Disallow: / | |
| 202 | - | |
| 203 | - User-agent: Applebot-Extended | |
| 204 | - Disallow: / | |
| 205 | - | |
| 206 | 109 | User-agent: AwarioRssBot |
| 207 | 110 | User-agent: AwarioSmartBot |
| 208 | 111 | Disallow: / |
| @@ -211,15 +114,6 @@ | ||
| 211 | 114 | Disallow: / |
| 212 | 115 | |
| 213 | 116 | User-agent: CCBot |
| 214 | - Disallow: / | |
| 215 | - | |
| 216 | - User-agent: ClaudeBot | |
| 217 | - Disallow: / | |
| 218 | - | |
| 219 | - User-agent: Claude-Web | |
| 220 | - Disallow: / | |
| 221 | - | |
| 222 | - User-agent: cohere-ai | |
| 223 | 117 | Disallow: / |
| 224 | 118 | |
| 225 | 119 | User-agent: DataForSeoBot |
| @@ -234,20 +128,13 @@ | ||
| 234 | 128 | User-agent: FriendlyCrawler |
| 235 | 129 | Disallow: / |
| 236 | 130 | |
| 237 | - User-agent: Google-Extended | |
| 238 | - Disallow: / | |
| 239 | - | |
| 240 | - User-agent: GPTBot | |
| 131 | + User-agent: Google-CloudVertexBot | |
| 241 | 132 | Disallow: / |
| 242 | 133 | |
| 243 | 134 | User-agent: ImagesiftBot |
| 244 | 135 | Disallow: / |
| 245 | 136 | |
| 246 | 137 | User-agent: magpie-crawler |
| 247 | - Disallow: / | |
| 248 | - | |
| 249 | - User-agent: Meta-ExternalAgent | |
| 250 | - User-agent: meta-externalagent | |
| 251 | 138 | Disallow: / |
| 252 | 139 | |
| 253 | 140 | User-agent: NewsNow |
| @@ -266,9 +153,6 @@ | ||
| 266 | 153 | User-agent: peer39_crawler/1.0 |
| 267 | 154 | Disallow: / |
| 268 | 155 | |
| 269 | - User-agent: PerplexityBot | |
| 270 | - Disallow: / | |
| 271 | - | |
| 272 | 156 | User-agent: Quora-Bot |
| 273 | 157 | Disallow: / |
| 274 | 158 | |
| @@ -276,6 +160,9 @@ | ||
| 276 | 160 | Disallow: / |
| 277 | 161 | |
| 278 | 162 | User-agent: TurnitinBot |
| 163 | + Disallow: / | |
| 164 | + | |
| 165 | + User-agent: YouBot | |
| 279 | 166 | Disallow: / |
| 280 | 167 | |
| 281 | 168 | # |