NeuralCrawl

Msn / robots.txt snapshot

← back to msn.com · fetched 2026-07-29T22:40:57Z (7d ago) · HTTP 200 · 5039 bytes · sha256 1045c7782df502d2 · raw

final URL: https://www.msn.com/robots.txt

1User-agent: *
2Disallow: /*/health/search/filter
3Disallow: /spartan
4Disallow: /pt-ao
5Disallow: /*preview=
6Disallow: /*/autos/marketplace/product/
7Disallow: /*/cars/marketplace/product/
8Disallow: /*?item=
9Disallow: /*&item=
10Disallow: /codeCoverage
11Disallow: /vpBaselines
12Disallow: /msnBaselines
13Disallow: /*/studio
14Disallow: /*/indexnowdelete/
15Allow: /*/shopping/*/ar-
16Allow: /*/shopping/*/ss-
17Allow: /*/shopping/*/vi-
18Disallow: /*/shopping/
19
20User-agent: AdsBot-Google
21Allow: /
22Disallow: /*/health/search/filter
23
24User-agent: Verity
25Crawl-delay: 3
26Disallow: /*/health/search/filter
27Disallow: /spartan
28Disallow: /pt-ao
29Disallow: /*preview=
30Disallow: /*/autos/marketplace/product/
31Disallow: /*/cars/marketplace/product/
32Disallow: /*?item=
33Disallow: /*&item=
34Disallow: /codeCoverage
35Disallow: /vpBaselines
36Disallow: /msnBaselines
37Disallow: /*/studio
38Disallow: /*/indexnowdelete/
39Allow: /*/shopping/*/ar-
40Allow: /*/shopping/*/ss-
41Allow: /*/shopping/*/vi-
42Disallow: /*/shopping/
43
44User-agent: ias_crawler
45Crawl-delay: 0.1
46Disallow: /*/health/search/filter
47Disallow: /spartan
48Disallow: /pt-ao
49Disallow: /*preview=
50Disallow: /*/autos/marketplace/product/
51Disallow: /*/cars/marketplace/product/
52Disallow: /*?item=
53Disallow: /*&item=
54Disallow: /codeCoverage
55Disallow: /vpBaselines
56Disallow: /msnBaselines
57Disallow: /*/studio
58Disallow: /*/indexnowdelete/
59Allow: /*/shopping/*/ar-
60Allow: /*/shopping/*/ss-
61Allow: /*/shopping/*/vi-
62Disallow: /*/shopping/
63
64User-agent: Mediapartners-Google
65Crawl-delay: 0.5
66Disallow: /*/health/search/filter
67Disallow: /spartan
68Disallow: /pt-ao
69Disallow: /*preview=
70Disallow: /*/autos/marketplace/product/
71Disallow: /*/cars/marketplace/product/
72Disallow: /*?item=
73Disallow: /*&item=
74Disallow: /codeCoverage
75Disallow: /vpBaselines
76Disallow: /msnBaselines
77Disallow: /*/studio
78Disallow: /*/indexnowdelete/
79Allow: /*/shopping/*/ar-
80Allow: /*/shopping/*/ss-
81Allow: /*/shopping/*/vi-
82Disallow: /*/shopping/
83
84User-agent: CriteoBot
85Crawl-delay: 0.5
86Disallow: /*/health/search/filter
87Disallow: /spartan
88Disallow: /pt-ao
89Disallow: /*preview=
90Disallow: /*/autos/marketplace/product/
91Disallow: /*/cars/marketplace/product/
92Disallow: /*?item=
93Disallow: /*&item=
94Disallow: /codeCoverage
95Disallow: /vpBaselines
96Disallow: /msnBaselines
97Disallow: /*/studio
98Disallow: /*/indexnowdelete/
99Allow: /*/shopping/*/ar-
100Allow: /*/shopping/*/ss-
101Allow: /*/shopping/*/vi-
102Disallow: /*/shopping/
103
104User-agent: yandex.com/bot
105Crawl-delay: 1
106Disallow: /*/weather/
107Disallow: /*/clima/
108Disallow: /*/cuaca/
109Disallow: /*/eltiempo/
110Disallow: /*/el-tiempo/
111Disallow: /*/havadurumu/
112Disallow: /*/idojaras/
113Disallow: /*/meteo/
114Disallow: /*/meteorologia/
115Disallow: /*/pocasi/
116Disallow: /*/pogoda/
117Disallow: /*/saa/
118Disallow: /*/vader/
119Disallow: /*/vejr/
120Disallow: /*/weer/
121Disallow: /*/wetter/
122Allow: /*/shopping/*/ar-
123Allow: /*/shopping/*/ss-
124Allow: /*/shopping/*/vi-
125Disallow: /*/shopping/
126
127User-agent: admantx-ussy04/3.2
128Allow: /
129
130User-agent: TTD-Content
131Allow: /
132
133User-agent: AmazonAdBot
134Allow: /
135
136User-agent: AI2Bot
137User-agent: Ai2Bot-Dolma
138User-agent: Amazonbot
139User-agent: anthropic-ai
140User-agent: Applebot
141User-agent: Applebot-Extended
142User-agent: Bedrockbot
143User-agent: Bytespider
144User-agent: CCBot
145User-agent: ChatGLM
146User-agent: ChatGPT-User
147User-agent: ClaudeBot
148User-agent: Claude-User
149User-agent: Claude-SearchBot
150User-agent: Claude-Web
151User-agent: cohere-ai
152User-agent: cohere-training-data-crawler
153User-agent: Cotoyogi
154User-agent: DeepSeekBot
155User-agent: Diffbot
156User-agent: DuckAssistBot
157User-agent: FacebookBot
158User-agent: FriendlyCrawler
159User-agent: Gensparkbot
160User-agent: Google-CloudVertexBot
161User-agent: Google-Extended
162User-agent: GoogleOther
163User-agent: GoogleOther-Image
164User-agent: GoogleOther-Video
165User-agent: GPTBot
166User-agent: Grok
167User-agent: iaskspider/2.0
168User-agent: ICC-Crawler
169User-agent: ImagesiftBot
170User-agent: Img2dataset
171User-agent: ISSCyberRiskCrawler
172User-agent: Kangaroo Bot
173User-agent: KunatoCrawler
174User-agent: magpie-crawler
175User-agent: Meltwater
176User-agent: Meta-ExternalAgent
177User-agent: Meta-ExternalFetcher
178User-agent: MistralAI
179User-agent: MistralAI-User
180User-agent: Momentumbot
181User-agent: OAI-SearchBot
182User-agent: omgili
183User-agent: Omgilibot
184User-agent: Other
185User-agent: PanguBot
186User-agent: Perplexity-ai
187User-agent: PerplexityBot
188User-agent: Perplexity-User
189User-agent: QualifiedBot
190User-agent: Scrapy
191User-agent: Seekr
192User-agent: Sidetrade indexer bot
193User-agent: TikTokSpider
194User-agent: Timpibot
195User-agent: Twitterbot
196User-agent: VelenPublicWebCrawler
197User-agent: WARDBot
198User-agent: Webzio-Extended
199User-agent: WRTNBot
200User-agent: YouBot
201Disallow: /
202
203Sitemap: https://www.msn.com/sitemaps/casualgames/sitemap-index.xml
204Sitemap: https://www.msn.com/sitemaps/consumption/index.xml
205Sitemap: https://www.msn.com/sitemaps/health/health-sitemap-index.xml
206Sitemap: https://www.msn.com/staticsb/statics/latest/0/casualgames/sitemaps/sitemap-index.xml
207Sitemap: https://www.msn.com/staticsb/statics/latest/0/finance/sitemaps/sitemap-index.xml
208Sitemap: https://www.msn.com/sitemaps/weather/sitemap-index.xml