NeuralCrawl

Msn / robots.txt snapshot

← back to msn.com · fetched 2026-06-26T13:23:15Z (3h ago) · HTTP 200 · 5005 bytes · sha256 a156370734723e1d · raw

final URL: https://www.msn.com/robots.txt

1User-agent: *
2Disallow: /*/health/search/filter
3Disallow: /spartan
4Disallow: /pt-ao
5Disallow: /*preview=
6Disallow: /*/autos/marketplace/product/
7Disallow: /*/cars/marketplace/product/
8Disallow: /*?item=
9Disallow: /*&item=
10Disallow: /codeCoverage
11Disallow: /vpBaselines
12Disallow: /msnBaselines
13Disallow: /*/studio
14Disallow: /*/indexnowdelete/
15Allow: /*/shopping/*/ar-
16Allow: /*/shopping/*/ss-
17Allow: /*/shopping/*/vi-
18Disallow: /*/shopping/
19
20User-agent: AdsBot-Google
21Allow: /
22Disallow: /*/health/search/filter
23
24User-agent: Verity
25Crawl-delay: 3
26Disallow: /*/health/search/filter
27Disallow: /spartan
28Disallow: /pt-ao
29Disallow: /*preview=
30Disallow: /*/autos/marketplace/product/
31Disallow: /*/cars/marketplace/product/
32Disallow: /*?item=
33Disallow: /*&item=
34Disallow: /codeCoverage
35Disallow: /vpBaselines
36Disallow: /msnBaselines
37Disallow: /*/studio
38Disallow: /*/indexnowdelete/
39Allow: /*/shopping/*/ar-
40Allow: /*/shopping/*/ss-
41Allow: /*/shopping/*/vi-
42Disallow: /*/shopping/
43
44User-agent: ias_crawler
45Crawl-delay: 0.1
46Disallow: /*/health/search/filter
47Disallow: /spartan
48Disallow: /pt-ao
49Disallow: /*preview=
50Disallow: /*/autos/marketplace/product/
51Disallow: /*/cars/marketplace/product/
52Disallow: /*?item=
53Disallow: /*&item=
54Disallow: /codeCoverage
55Disallow: /vpBaselines
56Disallow: /msnBaselines
57Disallow: /*/studio
58Disallow: /*/indexnowdelete/
59Allow: /*/shopping/*/ar-
60Allow: /*/shopping/*/ss-
61Allow: /*/shopping/*/vi-
62Disallow: /*/shopping/
63
64User-agent: Mediapartners-Google
65Crawl-delay: 0.5
66Disallow: /*/health/search/filter
67Disallow: /spartan
68Disallow: /pt-ao
69Disallow: /*preview=
70Disallow: /*/autos/marketplace/product/
71Disallow: /*/cars/marketplace/product/
72Disallow: /*?item=
73Disallow: /*&item=
74Disallow: /codeCoverage
75Disallow: /vpBaselines
76Disallow: /msnBaselines
77Disallow: /*/studio
78Disallow: /*/indexnowdelete/
79Allow: /*/shopping/*/ar-
80Allow: /*/shopping/*/ss-
81Allow: /*/shopping/*/vi-
82Disallow: /*/shopping/
83
84User-agent: CriteoBot
85Crawl-delay: 0.5
86Disallow: /*/health/search/filter
87Disallow: /spartan
88Disallow: /pt-ao
89Disallow: /*preview=
90Disallow: /*/autos/marketplace/product/
91Disallow: /*/cars/marketplace/product/
92Disallow: /*?item=
93Disallow: /*&item=
94Disallow: /codeCoverage
95Disallow: /vpBaselines
96Disallow: /msnBaselines
97Disallow: /*/studio
98Disallow: /*/indexnowdelete/
99Allow: /*/shopping/*/ar-
100Allow: /*/shopping/*/ss-
101Allow: /*/shopping/*/vi-
102Disallow: /*/shopping/
103
104User-agent: yandex.com/bot
105Crawl-delay: 1
106Disallow: /*/weather/
107Disallow: /*/clima/
108Disallow: /*/cuaca/
109Disallow: /*/eltiempo/
110Disallow: /*/el-tiempo/
111Disallow: /*/havadurumu/
112Disallow: /*/idojaras/
113Disallow: /*/meteo/
114Disallow: /*/meteorologia/
115Disallow: /*/pocasi/
116Disallow: /*/pogoda/
117Disallow: /*/saa/
118Disallow: /*/vader/
119Disallow: /*/vejr/
120Disallow: /*/weer/
121Disallow: /*/wetter/
122Allow: /*/shopping/*/ar-
123Allow: /*/shopping/*/ss-
124Allow: /*/shopping/*/vi-
125Disallow: /*/shopping/
126
127User-agent: admantx-ussy04/3.2
128Allow: /
129
130User-agent: TTD-Content
131Allow: /
132
133User-agent: AI2Bot
134User-agent: Ai2Bot-Dolma
135User-agent: Amazonbot
136User-agent: anthropic-ai
137User-agent: Applebot
138User-agent: Applebot-Extended
139User-agent: Bedrockbot
140User-agent: Bytespider
141User-agent: CCBot
142User-agent: ChatGLM
143User-agent: ChatGPT-User
144User-agent: ClaudeBot
145User-agent: Claude-User
146User-agent: Claude-SearchBot
147User-agent: Claude-Web
148User-agent: cohere-ai
149User-agent: cohere-training-data-crawler
150User-agent: Cotoyogi
151User-agent: DeepSeekBot
152User-agent: Diffbot
153User-agent: DuckAssistBot
154User-agent: FacebookBot
155User-agent: FriendlyCrawler
156User-agent: Gensparkbot
157User-agent: Google-CloudVertexBot
158User-agent: Google-Extended
159User-agent: GoogleOther
160User-agent: GoogleOther-Image
161User-agent: GoogleOther-Video
162User-agent: GPTBot
163User-agent: Grok
164User-agent: iaskspider/2.0
165User-agent: ICC-Crawler
166User-agent: ImagesiftBot
167User-agent: Img2dataset
168User-agent: ISSCyberRiskCrawler
169User-agent: Kangaroo Bot
170User-agent: KunatoCrawler
171User-agent: magpie-crawler
172User-agent: Meltwater
173User-agent: Meta-ExternalAgent
174User-agent: Meta-ExternalFetcher
175User-agent: MistralAI
176User-agent: MistralAI-User
177User-agent: Momentumbot
178User-agent: OAI-SearchBot
179User-agent: omgili
180User-agent: Omgilibot
181User-agent: Other
182User-agent: PanguBot
183User-agent: Perplexity-ai
184User-agent: PerplexityBot
185User-agent: Perplexity-User
186User-agent: QualifiedBot
187User-agent: Scrapy
188User-agent: Seekr
189User-agent: Sidetrade indexer bot
190User-agent: TikTokSpider
191User-agent: Timpibot
192User-agent: Twitterbot
193User-agent: VelenPublicWebCrawler
194User-agent: WARDBot
195User-agent: Webzio-Extended
196User-agent: WRTNBot
197User-agent: YouBot
198Disallow: /
199
200Sitemap: https://www.msn.com/sitemaps/casualgames/sitemap-index.xml
201Sitemap: https://www.msn.com/sitemaps/consumption/index.xml
202Sitemap: https://www.msn.com/sitemaps/health/health-sitemap-index.xml
203Sitemap: https://www.msn.com/staticsb/statics/latest/0/casualgames/sitemaps/sitemap-index.xml
204Sitemap: https://www.msn.com/staticsb/statics/latest/0/finance/sitemaps/sitemap-index.xml
205Sitemap: https://www.msn.com/sitemaps/weather/sitemap-index.xml