NeuralCrawl

The Washington Post / robots.txt snapshot

← back to washingtonpost.com · fetched 2026-08-10T20:55:49Z (2mo ago) · HTTP 200 · 5804 bytes · sha256 918970c4ae59aa1f · raw

final URL: https://www.washingtonpost.com/robots.txt

1User-agent: *
2Disallow: /*_print.html
3Disallow: /*_email.html
4Disallow: /*_singlePage.html
5Disallow: /*_allComments.html
6Disallow: /*_jsn.json
7Disallow: /*_jsonpStatic.js
8Disallow: /*_nitf.xml
9Disallow: /*_newsml.html
10Disallow: /*_qa.html
11Disallow: /*_meta.xml
12Disallow: /*_jsnp.js
13Disallow: /*_json.json
14Disallow: /*_search.html
15Disallow: /*_jsonp.js
16Disallow: /*_jsnpStatic.js
17Disallow: /*_rss.xml
18Disallow: /*_mobile.mobile
19Disallow: /*_mobile.xml
20Disallow: /*_allCommentsClassicBlog.html
21Disallow: /*_seo.html
22Disallow: /*_nimbusJson.json
23Disallow: /*_nimbusJsonp.js
24Disallow: /*_nimbusJsonpStatic.js
25Disallow: /*_modal.html
26Disallow: /todays_paper/
27Disallow: /rw/WashingtonPost/Content/Epaper/
28Disallow: /ac2/
29Disallow: /blogs/slow-ride/
30Disallow: /local/blogsandcolumns/slow-ride-story-tanked
31Disallow: /local/blogsandcolumns/slow-ride-story-achenblog
32Disallow: /local/blogsandcolumns/slow-ride-stream-tanked
33Disallow: /local/blogsandcolumns/slow-ride-front
34Disallow: /utils/
35Disallow: /jobs/JS_JobSearchResult
36Disallow: /jobs/UpdateJobEmployerCounterServlet
37Disallow: /jobs/JS_Login
38Disallow: /jobs/EU_UpdateJobEmployerCounter
39Disallow: /blogs/nationals-journal-beta/
40Disallow: /blogs/test/
41Disallow: /posttv-beta/
42Disallow: /posttv/sponsored-video/
43Disallow: /posttv/c/trendex/
44Disallow: /posttv/c/video_search/
45Disallow: /posttv/posttv/trendex
46Disallow: /posttv/c/embed/
47Disallow: /rweb/
48Disallow: /wp-stat/vrroom/
49Disallow: /classic-apps/
50Disallow: /news/test/
51Disallow: /tablet/
52Disallow: /news/tablet/
53Disallow: /sf/test/
54Disallow: /news/test-liveblog/
55Disallow: /pb/
56Allow: /pb/resources/
57Allow: /pb/gr/
58Allow: /pb/resource/
59Disallow: /homepage-video-test
60Disallow: /testpage-forhomepage
61Disallow: /knowmore
62Disallow: /test/
63Disallow: /sslsingle
64Disallow: /amphtml/news/test/
65Disallow: /amphtml/blogs/test/
66Disallow: /amphtml/classic-apps/
67Disallow: /amphtml/utils/
68Disallow: /newsletter/
69Disallow: /wp-dyn/
70Disallow: /wp-srv/
71Disallow: /bandito/
72Disallow: /Fragment/SysConfig/
73Disallow: /recipes/search/
74Disallow: /talk/
75Disallow: /wp-stat/ad/
76Disallow: /*?*outputType=comment
77Disallow: /pwapi-proxy/pwproxy/*
78Disallow: /pwapiv2/
79Disallow: /*?*outputType=accessibility
80Disallow: /wp-adv/
81Disallow: /newssearch/
82Disallow: /wp-admin/
83Disallow: /gdpr-consent/
84Disallow: /*?*outputType=tracking
85Disallow: /tetro/
86Disallow: /comments/
87Disallow: /comments
88Disallow: /search
89Disallow: /s/*
90Disallow: /embed/
91Disallow: /native/
92Disallow: /subscribe/braintree/
93Disallow: /subscribe/enterpriseportal/
94Disallow: /subscribe/foryouapi/
95Disallow: /subscribe/lagoon/
96Disallow: /subscribe/offers/service/
97Disallow: /subscribe/onsiteapi/
98Disallow: /subscribe/paywall/
99Disallow: /subscribe/person/
100Disallow: /subscribe/preferenceapi/
101Disallow: /subscribe/subscriptionapi/
102Disallow: /subscribe/user/
103Disallow: /subscribe/signin/
104Disallow: /subscribe/signup/
105Disallow: /wpost/proxy
106Disallow: /ehf/
107Disallow: /ehf/*
108Disallow: /subscribe/logging/*
109Disallow: /blogs/*
110Disallow: /gog/*
111Disallow: /arcio/fact-checker/
112Disallow: /recipes/*?q=
113Disallow: /cookie-policy/
114Disallow: /discussions/2021/09/13/cookie-policy/
115Disallow: /discussions/2021/09/13/cookie-policy-eea/
116Disallow: /privacy-policy/
117Disallow: /partials/
118
119User-agent: Twitterbot
120Allow: /posttv-beta/
121Disallow: /amphtml/*
122
123User-agent: SemrushBot
124Disallow: /licensing-syndication
125Disallow: /licensing-syndication/*
126
127User-agent: Amazonbot
128Disallow: /
129Allow: /wp-apps/imrs.php
130
131
132# Disallow Rules
133
134User-agent: AhrefsBot
135Disallow: /
136
137User-agent: anthropic-ai
138Disallow: /
139Allow: /creativegroup/
140Allow: /advertising/
141
142User-agent: Applebot-Extended
143Disallow: /
144Allow: /creativegroup/
145Allow: /advertising/
146
147User-agent: archive.org_bot
148Disallow: /
149
150User-agent: ArchiveBot
151Disallow: /
152
153User-agent: Arquivo-web-crawler
154Disallow: /
155
156User-agent: AwarioRssBot
157Disallow: /
158
159User-agent: AwarioSmartBot
160Disallow: /
161
162User-agent: Bytespider
163Disallow: /
164Allow: /creativegroup/
165Allow: /advertising/
166
167User-agent: CCBot
168Disallow: /
169Allow: /creativegroup/
170Allow: /advertising/
171
172User-agent: Claude-Web
173Disallow: /
174Allow: /creativegroup/
175Allow: /advertising/
176
177User-agent: ClaudeBot
178Disallow: /
179Allow: /creativegroup/
180Allow: /advertising/
181
182User-agent: cohere-training-data-crawler
183Disallow: /
184Allow: /creativegroup/
185Allow: /advertising/
186
187User-agent: DataForSeoBot
188Disallow: /
189
190User-agent: Diffbot
191Disallow: /
192Allow: /creativegroup/
193Allow: /advertising/
194
195User-agent: europarchive.org
196Disallow: /
197
198User-agent: FacebookBot
199Disallow: /
200Allow: /creativegroup/
201Allow: /advertising/
202
203User-agent: heritrix
204Disallow: /
205
206User-agent: ia_archiver
207Disallow: /
208
209User-agent: ia_archiver-web.archive.org
210Disallow: /
211
212User-agent: ImagesiftBot
213Disallow: /
214
215User-agent: LinkArchiver
216Disallow: /
217
218User-agent: meta-externalagent
219Disallow: /
220Allow: /creativegroup/
221Allow: /advertising/
222
223User-agent: meta-externalfetcher
224Disallow: /
225Allow: /creativegroup/
226Allow: /advertising/
227
228User-agent: Nicecrawler
229Disallow: /
230
231User-agent: omgili
232Disallow: /
233Allow: /creativegroup/
234Allow: /advertising/
235
236User-agent: omgilibot
237Disallow: /
238Allow: /creativegroup/
239Allow: /advertising/
240
241User-agent: PerplexityBot
242Disallow: /
243Allow: /creativegroup/
244Allow: /advertising/
245
246User-agent: Primalbot
247Disallow: /
248
249User-agent: web-archive-net.com.bot
250Disallow: /
251
252User-agent: YandexBot
253Disallow: /
254Allow: /creativegroup/
255Allow: /advertising/
256
257Sitemap: https://www.washingtonpost.com/sitemaps/sitemap.xml.gz
258Sitemap: https://www.washingtonpost.com/sitemaps/news-sitemap.xml.gz
259Sitemap: https://www.washingtonpost.com/sitemaps/author-sitemap.xml.gz
260Sitemap: https://www.washingtonpost.com/sitemaps/section-sitemap.xml.gz
261Sitemap: https://www.washingtonpost.com/elections/results/sitemap.xml
262Sitemap: https://www.washingtonpost.com/arcio/sitemap/video/index/
263Sitemap: https://www.washingtonpost.com/creativegroup/sitemaps/sitemap.xml