NeuralCrawl

The Washington Post / robots.txt snapshot

← back to washingtonpost.com · fetched 2026-06-20T01:10:30Z (18h ago) · HTTP 200 · 5783 bytes · sha256 66120dc2b705993c · raw

final URL: https://www.washingtonpost.com/robots.txt

1User-agent: *
2Disallow: /*_print.html
3Disallow: /*_email.html
4Disallow: /*_singlePage.html
5Disallow: /*_allComments.html
6Disallow: /*_jsn.json
7Disallow: /*_jsonpStatic.js
8Disallow: /*_nitf.xml
9Disallow: /*_newsml.html
10Disallow: /*_qa.html
11Disallow: /*_meta.xml
12Disallow: /*_jsnp.js
13Disallow: /*_json.json
14Disallow: /*_search.html
15Disallow: /*_jsonp.js
16Disallow: /*_jsnpStatic.js
17Disallow: /*_rss.xml
18Disallow: /*_mobile.mobile
19Disallow: /*_mobile.xml
20Disallow: /*_allCommentsClassicBlog.html
21Disallow: /*_seo.html
22Disallow: /*_nimbusJson.json
23Disallow: /*_nimbusJsonp.js
24Disallow: /*_nimbusJsonpStatic.js
25Disallow: /*_modal.html
26Disallow: /todays_paper/
27Disallow: /rw/WashingtonPost/Content/Epaper/
28Disallow: /ac2/
29Disallow: /blogs/slow-ride/
30Disallow: /local/blogsandcolumns/slow-ride-story-tanked
31Disallow: /local/blogsandcolumns/slow-ride-story-achenblog
32Disallow: /local/blogsandcolumns/slow-ride-stream-tanked
33Disallow: /local/blogsandcolumns/slow-ride-front
34Disallow: /utils/
35Disallow: /jobs/JS_JobSearchResult
36Disallow: /jobs/UpdateJobEmployerCounterServlet
37Disallow: /jobs/JS_Login
38Disallow: /jobs/EU_UpdateJobEmployerCounter
39Disallow: /blogs/nationals-journal-beta/
40Disallow: /blogs/test/
41Disallow: /posttv-beta/
42Disallow: /posttv/sponsored-video/
43Disallow: /posttv/c/trendex/
44Disallow: /posttv/c/video_search/
45Disallow: /posttv/posttv/trendex
46Disallow: /posttv/c/embed/
47Disallow: /rweb/
48Disallow: /wp-stat/vrroom/
49Disallow: /classic-apps/
50Disallow: /news/test/
51Disallow: /tablet/
52Disallow: /news/tablet/
53Disallow: /sf/test/
54Disallow: /news/test-liveblog/
55Disallow: /pb/
56Allow: /pb/resources/
57Allow: /pb/gr/
58Allow: /pb/resource/
59Disallow: /homepage-video-test
60Disallow: /testpage-forhomepage
61Disallow: /knowmore
62Disallow: /test/
63Disallow: /sslsingle
64Disallow: /amphtml/news/test/
65Disallow: /amphtml/blogs/test/
66Disallow: /amphtml/classic-apps/
67Disallow: /amphtml/utils/
68Disallow: /newsletter/
69Disallow: /wp-dyn/
70Disallow: /wp-srv/
71Disallow: /bandito/
72Disallow: /Fragment/SysConfig/
73Disallow: /recipes/search/
74Disallow: /talk/
75Disallow: /wp-stat/ad/
76Disallow: /*?*outputType=comment
77Disallow: /pwapi-proxy/pwproxy/*
78Disallow: /pwapiv2/
79Disallow: /*?*outputType=accessibility
80Disallow: /wp-adv/
81Disallow: /newssearch/
82Disallow: /wp-admin/
83Disallow: /gdpr-consent/
84Disallow: /*?*outputType=tracking
85Disallow: /tetro/
86Disallow: /comments/
87Disallow: /comments
88Disallow: /search
89Disallow: /s/*
90Disallow: /embed/
91Disallow: /native/
92Disallow: /subscribe/braintree/
93Disallow: /subscribe/enterpriseportal/
94Disallow: /subscribe/foryouapi/
95Disallow: /subscribe/lagoon/
96Disallow: /subscribe/offers/service/
97Disallow: /subscribe/onsiteapi/
98Disallow: /subscribe/paywall/
99Disallow: /subscribe/person/
100Disallow: /subscribe/preferenceapi/
101Disallow: /subscribe/subscriptionapi/
102Disallow: /subscribe/user/
103Disallow: /subscribe/signin/
104Disallow: /subscribe/signup/
105Disallow: /wpost/proxy
106Disallow: /ehf/
107Disallow: /ehf/*
108Disallow: /subscribe/logging/*
109Disallow: /blogs/*
110Disallow: /gog/*
111Disallow: /arcio/fact-checker/
112Disallow: /recipes/*?q=
113Disallow: /cookie-policy/
114Disallow: /discussions/2021/09/13/cookie-policy/
115Disallow: /discussions/2021/09/13/cookie-policy-eea/
116Disallow: /privacy-policy/
117
118User-agent: Twitterbot
119Allow: /posttv-beta/
120Disallow: /amphtml/*
121
122User-agent: SemrushBot
123Disallow: /licensing-syndication
124Disallow: /licensing-syndication/*
125
126User-agent: Amazonbot
127Disallow: /
128Allow: /wp-apps/imrs.php
129
130
131# Disallow Rules
132
133User-agent: AhrefsBot
134Disallow: /
135
136User-agent: anthropic-ai
137Disallow: /
138Allow: /creativegroup/
139Allow: /advertising/
140
141User-agent: Applebot-Extended
142Disallow: /
143Allow: /creativegroup/
144Allow: /advertising/
145
146User-agent: archive.org_bot
147Disallow: /
148
149User-agent: ArchiveBot
150Disallow: /
151
152User-agent: Arquivo-web-crawler
153Disallow: /
154
155User-agent: AwarioRssBot
156Disallow: /
157
158User-agent: AwarioSmartBot
159Disallow: /
160
161User-agent: Bytespider
162Disallow: /
163Allow: /creativegroup/
164Allow: /advertising/
165
166User-agent: CCBot
167Disallow: /
168Allow: /creativegroup/
169Allow: /advertising/
170
171User-agent: Claude-Web
172Disallow: /
173Allow: /creativegroup/
174Allow: /advertising/
175
176User-agent: ClaudeBot
177Disallow: /
178Allow: /creativegroup/
179Allow: /advertising/
180
181User-agent: cohere-training-data-crawler
182Disallow: /
183Allow: /creativegroup/
184Allow: /advertising/
185
186User-agent: DataForSeoBot
187Disallow: /
188
189User-agent: Diffbot
190Disallow: /
191Allow: /creativegroup/
192Allow: /advertising/
193
194User-agent: europarchive.org
195Disallow: /
196
197User-agent: FacebookBot
198Disallow: /
199Allow: /creativegroup/
200Allow: /advertising/
201
202User-agent: heritrix
203Disallow: /
204
205User-agent: ia_archiver
206Disallow: /
207
208User-agent: ia_archiver-web.archive.org
209Disallow: /
210
211User-agent: ImagesiftBot
212Disallow: /
213
214User-agent: LinkArchiver
215Disallow: /
216
217User-agent: meta-externalagent
218Disallow: /
219Allow: /creativegroup/
220Allow: /advertising/
221
222User-agent: meta-externalfetcher
223Disallow: /
224Allow: /creativegroup/
225Allow: /advertising/
226
227User-agent: Nicecrawler
228Disallow: /
229
230User-agent: omgili
231Disallow: /
232Allow: /creativegroup/
233Allow: /advertising/
234
235User-agent: omgilibot
236Disallow: /
237Allow: /creativegroup/
238Allow: /advertising/
239
240User-agent: PerplexityBot
241Disallow: /
242Allow: /creativegroup/
243Allow: /advertising/
244
245User-agent: Primalbot
246Disallow: /
247
248User-agent: web-archive-net.com.bot
249Disallow: /
250
251User-agent: YandexBot
252Disallow: /
253Allow: /creativegroup/
254Allow: /advertising/
255
256Sitemap: https://www.washingtonpost.com/sitemaps/sitemap.xml.gz
257Sitemap: https://www.washingtonpost.com/sitemaps/news-sitemap.xml.gz
258Sitemap: https://www.washingtonpost.com/sitemaps/author-sitemap.xml.gz
259Sitemap: https://www.washingtonpost.com/sitemaps/section-sitemap.xml.gz
260Sitemap: https://www.washingtonpost.com/elections/results/sitemap.xml
261Sitemap: https://www.washingtonpost.com/arcio/sitemap/video/index/
262Sitemap: https://www.washingtonpost.com/creativegroup/sitemaps/sitemap.xml