NeuralCrawl

BuzzFeed / robots.txt snapshot

← back to buzzfeed.com · fetched 2026-06-25T22:38:12Z (6h ago) · HTTP 200 · 4496 bytes · sha256 805024daa500ad35 · raw

final URL: https://www.buzzfeed.com/robots.txt

1User-agent: msnbot
2Crawl-delay: 120
3Disallow: /*.xml$
4Disallow: /buzz/*.xml$
5Disallow: /category/*.xml$
6Disallow: /mobile/
7Disallow: *?s=mobile
8Disallow: *?s=bpage-next
9Disallow: *?s=lightbox
10Disallow: /fabordrab/
11Disallow: /bfmp/
12Disallow: /buzzfeed/
13Disallow: /contest
14Disallow: /contests
15Disallow: /plugin/
16Disallow: /embed/
17Disallow: /_comments/
18Disallow: /bookmarks/
19Disallow: /drafts/
20
21User-agent: *
22Disallow: /buzz/*.xml$
23Disallow: /category/*.xml$
24Disallow: /mobile/
25Disallow: *?s=bpage-next
26Disallow: *?s=lightbox
27Disallow: *?s=feedpager
28Disallow: /fabordrab/
29Disallow: /bfmp/
30Disallow: /buzzfeed/
31Disallow: /contest
32Disallow: /contests
33Disallow: /_ga/
34Disallow: /static/
35Disallow: /dashboard/
36Disallow: /plugin/
37Disallow: /api/
38Disallow: /buzzfeed/api/
39Disallow: /embed/
40Disallow: /_comments/
41Disallow: /bookmarks/
42Disallow: /drafts/
43Disallow: /bpage-preview/
44Disallow: /cms_preview_ui/
45Disallow: /cms_preview_ui_static/
46Disallow: /search/
47
48User-agent: Amazonbot
49Allow: /rss-feeds/feeds/bf-amazon-ai-grounding.extended.xml
50Disallow: /
51
52User-agent: anthropic-ai
53Disallow: /
54
55User-agent: Applebot-Extended
56Disallow: /
57
58User-agent: CCBot
59Disallow: /
60
61User-agent: ClaudeBot
62Disallow: /
63
64User-agent: Claude-Web
65Disallow: /
66
67User-agent: cohere-ai
68Disallow: /
69
70User-agent: Diffbot
71Disallow: /
72
73User-agent: discobot
74Disallow: /
75
76User-agent: dotbot
77Disallow: /
78
79User-agent: FacebookBot
80Disallow: /
81
82User-agent: google-extended
83Disallow: /
84
85User-agent: Google-Extended
86Disallow: /
87
88User-agent: Googlebot
89Disallow: /comments-api/
90Disallow: /content-reactions-api/
91
92User-agent: GPTBot
93Disallow: /
94
95User-agent: PerplexityBot
96Disallow: /
97
98User-agent: TurnitinBot
99Disallow: /
100
101User-agent: magpie-crawler
102Disallow: /
103
104User-agent: meta-externalagent
105Disallow: /
106
107User-agent: Meta-ExternalFetcher
108Disallow: /
109
110User-agent: Bytespider
111Disallow: /
112
113User-agent: DuckAssistBot
114Disallow: /
115
116User-agent: Perplexity-User
117Disallow: /
118
119User-agent: Timpibot
120Disallow: /
121
122User-agent: Ai2Bot-Dolma
123Disallow: /
124
125User-agent: omgilibot
126Disallow: /
127
128User-agent: TtoYouBot
129Disallow: /
130
131User-agent: WebGPTBot
132Disallow: /
133
134User-agent: Slurp
135Crawl-delay: 4
136
137User-agent: Verity/1.1
138Crawl-delay: 20
139Allow: /$
140Allow: /*
141Disallow: /*.xml$
142Disallow: /buzz/*.xml$
143Disallow: /category/*.xml$
144Disallow: /mobile/
145Disallow: *?s=mobile
146Disallow: *?s=bpage-next
147Disallow: *?s=lightbox
148Disallow: *?s=feedpager
149Disallow: /fabordrab/
150Disallow: /bfmp/
151Disallow: /buzzfeed/
152Disallow: /contest
153Disallow: /contests
154Disallow: /_ga/
155Disallow: /static/
156Disallow: /dashboard/
157Disallow: /plugin/
158Disallow: /api/
159Disallow: /buzzfeed/api/
160Disallow: /embed/
161Disallow: /_comments/
162Disallow: /bookmarks/
163Disallow: /drafts/
164Disallow: /bpage-preview/
165Disallow: /cms_preview_ui/
166Disallow: /cms_preview_ui_static/
167Disallow: /search/
168
169User-agent: Scope3/2.0 (scope3.com)
170Allow: /$
171Allow: /*
172Disallow: /*.xml$
173Disallow: /buzz/*.xml$
174Disallow: /category/*.xml$
175Disallow: /mobile/
176Disallow: *?s=mobile
177Disallow: *?s=bpage-next
178Disallow: *?s=lightbox
179Disallow: *?s=feedpager
180Disallow: /fabordrab/
181Disallow: /bfmp/
182Disallow: /buzzfeed/
183Disallow: /contest
184Disallow: /contests
185Disallow: /_ga/
186Disallow: /static/
187Disallow: /dashboard/
188Disallow: /plugin/
189Disallow: /api/
190Disallow: /buzzfeed/api/
191Disallow: /embed/
192Disallow: /_comments/
193Disallow: /bookmarks/
194Disallow: /drafts/
195Disallow: /bpage-preview/
196Disallow: /cms_preview_ui/
197Disallow: /cms_preview_ui_static/
198Disallow: /search/
199
200User-agent: AmazonAdBot
201Allow: /$
202Allow: /*
203Disallow: /*.xml$
204Disallow: /buzz/*.xml$
205Disallow: /category/*.xml$
206Disallow: /mobile/
207Disallow: *?s=bpage-next
208Disallow: *?s=lightbox
209Disallow: *?s=feedpager
210Disallow: /fabordrab/
211Disallow: /bfmp/
212Disallow: /buzzfeed/
213Disallow: /contest
214Disallow: /contests
215Disallow: /_ga/
216Disallow: /static/
217Disallow: /dashboard/
218Disallow: /plugin/
219Disallow: /api/
220Disallow: /buzzfeed/api/
221Disallow: /embed/
222Disallow: /_comments/
223Disallow: /bookmarks/
224Disallow: /drafts/
225Disallow: /bpage-preview/
226Disallow: /cms_preview_ui/
227Disallow: /cms_preview_ui_static/
228Disallow: /search/
229
230Sitemap: https://www.buzzfeed.com/sitemap/buzzfeed.xml
231Sitemap: https://www.buzzfeed.com/sitemap/tasty.xml
232Sitemap: https://www.buzzfeed.com/sitemap/video.xml
233Sitemap: https://www.buzzfeed.com/sitemap/shopping.xml
234Sitemap: https://www.buzzfeed.com/sitemap/news.xml
235Sitemap: https://www.buzzfeed.com/sitemap/buzzfeed-community.xml
236Sitemap: https://www.buzzfeed.com/sitemap/editions.xml
237Sitemap: https://www.buzzfeed.com/sitemap/news-archive.xml
238
239User-agent: yacybot
240Disallow: /