NeuralCrawl

BuzzFeed / robots.txt snapshot

← back to buzzfeed.com · fetched 2026-06-20T01:10:30Z (17h ago) · HTTP 200 · 4436 bytes · sha256 e934c29ddd8e817e · raw

final URL: https://www.buzzfeed.com/robots.txt

1User-agent: msnbot
2Crawl-delay: 120
3Disallow: /*.xml$
4Disallow: /buzz/*.xml$
5Disallow: /category/*.xml$
6Disallow: /mobile/
7Disallow: *?s=mobile
8Disallow: *?s=bpage-next
9Disallow: *?s=lightbox
10Disallow: /fabordrab/
11Disallow: /bfmp/
12Disallow: /buzzfeed/
13Disallow: /contest
14Disallow: /contests
15Disallow: /plugin/
16Disallow: /embed/
17Disallow: /_comments/
18Disallow: /bookmarks/
19Disallow: /drafts/
20
21User-agent: *
22Disallow: /buzz/*.xml$
23Disallow: /category/*.xml$
24Disallow: /mobile/
25Disallow: *?s=bpage-next
26Disallow: *?s=lightbox
27Disallow: *?s=feedpager
28Disallow: /fabordrab/
29Disallow: /bfmp/
30Disallow: /buzzfeed/
31Disallow: /contest
32Disallow: /contests
33Disallow: /_ga/
34Disallow: /static/
35Disallow: /dashboard/
36Disallow: /plugin/
37Disallow: /api/
38Disallow: /buzzfeed/api/
39Disallow: /embed/
40Disallow: /_comments/
41Disallow: /bookmarks/
42Disallow: /drafts/
43Disallow: /bpage-preview/
44Disallow: /cms_preview_ui/
45Disallow: /cms_preview_ui_static/
46Disallow: /search/
47
48User-agent: Amazonbot
49Disallow: /
50
51User-agent: anthropic-ai
52Disallow: /
53
54User-agent: Applebot-Extended
55Disallow: /
56
57User-agent: CCBot
58Disallow: /
59
60User-agent: ClaudeBot
61Disallow: /
62
63User-agent: Claude-Web
64Disallow: /
65
66User-agent: cohere-ai
67Disallow: /
68
69User-agent: Diffbot
70Disallow: /
71
72User-agent: discobot
73Disallow: /
74
75User-agent: dotbot
76Disallow: /
77
78User-agent: FacebookBot
79Disallow: /
80
81User-agent: google-extended
82Disallow: /
83
84User-agent: Google-Extended
85Disallow: /
86
87User-agent: Googlebot
88Disallow: /comments-api/
89Disallow: /content-reactions-api/
90
91User-agent: GPTBot
92Disallow: /
93
94User-agent: PerplexityBot
95Disallow: /
96
97User-agent: TurnitinBot
98Disallow: /
99
100User-agent: magpie-crawler
101Disallow: /
102
103User-agent: meta-externalagent
104Disallow: /
105
106User-agent: Meta-ExternalFetcher
107Disallow: /
108
109User-agent: Bytespider
110Disallow: /
111
112User-agent: DuckAssistBot
113Disallow: /
114
115User-agent: Perplexity-User
116Disallow: /
117
118User-agent: Timpibot
119Disallow: /
120
121User-agent: Ai2Bot-Dolma
122Disallow: /
123
124User-agent: omgilibot
125Disallow: /
126
127User-agent: TtoYouBot
128Disallow: /
129
130User-agent: WebGPTBot
131Disallow: /
132
133User-agent: Slurp
134Crawl-delay: 4
135
136User-agent: Verity/1.1
137Crawl-delay: 20
138Allow: /$
139Allow: /*
140Disallow: /*.xml$
141Disallow: /buzz/*.xml$
142Disallow: /category/*.xml$
143Disallow: /mobile/
144Disallow: *?s=mobile
145Disallow: *?s=bpage-next
146Disallow: *?s=lightbox
147Disallow: *?s=feedpager
148Disallow: /fabordrab/
149Disallow: /bfmp/
150Disallow: /buzzfeed/
151Disallow: /contest
152Disallow: /contests
153Disallow: /_ga/
154Disallow: /static/
155Disallow: /dashboard/
156Disallow: /plugin/
157Disallow: /api/
158Disallow: /buzzfeed/api/
159Disallow: /embed/
160Disallow: /_comments/
161Disallow: /bookmarks/
162Disallow: /drafts/
163Disallow: /bpage-preview/
164Disallow: /cms_preview_ui/
165Disallow: /cms_preview_ui_static/
166Disallow: /search/
167
168User-agent: Scope3/2.0 (scope3.com)
169Allow: /$
170Allow: /*
171Disallow: /*.xml$
172Disallow: /buzz/*.xml$
173Disallow: /category/*.xml$
174Disallow: /mobile/
175Disallow: *?s=mobile
176Disallow: *?s=bpage-next
177Disallow: *?s=lightbox
178Disallow: *?s=feedpager
179Disallow: /fabordrab/
180Disallow: /bfmp/
181Disallow: /buzzfeed/
182Disallow: /contest
183Disallow: /contests
184Disallow: /_ga/
185Disallow: /static/
186Disallow: /dashboard/
187Disallow: /plugin/
188Disallow: /api/
189Disallow: /buzzfeed/api/
190Disallow: /embed/
191Disallow: /_comments/
192Disallow: /bookmarks/
193Disallow: /drafts/
194Disallow: /bpage-preview/
195Disallow: /cms_preview_ui/
196Disallow: /cms_preview_ui_static/
197Disallow: /search/
198
199User-agent: AmazonAdBot
200Allow: /$
201Allow: /*
202Disallow: /*.xml$
203Disallow: /buzz/*.xml$
204Disallow: /category/*.xml$
205Disallow: /mobile/
206Disallow: *?s=bpage-next
207Disallow: *?s=lightbox
208Disallow: *?s=feedpager
209Disallow: /fabordrab/
210Disallow: /bfmp/
211Disallow: /buzzfeed/
212Disallow: /contest
213Disallow: /contests
214Disallow: /_ga/
215Disallow: /static/
216Disallow: /dashboard/
217Disallow: /plugin/
218Disallow: /api/
219Disallow: /buzzfeed/api/
220Disallow: /embed/
221Disallow: /_comments/
222Disallow: /bookmarks/
223Disallow: /drafts/
224Disallow: /bpage-preview/
225Disallow: /cms_preview_ui/
226Disallow: /cms_preview_ui_static/
227Disallow: /search/
228
229Sitemap: https://www.buzzfeed.com/sitemap/buzzfeed.xml
230Sitemap: https://www.buzzfeed.com/sitemap/tasty.xml
231Sitemap: https://www.buzzfeed.com/sitemap/video.xml
232Sitemap: https://www.buzzfeed.com/sitemap/shopping.xml
233Sitemap: https://www.buzzfeed.com/sitemap/news.xml
234Sitemap: https://www.buzzfeed.com/sitemap/buzzfeed-community.xml
235Sitemap: https://www.buzzfeed.com/sitemap/editions.xml
236Sitemap: https://www.buzzfeed.com/sitemap/news-archive.xml
237
238User-agent: yacybot
239Disallow: /