NeuralCrawl

Threads / robots.txt snapshot

← back to threads.net · fetched 2026-06-20T01:10:30Z (17h ago) · HTTP 200 · 7507 bytes · sha256 c7927f1dccd93c9e · raw

final URL: https://www.threads.com/robots.txt

1# Notice: Collection of data on Threads through automated means is
2# prohibited unless you have express written permission from Threads
3# and may only be conducted for the limited purpose contained in said
4# permission.
5# All authorized user-agents listed on this page must comply with Meta’s
6# Automated Data Collection Terms available at:
7# https://www.facebook.com/legal/automated_data_collection_terms
8User-agent: Amazonbot
9Disallow: /
10
11User-agent: Applebot-Extended
12Disallow: /
13
14User-agent: Brightbot
15Disallow: /
16
17User-agent: ClaudeBot
18Disallow: /
19
20User-agent: Google-Extended
21Disallow: /
22
23User-agent: GPTBot
24Disallow: /
25
26User-agent: PerplexityBot
27Disallow: /
28
29User-agent: PetalBot
30Disallow: /
31
32User-agent: Scrapy
33Disallow: /
34
35User-agent: uptimerobot
36Disallow: /
37
38User-agent: viberbot
39Disallow: /
40
41User-agent: YaK
42Disallow: /
43
44User-agent: Yandex
45Disallow: /
46
47User-agent: Yeti
48Disallow: /
49
50User-agent: Applebot
51Disallow: /accounts/
52Disallow: /activity/
53Disallow: /ajax/
54Disallow: /client_error/
55Disallow: /deck/
56Disallow: /intent/
57Disallow: /logging/
58Disallow: /onboarding/
59Disallow: /publicapi/
60Disallow: /qp/batch_fetch_web/
61Disallow: /query/
62Disallow: /settings/
63
64User-agent: Bingbot
65Disallow: /*/c/
66Disallow: /*/comments/
67Disallow: /*/liked_by/
68Disallow: /*/p/
69Disallow: /accounts/
70Disallow: /activity/
71Disallow: /ajax/
72Disallow: /client_error/
73Disallow: /deck/
74Disallow: /intent/
75Disallow: /logging/
76Disallow: /onboarding/
77Disallow: /publicapi/
78Disallow: /qp/batch_fetch_web/
79Disallow: /query/
80Disallow: /settings/
81
82User-agent: Discordbot
83Disallow: /accounts/
84Disallow: /activity/
85Disallow: /ajax/
86Disallow: /client_error/
87Disallow: /deck/
88Disallow: /intent/
89Disallow: /logging/
90Disallow: /onboarding/
91Disallow: /publicapi/
92Disallow: /qp/batch_fetch_web/
93Disallow: /query/
94Disallow: /settings/
95
96User-agent: DuckDuckBot
97Disallow: /accounts/
98Disallow: /activity/
99Disallow: /ajax/
100Disallow: /client_error/
101Disallow: /deck/
102Disallow: /intent/
103Disallow: /logging/
104Disallow: /onboarding/
105Disallow: /publicapi/
106Disallow: /qp/batch_fetch_web/
107Disallow: /query/
108Disallow: /settings/
109
110User-agent: facebookexternalhit
111Disallow: /accounts/
112Disallow: /activity/
113Disallow: /ajax/
114Disallow: /client_error/
115Disallow: /deck/
116Disallow: /intent/
117Disallow: /logging/
118Disallow: /onboarding/
119Disallow: /publicapi/
120Disallow: /qp/batch_fetch_web/
121Disallow: /query/
122Disallow: /settings/
123
124User-agent: Googlebot
125Disallow: /*/c/
126Disallow: /*/comments/
127Disallow: /*/liked_by/
128Disallow: /*/p/
129Disallow: /accounts/
130Disallow: /activity/
131Disallow: /ajax/
132Disallow: /client_error/
133Disallow: /deck/
134Disallow: /intent/
135Disallow: /logging/
136Disallow: /onboarding/
137Disallow: /publicapi/
138Disallow: /qp/batch_fetch_web/
139Disallow: /query/
140Disallow: /settings/
141
142User-agent: Googlebot-Image
143Disallow: /accounts/
144Disallow: /activity/
145Disallow: /ajax/
146Disallow: /client_error/
147Disallow: /deck/
148Disallow: /intent/
149Disallow: /logging/
150Disallow: /onboarding/
151Disallow: /publicapi/
152Disallow: /qp/batch_fetch_web/
153Disallow: /query/
154Disallow: /settings/
155
156User-agent: Googlebot-Video
157Disallow: /accounts/
158Disallow: /activity/
159Disallow: /ajax/
160Disallow: /client_error/
161Disallow: /deck/
162Disallow: /intent/
163Disallow: /logging/
164Disallow: /onboarding/
165Disallow: /publicapi/
166Disallow: /qp/batch_fetch_web/
167Disallow: /query/
168Disallow: /settings/
169
170User-agent: Googlebot-News
171Disallow: /accounts/
172Disallow: /activity/
173Disallow: /ajax/
174Disallow: /client_error/
175Disallow: /deck/
176Disallow: /intent/
177Disallow: /logging/
178Disallow: /onboarding/
179Disallow: /publicapi/
180Disallow: /qp/batch_fetch_web/
181Disallow: /query/
182Disallow: /settings/
183
184User-agent: Google-InspectionTool
185Disallow: /accounts/
186Disallow: /activity/
187Disallow: /ajax/
188Disallow: /client_error/
189Disallow: /deck/
190Disallow: /intent/
191Disallow: /logging/
192Disallow: /onboarding/
193Disallow: /publicapi/
194Disallow: /qp/batch_fetch_web/
195Disallow: /query/
196Disallow: /settings/
197
198User-agent: LinkedInBot
199Disallow: /accounts/
200Disallow: /activity/
201Disallow: /ajax/
202Disallow: /client_error/
203Disallow: /deck/
204Disallow: /intent/
205Disallow: /logging/
206Disallow: /onboarding/
207Disallow: /publicapi/
208Disallow: /qp/batch_fetch_web/
209Disallow: /query/
210Disallow: /settings/
211
212User-agent: msnbot
213Disallow: /accounts/
214Disallow: /activity/
215Disallow: /ajax/
216Disallow: /client_error/
217Disallow: /deck/
218Disallow: /intent/
219Disallow: /logging/
220Disallow: /onboarding/
221Disallow: /publicapi/
222Disallow: /qp/batch_fetch_web/
223Disallow: /query/
224Disallow: /settings/
225
226User-agent: Pinterestbot
227Disallow: /accounts/
228Disallow: /activity/
229Disallow: /ajax/
230Disallow: /client_error/
231Disallow: /deck/
232Disallow: /intent/
233Disallow: /logging/
234Disallow: /onboarding/
235Disallow: /publicapi/
236Disallow: /qp/batch_fetch_web/
237Disallow: /query/
238Disallow: /settings/
239
240User-agent: Screaming Frog SEO Spider
241Disallow: /accounts/
242Disallow: /activity/
243Disallow: /ajax/
244Disallow: /client_error/
245Disallow: /deck/
246Disallow: /intent/
247Disallow: /logging/
248Disallow: /onboarding/
249Disallow: /publicapi/
250Disallow: /qp/batch_fetch_web/
251Disallow: /query/
252Disallow: /settings/
253
254User-agent: seznambot
255Disallow: /accounts/
256Disallow: /activity/
257Disallow: /ajax/
258Disallow: /client_error/
259Disallow: /deck/
260Disallow: /intent/
261Disallow: /logging/
262Disallow: /onboarding/
263Disallow: /publicapi/
264Disallow: /qp/batch_fetch_web/
265Disallow: /query/
266Disallow: /settings/
267
268User-agent: Slurp
269Disallow: /accounts/
270Disallow: /activity/
271Disallow: /ajax/
272Disallow: /client_error/
273Disallow: /deck/
274Disallow: /intent/
275Disallow: /logging/
276Disallow: /onboarding/
277Disallow: /publicapi/
278Disallow: /qp/batch_fetch_web/
279Disallow: /query/
280Disallow: /settings/
281
282User-agent: teoma
283Disallow: /accounts/
284Disallow: /activity/
285Disallow: /ajax/
286Disallow: /client_error/
287Disallow: /deck/
288Disallow: /intent/
289Disallow: /logging/
290Disallow: /onboarding/
291Disallow: /publicapi/
292Disallow: /qp/batch_fetch_web/
293Disallow: /query/
294Disallow: /settings/
295
296User-agent: TelegramBot
297Disallow: /accounts/
298Disallow: /activity/
299Disallow: /ajax/
300Disallow: /client_error/
301Disallow: /deck/
302Disallow: /intent/
303Disallow: /logging/
304Disallow: /onboarding/
305Disallow: /publicapi/
306Disallow: /qp/batch_fetch_web/
307Disallow: /query/
308Disallow: /settings/
309
310User-agent: Twitterbot
311Disallow: /accounts/
312Disallow: /activity/
313Disallow: /ajax/
314Disallow: /client_error/
315Disallow: /deck/
316Disallow: /intent/
317Disallow: /logging/
318Disallow: /onboarding/
319Disallow: /publicapi/
320Disallow: /qp/batch_fetch_web/
321Disallow: /query/
322Disallow: /settings/
323
324User-agent: *
325Disallow: /
326
327Sitemap: https://www.threads.com/sitemap/threads_profiles_sitemap.xml.gz
328Sitemap: https://www.threads.com/sitemap/threads_profiles_sitemap_tier_1.xml.gz
329Sitemap: https://www.threads.com/sitemap/threads_profiles_sitemap_tier_2.xml.gz
330Sitemap: https://www.threads.com/sitemap/threads_profiles_sitemap_tier_3.xml.gz
331Sitemap: https://www.threads.com/sitemap/threads_profiles_sitemap_tier_4.xml.gz
332Sitemap: https://www.threads.com/sitemap/threads_profiles_sitemap_tier_5.xml.gz
333Sitemap: https://www.threads.com/sitemap/us_top_creators_sitemap.xml.gz
334Sitemap: https://www.threads.com/sitemap/us_emerging_creators_sitemap.xml.gz
335Sitemap: https://www.threads.com/sitemap/us_quality_tagged_tail_sitemap.xml.gz
336Sitemap: https://www.threads.com/sitemap/us_active_producers_sitemap.xml.gz
337Sitemap: https://www.threads.com/sitemap/us_micro_producers_sitemap.xml.gz
338Sitemap: https://www.threads.com/sitemap/threads_tags_hashtag_sitemap.xml.gz
339Sitemap: https://www.threads.com/sitemap/threads_tags_gsc_bulk_keywords_sitemap.xml.gz
340Sitemap: https://www.threads.com/sitemap/threads_tags_typeahead_sitemap.xml.gz
341