NeuralCrawl

Threads / robots.txt snapshot

← back to threads.net · fetched 2026-11-05T19:53:47Z (1d ago) · HTTP 200 · 8624 bytes · sha256 aaf0cd8d1ad02bd8 · raw

final URL: https://www.threads.com/robots.txt

1# Notice: Collection of data on Threads through automated means is
2# prohibited unless you have express written permission from Threads
3# and may only be conducted for the limited purpose contained in said
4# permission.
5# All authorized user-agents listed on this page must comply with Meta’s
6# Automated Data Collection Terms available at:
7# https://www.facebook.com/legal/automated_data_collection_terms
8User-agent: __usage_ai-training
9Disallow: /
10
11User-agent: Amazonbot
12Disallow: /
13
14User-agent: Applebot-Extended
15Disallow: /
16
17User-agent: Brightbot
18Disallow: /
19
20User-agent: ClaudeBot
21Disallow: /
22
23User-agent: Google-Extended
24Disallow: /
25
26User-agent: GPTBot
27Disallow: /
28
29User-agent: PerplexityBot
30Disallow: /
31
32User-agent: PetalBot
33Disallow: /
34
35User-agent: Scrapy
36Disallow: /
37
38User-agent: uptimerobot
39Disallow: /
40
41User-agent: viberbot
42Disallow: /
43
44User-agent: YaK
45Disallow: /
46
47User-agent: Yandex
48Disallow: /
49
50User-agent: Yeti
51Disallow: /
52
53User-agent: Applebot
54Disallow: /accounts/
55Disallow: /activity/
56Disallow: /ajax/
57Disallow: /client_error/
58Disallow: /deck/
59Disallow: /intent/
60Disallow: /logging/
61Disallow: /onboarding/
62Disallow: /publicapi/
63Disallow: /qp/batch_fetch_web/
64Disallow: /query/
65Disallow: /settings/
66
67User-agent: Bingbot
68Disallow: /*/c/
69Disallow: /*/comments/
70Disallow: /*/liked_by/
71Disallow: /*/p/
72Disallow: /accounts/
73Disallow: /activity/
74Disallow: /ajax/
75Disallow: /client_error/
76Disallow: /deck/
77Disallow: /intent/
78Disallow: /logging/
79Disallow: /onboarding/
80Disallow: /publicapi/
81Disallow: /qp/batch_fetch_web/
82Disallow: /query/
83Disallow: /settings/
84
85User-agent: deepcrawl
86Disallow: /accounts/
87Disallow: /activity/
88Disallow: /ajax/
89Disallow: /client_error/
90Disallow: /deck/
91Disallow: /intent/
92Disallow: /logging/
93Disallow: /onboarding/
94Disallow: /publicapi/
95Disallow: /qp/batch_fetch_web/
96Disallow: /query/
97Disallow: /settings/
98
99User-agent: Discordbot
100Disallow: /accounts/
101Disallow: /activity/
102Disallow: /ajax/
103Disallow: /client_error/
104Disallow: /deck/
105Disallow: /intent/
106Disallow: /logging/
107Disallow: /onboarding/
108Disallow: /publicapi/
109Disallow: /qp/batch_fetch_web/
110Disallow: /query/
111Disallow: /settings/
112
113User-agent: DuckDuckBot
114Disallow: /accounts/
115Disallow: /activity/
116Disallow: /ajax/
117Disallow: /client_error/
118Disallow: /deck/
119Disallow: /intent/
120Disallow: /logging/
121Disallow: /onboarding/
122Disallow: /publicapi/
123Disallow: /qp/batch_fetch_web/
124Disallow: /query/
125Disallow: /settings/
126
127User-agent: facebookexternalhit
128Disallow: /accounts/
129Disallow: /activity/
130Disallow: /ajax/
131Disallow: /client_error/
132Disallow: /deck/
133Disallow: /intent/
134Disallow: /logging/
135Disallow: /onboarding/
136Disallow: /publicapi/
137Disallow: /qp/batch_fetch_web/
138Disallow: /query/
139Disallow: /settings/
140
141User-agent: Googlebot
142Disallow: /*/c/
143Disallow: /*/comments/
144Disallow: /*/liked_by/
145Disallow: /*/p/
146Disallow: /accounts/
147Disallow: /activity/
148Disallow: /ajax/
149Disallow: /client_error/
150Disallow: /deck/
151Disallow: /intent/
152Disallow: /logging/
153Disallow: /onboarding/
154Disallow: /publicapi/
155Disallow: /qp/batch_fetch_web/
156Disallow: /query/
157Disallow: /settings/
158
159User-agent: Googlebot-Image
160Disallow: /accounts/
161Disallow: /activity/
162Disallow: /ajax/
163Disallow: /client_error/
164Disallow: /deck/
165Disallow: /intent/
166Disallow: /logging/
167Disallow: /onboarding/
168Disallow: /publicapi/
169Disallow: /qp/batch_fetch_web/
170Disallow: /query/
171Disallow: /settings/
172
173User-agent: Googlebot-Video
174Disallow: /accounts/
175Disallow: /activity/
176Disallow: /ajax/
177Disallow: /client_error/
178Disallow: /deck/
179Disallow: /intent/
180Disallow: /logging/
181Disallow: /onboarding/
182Disallow: /publicapi/
183Disallow: /qp/batch_fetch_web/
184Disallow: /query/
185Disallow: /settings/
186
187User-agent: Googlebot-News
188Disallow: /accounts/
189Disallow: /activity/
190Disallow: /ajax/
191Disallow: /client_error/
192Disallow: /deck/
193Disallow: /intent/
194Disallow: /logging/
195Disallow: /onboarding/
196Disallow: /publicapi/
197Disallow: /qp/batch_fetch_web/
198Disallow: /query/
199Disallow: /settings/
200
201User-agent: Google-InspectionTool
202Disallow: /accounts/
203Disallow: /activity/
204Disallow: /ajax/
205Disallow: /client_error/
206Disallow: /deck/
207Disallow: /intent/
208Disallow: /logging/
209Disallow: /onboarding/
210Disallow: /publicapi/
211Disallow: /qp/batch_fetch_web/
212Disallow: /query/
213Disallow: /settings/
214
215User-agent: lumar
216Disallow: /accounts/
217Disallow: /activity/
218Disallow: /ajax/
219Disallow: /client_error/
220Disallow: /deck/
221Disallow: /intent/
222Disallow: /logging/
223Disallow: /onboarding/
224Disallow: /publicapi/
225Disallow: /qp/batch_fetch_web/
226Disallow: /query/
227Disallow: /settings/
228
229User-agent: LinkedInBot
230Disallow: /accounts/
231Disallow: /activity/
232Disallow: /ajax/
233Disallow: /client_error/
234Disallow: /deck/
235Disallow: /intent/
236Disallow: /logging/
237Disallow: /onboarding/
238Disallow: /publicapi/
239Disallow: /qp/batch_fetch_web/
240Disallow: /query/
241Disallow: /settings/
242
243User-agent: msnbot
244Disallow: /accounts/
245Disallow: /activity/
246Disallow: /ajax/
247Disallow: /client_error/
248Disallow: /deck/
249Disallow: /intent/
250Disallow: /logging/
251Disallow: /onboarding/
252Disallow: /publicapi/
253Disallow: /qp/batch_fetch_web/
254Disallow: /query/
255Disallow: /settings/
256
257User-agent: Pinterestbot
258Disallow: /accounts/
259Disallow: /activity/
260Disallow: /ajax/
261Disallow: /client_error/
262Disallow: /deck/
263Disallow: /intent/
264Disallow: /logging/
265Disallow: /onboarding/
266Disallow: /publicapi/
267Disallow: /qp/batch_fetch_web/
268Disallow: /query/
269Disallow: /settings/
270
271User-agent: Screaming Frog SEO Spider
272Disallow: /accounts/
273Disallow: /activity/
274Disallow: /ajax/
275Disallow: /client_error/
276Disallow: /deck/
277Disallow: /intent/
278Disallow: /logging/
279Disallow: /onboarding/
280Disallow: /publicapi/
281Disallow: /qp/batch_fetch_web/
282Disallow: /query/
283Disallow: /settings/
284
285User-agent: seznambot
286Disallow: /accounts/
287Disallow: /activity/
288Disallow: /ajax/
289Disallow: /client_error/
290Disallow: /deck/
291Disallow: /intent/
292Disallow: /logging/
293Disallow: /onboarding/
294Disallow: /publicapi/
295Disallow: /qp/batch_fetch_web/
296Disallow: /query/
297Disallow: /settings/
298
299User-agent: Slurp
300Disallow: /accounts/
301Disallow: /activity/
302Disallow: /ajax/
303Disallow: /client_error/
304Disallow: /deck/
305Disallow: /intent/
306Disallow: /logging/
307Disallow: /onboarding/
308Disallow: /publicapi/
309Disallow: /qp/batch_fetch_web/
310Disallow: /query/
311Disallow: /settings/
312
313User-agent: teoma
314Disallow: /accounts/
315Disallow: /activity/
316Disallow: /ajax/
317Disallow: /client_error/
318Disallow: /deck/
319Disallow: /intent/
320Disallow: /logging/
321Disallow: /onboarding/
322Disallow: /publicapi/
323Disallow: /qp/batch_fetch_web/
324Disallow: /query/
325Disallow: /settings/
326
327User-agent: TelegramBot
328Disallow: /accounts/
329Disallow: /activity/
330Disallow: /ajax/
331Disallow: /client_error/
332Disallow: /deck/
333Disallow: /intent/
334Disallow: /logging/
335Disallow: /onboarding/
336Disallow: /publicapi/
337Disallow: /qp/batch_fetch_web/
338Disallow: /query/
339Disallow: /settings/
340
341User-agent: Twitterbot
342Disallow: /accounts/
343Disallow: /activity/
344Disallow: /ajax/
345Disallow: /client_error/
346Disallow: /deck/
347Disallow: /intent/
348Disallow: /logging/
349Disallow: /onboarding/
350Disallow: /publicapi/
351Disallow: /qp/batch_fetch_web/
352Disallow: /query/
353Disallow: /settings/
354
355User-agent: redditbot
356Disallow: /accounts/
357Disallow: /activity/
358Disallow: /ajax/
359Disallow: /client_error/
360Disallow: /deck/
361Disallow: /intent/
362Disallow: /logging/
363Disallow: /onboarding/
364Disallow: /publicapi/
365Disallow: /qp/batch_fetch_web/
366Disallow: /query/
367Disallow: /settings/
368
369User-agent: *
370Disallow: /
371
372Sitemap: https://www.threads.com/sitemap/threads_profiles_sitemap.xml.gz
373Sitemap: https://www.threads.com/sitemap/threads_profiles_sitemap_tier_1.xml.gz
374Sitemap: https://www.threads.com/sitemap/threads_profiles_sitemap_tier_2.xml.gz
375Sitemap: https://www.threads.com/sitemap/threads_profiles_sitemap_tier_3.xml.gz
376Sitemap: https://www.threads.com/sitemap/threads_profiles_sitemap_tier_4.xml.gz
377Sitemap: https://www.threads.com/sitemap/threads_profiles_sitemap_tier_5.xml.gz
378Sitemap: https://www.threads.com/sitemap/us_top_creators_sitemap.xml.gz
379Sitemap: https://www.threads.com/sitemap/us_emerging_creators_sitemap.xml.gz
380Sitemap: https://www.threads.com/sitemap/us_quality_tagged_tail_sitemap.xml.gz
381Sitemap: https://www.threads.com/sitemap/us_active_producers_sitemap.xml.gz
382Sitemap: https://www.threads.com/sitemap/us_micro_producers_sitemap.xml.gz
383Sitemap: https://www.threads.com/sitemap/threads_high_potential_posts_v2_sitemap.xml.gz
384Sitemap: https://www.threads.com/sitemap/threads_question_posts_sitemap.xml.gz
385Sitemap: https://www.threads.com/sitemap/threads_tech_posts_sitemap.xml.gz
386Sitemap: https://www.threads.com/sitemap/threads_tags_hashtag_sitemap.xml.gz
387Sitemap: https://www.threads.com/sitemap/threads_tags_gsc_bulk_keywords_sitemap.xml.gz
388Sitemap: https://www.threads.com/sitemap/threads_tags_typeahead_sitemap.xml.gz
389