NeuralCrawl

Instagram / robots.txt snapshot

← back to instagram.com · fetched 2026-10-07T21:56:40Z (30d ago) · HTTP 200 · 7526 bytes · sha256 ab723f021ad86e68 · raw

final URL: https://www.instagram.com/robots.txt

1# Notice: Collection of data on Instagram through automated means is
2# prohibited unless you have express written permission from Instagram
3# and may only be conducted for the limited purpose contained in said
4# permission.
5# All authorized user-agents listed on this page must comply with Meta’s
6# Automated Data Collection Terms available at:
7# https://www.facebook.com/legal/automated_data_collection_terms
8User-agent: PetalBot
9Disallow: /
10
11User-agent: Scrapy
12Disallow: /
13
14User-agent: uptimerobot
15Disallow: /
16
17User-agent: viberbot
18Disallow: /
19
20User-agent: YaK
21Disallow: /
22
23User-agent: Yandex
24Disallow: /
25
26User-agent: Yeti
27Disallow: /
28
29User-agent: Applebot
30Disallow: /accounts/login/*?next=
31Disallow: /ajax/
32Disallow: /client_error/
33Disallow: /direct/
34Disallow: /logging/
35Disallow: /publicapi/
36Disallow: /qp/batch_fetch_web/
37Disallow: /query/
38Allow: /places/c/
39
40User-agent: Bingbot
41Disallow: /*/c/
42Disallow: /*/comments/
43Disallow: /*/liked_by/
44Disallow: /accounts/login/*?next=
45Disallow: /ajax/
46Disallow: /client_error/
47Disallow: /direct/
48Disallow: /logging/
49Disallow: /publicapi/
50Disallow: /qp/batch_fetch_web/
51Disallow: /query/
52Allow: /places/c/
53
54User-agent: deepcrawl
55Disallow: /accounts/login/*?next=
56Disallow: /ajax/
57Disallow: /client_error/
58Disallow: /direct/
59Disallow: /logging/
60Disallow: /publicapi/
61Disallow: /qp/batch_fetch_web/
62Disallow: /query/
63Allow: /places/c/
64
65User-agent: Discordbot
66Disallow: /accounts/login/*?next=
67Disallow: /ajax/
68Disallow: /client_error/
69Disallow: /direct/
70Disallow: /logging/
71Disallow: /publicapi/
72Disallow: /qp/batch_fetch_web/
73Disallow: /query/
74Allow: /places/c/
75
76User-agent: DuckDuckBot
77Disallow: /accounts/login/*?next=
78Disallow: /ajax/
79Disallow: /client_error/
80Disallow: /direct/
81Disallow: /logging/
82Disallow: /publicapi/
83Disallow: /qp/batch_fetch_web/
84Disallow: /query/
85Allow: /places/c/
86
87User-agent: facebookexternalhit
88Disallow: /accounts/login/*?next=
89Disallow: /ajax/
90Disallow: /client_error/
91Disallow: /direct/
92Disallow: /logging/
93Disallow: /publicapi/
94Disallow: /qp/batch_fetch_web/
95Disallow: /query/
96Allow: /places/c/
97
98User-agent: Googlebot
99Disallow: /*/c/
100Disallow: /*/comments/
101Disallow: /*/liked_by/
102Disallow: /accounts/login/*?next=
103Disallow: /ajax/
104Disallow: /client_error/
105Disallow: /direct/
106Disallow: /logging/
107Disallow: /publicapi/
108Disallow: /qp/batch_fetch_web/
109Disallow: /query/
110Allow: /places/c/
111
112User-agent: Googlebot-Image
113Disallow: /accounts/login/*?next=
114Disallow: /ajax/
115Disallow: /client_error/
116Disallow: /direct/
117Disallow: /logging/
118Disallow: /publicapi/
119Disallow: /qp/batch_fetch_web/
120Disallow: /query/
121Allow: /places/c/
122
123User-agent: Googlebot-Video
124Disallow: /accounts/login/*?next=
125Disallow: /ajax/
126Disallow: /client_error/
127Disallow: /direct/
128Disallow: /logging/
129Disallow: /publicapi/
130Disallow: /qp/batch_fetch_web/
131Disallow: /query/
132Allow: /places/c/
133
134User-agent: Googlebot-News
135Disallow: /accounts/login/*?next=
136Disallow: /ajax/
137Disallow: /client_error/
138Disallow: /direct/
139Disallow: /logging/
140Disallow: /publicapi/
141Disallow: /qp/batch_fetch_web/
142Disallow: /query/
143Allow: /places/c/
144
145User-agent: Google-InspectionTool
146Disallow: /accounts/login/*?next=
147Disallow: /ajax/
148Disallow: /client_error/
149Disallow: /direct/
150Disallow: /logging/
151Disallow: /publicapi/
152Disallow: /qp/batch_fetch_web/
153Disallow: /query/
154Allow: /places/c/
155
156User-agent: lumar
157Disallow: /accounts/login/*?next=
158Disallow: /ajax/
159Disallow: /client_error/
160Disallow: /direct/
161Disallow: /logging/
162Disallow: /publicapi/
163Disallow: /qp/batch_fetch_web/
164Disallow: /query/
165Allow: /places/c/
166
167User-agent: LinkedInBot
168Disallow: /accounts/login/*?next=
169Disallow: /ajax/
170Disallow: /client_error/
171Disallow: /direct/
172Disallow: /logging/
173Disallow: /publicapi/
174Disallow: /qp/batch_fetch_web/
175Disallow: /query/
176Allow: /places/c/
177
178User-agent: msnbot
179Disallow: /accounts/login/*?next=
180Disallow: /ajax/
181Disallow: /client_error/
182Disallow: /direct/
183Disallow: /logging/
184Disallow: /publicapi/
185Disallow: /qp/batch_fetch_web/
186Disallow: /query/
187Allow: /places/c/
188
189User-agent: Pinterestbot
190Disallow: /accounts/login/*?next=
191Disallow: /ajax/
192Disallow: /client_error/
193Disallow: /direct/
194Disallow: /logging/
195Disallow: /publicapi/
196Disallow: /qp/batch_fetch_web/
197Disallow: /query/
198Allow: /places/c/
199
200User-agent: Screaming Frog SEO Spider
201Disallow: /accounts/login/*?next=
202Disallow: /ajax/
203Disallow: /client_error/
204Disallow: /direct/
205Disallow: /logging/
206Disallow: /publicapi/
207Disallow: /qp/batch_fetch_web/
208Disallow: /query/
209Allow: /places/c/
210
211User-agent: seznambot
212Disallow: /accounts/login/*?next=
213Disallow: /ajax/
214Disallow: /client_error/
215Disallow: /direct/
216Disallow: /logging/
217Disallow: /publicapi/
218Disallow: /qp/batch_fetch_web/
219Disallow: /query/
220Allow: /places/c/
221
222User-agent: Slurp
223Disallow: /accounts/login/*?next=
224Disallow: /ajax/
225Disallow: /client_error/
226Disallow: /direct/
227Disallow: /logging/
228Disallow: /publicapi/
229Disallow: /qp/batch_fetch_web/
230Disallow: /query/
231Allow: /places/c/
232
233User-agent: teoma
234Disallow: /accounts/login/*?next=
235Disallow: /ajax/
236Disallow: /client_error/
237Disallow: /direct/
238Disallow: /logging/
239Disallow: /publicapi/
240Disallow: /qp/batch_fetch_web/
241Disallow: /query/
242Allow: /places/c/
243
244User-agent: TelegramBot
245Disallow: /accounts/login/*?next=
246Disallow: /ajax/
247Disallow: /client_error/
248Disallow: /direct/
249Disallow: /logging/
250Disallow: /publicapi/
251Disallow: /qp/batch_fetch_web/
252Disallow: /query/
253Allow: /places/c/
254
255User-agent: Twitterbot
256Disallow: /accounts/login/*?next=
257Disallow: /ajax/
258Disallow: /client_error/
259Disallow: /direct/
260Disallow: /logging/
261Disallow: /publicapi/
262Disallow: /qp/batch_fetch_web/
263Disallow: /query/
264Allow: /places/c/
265
266User-agent: Amazonbot
267Disallow: /accounts/login/*?next=
268Disallow: /ajax/
269Disallow: /client_error/
270Disallow: /direct/
271Disallow: /logging/
272Disallow: /publicapi/
273Disallow: /qp/batch_fetch_web/
274Disallow: /query/
275Allow: /places/c/
276
277User-agent: Applebot-Extended
278Disallow: /accounts/login/*?next=
279Disallow: /ajax/
280Disallow: /client_error/
281Disallow: /direct/
282Disallow: /logging/
283Disallow: /publicapi/
284Disallow: /qp/batch_fetch_web/
285Disallow: /query/
286Allow: /places/c/
287
288User-agent: Brightbot
289Disallow: /accounts/login/*?next=
290Disallow: /ajax/
291Disallow: /client_error/
292Disallow: /direct/
293Disallow: /logging/
294Disallow: /publicapi/
295Disallow: /qp/batch_fetch_web/
296Disallow: /query/
297Allow: /places/c/
298
299User-agent: ClaudeBot
300Disallow: /accounts/login/*?next=
301Disallow: /ajax/
302Disallow: /client_error/
303Disallow: /direct/
304Disallow: /logging/
305Disallow: /publicapi/
306Disallow: /qp/batch_fetch_web/
307Disallow: /query/
308Allow: /places/c/
309
310User-agent: Google-Extended
311Disallow: /accounts/login/*?next=
312Disallow: /ajax/
313Disallow: /client_error/
314Disallow: /direct/
315Disallow: /logging/
316Disallow: /publicapi/
317Disallow: /qp/batch_fetch_web/
318Disallow: /query/
319Allow: /places/c/
320
321User-agent: GPTBot
322Disallow: /accounts/login/*?next=
323Disallow: /ajax/
324Disallow: /client_error/
325Disallow: /direct/
326Disallow: /logging/
327Disallow: /publicapi/
328Disallow: /qp/batch_fetch_web/
329Disallow: /query/
330Allow: /places/c/
331
332User-agent: PerplexityBot
333Disallow: /accounts/login/*?next=
334Disallow: /ajax/
335Disallow: /client_error/
336Disallow: /direct/
337Disallow: /logging/
338Disallow: /publicapi/
339Disallow: /qp/batch_fetch_web/
340Disallow: /query/
341Allow: /places/c/
342
343User-agent: *
344Disallow: /
345
346Sitemap: https://www.instagram.com/sitemap/ig_places_sitemap.xml.gz
347Sitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap.xml.gz
348Sitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap_non_media_eligible.xml.gz
349Sitemap: https://www.instagram.com/sitemap/dim_ig_logged_out_web_keyword_ent.xml.gz
350