NeuralCrawl

Instagram / robots.txt snapshot

← back to instagram.com · fetched 2026-10-15T06:25:59Z (23d ago) · HTTP 200 · 7334 bytes · sha256 c326b09ffb3c383b · raw

final URL: https://www.instagram.com/robots.txt

1# Notice: Collection of data on Instagram through automated means is
2# prohibited unless you have express written permission from Instagram
3# and may only be conducted for the limited purpose contained in said
4# permission.
5# All authorized user-agents listed on this page must comply with Meta’s
6# Automated Data Collection Terms available at:
7# https://www.facebook.com/legal/automated_data_collection_terms
8User-agent: Google-Extended
9Disallow: /
10
11User-agent: PetalBot
12Disallow: /
13
14User-agent: Scrapy
15Disallow: /
16
17User-agent: uptimerobot
18Disallow: /
19
20User-agent: viberbot
21Disallow: /
22
23User-agent: YaK
24Disallow: /
25
26User-agent: Yandex
27Disallow: /
28
29User-agent: Yeti
30Disallow: /
31
32User-agent: Applebot
33Disallow: /accounts/login/*?next=
34Disallow: /ajax/
35Disallow: /client_error/
36Disallow: /direct/
37Disallow: /logging/
38Disallow: /publicapi/
39Disallow: /qp/batch_fetch_web/
40Disallow: /query/
41Allow: /places/c/
42
43User-agent: Bingbot
44Disallow: /*/c/
45Disallow: /*/comments/
46Disallow: /*/liked_by/
47Disallow: /accounts/login/*?next=
48Disallow: /ajax/
49Disallow: /client_error/
50Disallow: /direct/
51Disallow: /logging/
52Disallow: /publicapi/
53Disallow: /qp/batch_fetch_web/
54Disallow: /query/
55Allow: /places/c/
56
57User-agent: deepcrawl
58Disallow: /accounts/login/*?next=
59Disallow: /ajax/
60Disallow: /client_error/
61Disallow: /direct/
62Disallow: /logging/
63Disallow: /publicapi/
64Disallow: /qp/batch_fetch_web/
65Disallow: /query/
66Allow: /places/c/
67
68User-agent: Discordbot
69Disallow: /accounts/login/*?next=
70Disallow: /ajax/
71Disallow: /client_error/
72Disallow: /direct/
73Disallow: /logging/
74Disallow: /publicapi/
75Disallow: /qp/batch_fetch_web/
76Disallow: /query/
77Allow: /places/c/
78
79User-agent: DuckDuckBot
80Disallow: /accounts/login/*?next=
81Disallow: /ajax/
82Disallow: /client_error/
83Disallow: /direct/
84Disallow: /logging/
85Disallow: /publicapi/
86Disallow: /qp/batch_fetch_web/
87Disallow: /query/
88Allow: /places/c/
89
90User-agent: facebookexternalhit
91Disallow: /accounts/login/*?next=
92Disallow: /ajax/
93Disallow: /client_error/
94Disallow: /direct/
95Disallow: /logging/
96Disallow: /publicapi/
97Disallow: /qp/batch_fetch_web/
98Disallow: /query/
99Allow: /places/c/
100
101User-agent: Googlebot
102Disallow: /*/c/
103Disallow: /*/comments/
104Disallow: /*/liked_by/
105Disallow: /accounts/login/*?next=
106Disallow: /ajax/
107Disallow: /client_error/
108Disallow: /direct/
109Disallow: /logging/
110Disallow: /publicapi/
111Disallow: /qp/batch_fetch_web/
112Disallow: /query/
113Allow: /places/c/
114
115User-agent: Googlebot-Image
116Disallow: /accounts/login/*?next=
117Disallow: /ajax/
118Disallow: /client_error/
119Disallow: /direct/
120Disallow: /logging/
121Disallow: /publicapi/
122Disallow: /qp/batch_fetch_web/
123Disallow: /query/
124Allow: /places/c/
125
126User-agent: Googlebot-Video
127Disallow: /accounts/login/*?next=
128Disallow: /ajax/
129Disallow: /client_error/
130Disallow: /direct/
131Disallow: /logging/
132Disallow: /publicapi/
133Disallow: /qp/batch_fetch_web/
134Disallow: /query/
135Allow: /places/c/
136
137User-agent: Googlebot-News
138Disallow: /accounts/login/*?next=
139Disallow: /ajax/
140Disallow: /client_error/
141Disallow: /direct/
142Disallow: /logging/
143Disallow: /publicapi/
144Disallow: /qp/batch_fetch_web/
145Disallow: /query/
146Allow: /places/c/
147
148User-agent: Google-InspectionTool
149Disallow: /accounts/login/*?next=
150Disallow: /ajax/
151Disallow: /client_error/
152Disallow: /direct/
153Disallow: /logging/
154Disallow: /publicapi/
155Disallow: /qp/batch_fetch_web/
156Disallow: /query/
157Allow: /places/c/
158
159User-agent: lumar
160Disallow: /accounts/login/*?next=
161Disallow: /ajax/
162Disallow: /client_error/
163Disallow: /direct/
164Disallow: /logging/
165Disallow: /publicapi/
166Disallow: /qp/batch_fetch_web/
167Disallow: /query/
168Allow: /places/c/
169
170User-agent: LinkedInBot
171Disallow: /accounts/login/*?next=
172Disallow: /ajax/
173Disallow: /client_error/
174Disallow: /direct/
175Disallow: /logging/
176Disallow: /publicapi/
177Disallow: /qp/batch_fetch_web/
178Disallow: /query/
179Allow: /places/c/
180
181User-agent: msnbot
182Disallow: /accounts/login/*?next=
183Disallow: /ajax/
184Disallow: /client_error/
185Disallow: /direct/
186Disallow: /logging/
187Disallow: /publicapi/
188Disallow: /qp/batch_fetch_web/
189Disallow: /query/
190Allow: /places/c/
191
192User-agent: Pinterestbot
193Disallow: /accounts/login/*?next=
194Disallow: /ajax/
195Disallow: /client_error/
196Disallow: /direct/
197Disallow: /logging/
198Disallow: /publicapi/
199Disallow: /qp/batch_fetch_web/
200Disallow: /query/
201Allow: /places/c/
202
203User-agent: Screaming Frog SEO Spider
204Disallow: /accounts/login/*?next=
205Disallow: /ajax/
206Disallow: /client_error/
207Disallow: /direct/
208Disallow: /logging/
209Disallow: /publicapi/
210Disallow: /qp/batch_fetch_web/
211Disallow: /query/
212Allow: /places/c/
213
214User-agent: seznambot
215Disallow: /accounts/login/*?next=
216Disallow: /ajax/
217Disallow: /client_error/
218Disallow: /direct/
219Disallow: /logging/
220Disallow: /publicapi/
221Disallow: /qp/batch_fetch_web/
222Disallow: /query/
223Allow: /places/c/
224
225User-agent: Slurp
226Disallow: /accounts/login/*?next=
227Disallow: /ajax/
228Disallow: /client_error/
229Disallow: /direct/
230Disallow: /logging/
231Disallow: /publicapi/
232Disallow: /qp/batch_fetch_web/
233Disallow: /query/
234Allow: /places/c/
235
236User-agent: teoma
237Disallow: /accounts/login/*?next=
238Disallow: /ajax/
239Disallow: /client_error/
240Disallow: /direct/
241Disallow: /logging/
242Disallow: /publicapi/
243Disallow: /qp/batch_fetch_web/
244Disallow: /query/
245Allow: /places/c/
246
247User-agent: TelegramBot
248Disallow: /accounts/login/*?next=
249Disallow: /ajax/
250Disallow: /client_error/
251Disallow: /direct/
252Disallow: /logging/
253Disallow: /publicapi/
254Disallow: /qp/batch_fetch_web/
255Disallow: /query/
256Allow: /places/c/
257
258User-agent: Twitterbot
259Disallow: /accounts/login/*?next=
260Disallow: /ajax/
261Disallow: /client_error/
262Disallow: /direct/
263Disallow: /logging/
264Disallow: /publicapi/
265Disallow: /qp/batch_fetch_web/
266Disallow: /query/
267Allow: /places/c/
268
269User-agent: Amazonbot
270Disallow: /accounts/login/*?next=
271Disallow: /ajax/
272Disallow: /client_error/
273Disallow: /direct/
274Disallow: /logging/
275Disallow: /publicapi/
276Disallow: /qp/batch_fetch_web/
277Disallow: /query/
278Allow: /places/c/
279
280User-agent: Applebot-Extended
281Disallow: /accounts/login/*?next=
282Disallow: /ajax/
283Disallow: /client_error/
284Disallow: /direct/
285Disallow: /logging/
286Disallow: /publicapi/
287Disallow: /qp/batch_fetch_web/
288Disallow: /query/
289Allow: /places/c/
290
291User-agent: Brightbot
292Disallow: /accounts/login/*?next=
293Disallow: /ajax/
294Disallow: /client_error/
295Disallow: /direct/
296Disallow: /logging/
297Disallow: /publicapi/
298Disallow: /qp/batch_fetch_web/
299Disallow: /query/
300Allow: /places/c/
301
302User-agent: ClaudeBot
303Disallow: /accounts/login/*?next=
304Disallow: /ajax/
305Disallow: /client_error/
306Disallow: /direct/
307Disallow: /logging/
308Disallow: /publicapi/
309Disallow: /qp/batch_fetch_web/
310Disallow: /query/
311Allow: /places/c/
312
313User-agent: GPTBot
314Disallow: /accounts/login/*?next=
315Disallow: /ajax/
316Disallow: /client_error/
317Disallow: /direct/
318Disallow: /logging/
319Disallow: /publicapi/
320Disallow: /qp/batch_fetch_web/
321Disallow: /query/
322Allow: /places/c/
323
324User-agent: PerplexityBot
325Disallow: /accounts/login/*?next=
326Disallow: /ajax/
327Disallow: /client_error/
328Disallow: /direct/
329Disallow: /logging/
330Disallow: /publicapi/
331Disallow: /qp/batch_fetch_web/
332Disallow: /query/
333Allow: /places/c/
334
335User-agent: *
336Disallow: /
337
338Sitemap: https://www.instagram.com/sitemap/ig_places_sitemap.xml.gz
339Sitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap.xml.gz
340Sitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap_non_media_eligible.xml.gz
341Sitemap: https://www.instagram.com/sitemap/dim_ig_logged_out_web_keyword_ent.xml.gz
342