NeuralCrawl

Instagram / robots.txt snapshot

← back to instagram.com · fetched 2026-06-20T01:10:30Z (18h ago) · HTTP 200 · 5878 bytes · sha256 62e4fe4707ff0516 · raw

final URL: https://www.instagram.com/robots.txt

1# Notice: Collection of data on Instagram through automated means is
2# prohibited unless you have express written permission from Instagram
3# and may only be conducted for the limited purpose contained in said
4# permission.
5# All authorized user-agents listed on this page must comply with Meta’s
6# Automated Data Collection Terms available at:
7# https://www.facebook.com/legal/automated_data_collection_terms
8User-agent: Amazonbot
9Disallow: /
10
11User-agent: Applebot-Extended
12Disallow: /
13
14User-agent: Brightbot
15Disallow: /
16
17User-agent: ClaudeBot
18Disallow: /
19
20User-agent: Google-Extended
21Disallow: /
22
23User-agent: GPTBot
24Disallow: /
25
26User-agent: PerplexityBot
27Disallow: /
28
29User-agent: PetalBot
30Disallow: /
31
32User-agent: Scrapy
33Disallow: /
34
35User-agent: uptimerobot
36Disallow: /
37
38User-agent: viberbot
39Disallow: /
40
41User-agent: YaK
42Disallow: /
43
44User-agent: Yandex
45Disallow: /
46
47User-agent: Yeti
48Disallow: /
49
50User-agent: Applebot
51Disallow: /accounts/login/*?next=
52Disallow: /ajax/
53Disallow: /client_error/
54Disallow: /direct/
55Disallow: /logging/
56Disallow: /publicapi/
57Disallow: /qp/batch_fetch_web/
58Disallow: /query/
59Allow: /places/c/
60
61User-agent: Bingbot
62Disallow: /*/c/
63Disallow: /*/comments/
64Disallow: /*/liked_by/
65Disallow: /accounts/login/*?next=
66Disallow: /ajax/
67Disallow: /client_error/
68Disallow: /direct/
69Disallow: /logging/
70Disallow: /publicapi/
71Disallow: /qp/batch_fetch_web/
72Disallow: /query/
73Allow: /places/c/
74
75User-agent: Discordbot
76Disallow: /accounts/login/*?next=
77Disallow: /ajax/
78Disallow: /client_error/
79Disallow: /direct/
80Disallow: /logging/
81Disallow: /publicapi/
82Disallow: /qp/batch_fetch_web/
83Disallow: /query/
84Allow: /places/c/
85
86User-agent: DuckDuckBot
87Disallow: /accounts/login/*?next=
88Disallow: /ajax/
89Disallow: /client_error/
90Disallow: /direct/
91Disallow: /logging/
92Disallow: /publicapi/
93Disallow: /qp/batch_fetch_web/
94Disallow: /query/
95Allow: /places/c/
96
97User-agent: facebookexternalhit
98Disallow: /accounts/login/*?next=
99Disallow: /ajax/
100Disallow: /client_error/
101Disallow: /direct/
102Disallow: /logging/
103Disallow: /publicapi/
104Disallow: /qp/batch_fetch_web/
105Disallow: /query/
106Allow: /places/c/
107
108User-agent: Googlebot
109Disallow: /*/c/
110Disallow: /*/comments/
111Disallow: /*/liked_by/
112Disallow: /accounts/login/*?next=
113Disallow: /ajax/
114Disallow: /client_error/
115Disallow: /direct/
116Disallow: /logging/
117Disallow: /publicapi/
118Disallow: /qp/batch_fetch_web/
119Disallow: /query/
120Allow: /places/c/
121
122User-agent: Googlebot-Image
123Disallow: /accounts/login/*?next=
124Disallow: /ajax/
125Disallow: /client_error/
126Disallow: /direct/
127Disallow: /logging/
128Disallow: /publicapi/
129Disallow: /qp/batch_fetch_web/
130Disallow: /query/
131Allow: /places/c/
132
133User-agent: Googlebot-Video
134Disallow: /accounts/login/*?next=
135Disallow: /ajax/
136Disallow: /client_error/
137Disallow: /direct/
138Disallow: /logging/
139Disallow: /publicapi/
140Disallow: /qp/batch_fetch_web/
141Disallow: /query/
142Allow: /places/c/
143
144User-agent: Googlebot-News
145Disallow: /accounts/login/*?next=
146Disallow: /ajax/
147Disallow: /client_error/
148Disallow: /direct/
149Disallow: /logging/
150Disallow: /publicapi/
151Disallow: /qp/batch_fetch_web/
152Disallow: /query/
153Allow: /places/c/
154
155User-agent: Google-InspectionTool
156Disallow: /accounts/login/*?next=
157Disallow: /ajax/
158Disallow: /client_error/
159Disallow: /direct/
160Disallow: /logging/
161Disallow: /publicapi/
162Disallow: /qp/batch_fetch_web/
163Disallow: /query/
164Allow: /places/c/
165
166User-agent: LinkedInBot
167Disallow: /accounts/login/*?next=
168Disallow: /ajax/
169Disallow: /client_error/
170Disallow: /direct/
171Disallow: /logging/
172Disallow: /publicapi/
173Disallow: /qp/batch_fetch_web/
174Disallow: /query/
175Allow: /places/c/
176
177User-agent: msnbot
178Disallow: /accounts/login/*?next=
179Disallow: /ajax/
180Disallow: /client_error/
181Disallow: /direct/
182Disallow: /logging/
183Disallow: /publicapi/
184Disallow: /qp/batch_fetch_web/
185Disallow: /query/
186Allow: /places/c/
187
188User-agent: Pinterestbot
189Disallow: /accounts/login/*?next=
190Disallow: /ajax/
191Disallow: /client_error/
192Disallow: /direct/
193Disallow: /logging/
194Disallow: /publicapi/
195Disallow: /qp/batch_fetch_web/
196Disallow: /query/
197Allow: /places/c/
198
199User-agent: Screaming Frog SEO Spider
200Disallow: /accounts/login/*?next=
201Disallow: /ajax/
202Disallow: /client_error/
203Disallow: /direct/
204Disallow: /logging/
205Disallow: /publicapi/
206Disallow: /qp/batch_fetch_web/
207Disallow: /query/
208Allow: /places/c/
209
210User-agent: seznambot
211Disallow: /accounts/login/*?next=
212Disallow: /ajax/
213Disallow: /client_error/
214Disallow: /direct/
215Disallow: /logging/
216Disallow: /publicapi/
217Disallow: /qp/batch_fetch_web/
218Disallow: /query/
219Allow: /places/c/
220
221User-agent: Slurp
222Disallow: /accounts/login/*?next=
223Disallow: /ajax/
224Disallow: /client_error/
225Disallow: /direct/
226Disallow: /logging/
227Disallow: /publicapi/
228Disallow: /qp/batch_fetch_web/
229Disallow: /query/
230Allow: /places/c/
231
232User-agent: teoma
233Disallow: /accounts/login/*?next=
234Disallow: /ajax/
235Disallow: /client_error/
236Disallow: /direct/
237Disallow: /logging/
238Disallow: /publicapi/
239Disallow: /qp/batch_fetch_web/
240Disallow: /query/
241Allow: /places/c/
242
243User-agent: TelegramBot
244Disallow: /accounts/login/*?next=
245Disallow: /ajax/
246Disallow: /client_error/
247Disallow: /direct/
248Disallow: /logging/
249Disallow: /publicapi/
250Disallow: /qp/batch_fetch_web/
251Disallow: /query/
252Allow: /places/c/
253
254User-agent: Twitterbot
255Disallow: /accounts/login/*?next=
256Disallow: /ajax/
257Disallow: /client_error/
258Disallow: /direct/
259Disallow: /logging/
260Disallow: /publicapi/
261Disallow: /qp/batch_fetch_web/
262Disallow: /query/
263Allow: /places/c/
264
265User-agent: *
266Disallow: /
267
268Sitemap: https://www.instagram.com/sitemap/ig_places_sitemap.xml.gz
269Sitemap: https://www.instagram.com/sitemap/ig_seo_images_sitemap.xml.gz
270Sitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap.xml.gz
271Sitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap_non_media_eligible.xml.gz
272Sitemap: https://www.instagram.com/sitemap/ig_seo_location_sitemap.xml.gz
273Sitemap: https://www.instagram.com/sitemap/dim_ig_logged_out_web_keyword_ent.xml.gz
274