NeuralCrawl

Instagram / robots.txt snapshot

← back to instagram.com · fetched 2026-10-18T13:46:13Z (1d ago) · HTTP 200 · 6182 bytes · sha256 fe84b3913976aa37 · raw

final URL: https://www.instagram.com/robots.txt

1# Notice: Collection of data on Instagram through automated means is
2# prohibited unless you have express written permission from Instagram
3# and may only be conducted for the limited purpose contained in said
4# permission.
5# All authorized user-agents listed on this page must comply with Meta’s
6# Automated Data Collection Terms available at:
7# https://www.facebook.com/legal/automated_data_collection_terms
8User-agent: Amazonbot
9Disallow: /
10
11User-agent: Applebot-Extended
12Disallow: /
13
14User-agent: Brightbot
15Disallow: /
16
17User-agent: ClaudeBot
18Disallow: /
19
20User-agent: Google-Extended
21Disallow: /
22
23User-agent: GPTBot
24Disallow: /
25
26User-agent: PerplexityBot
27Disallow: /
28
29User-agent: PetalBot
30Disallow: /
31
32User-agent: Scrapy
33Disallow: /
34
35User-agent: uptimerobot
36Disallow: /
37
38User-agent: viberbot
39Disallow: /
40
41User-agent: YaK
42Disallow: /
43
44User-agent: Yandex
45Disallow: /
46
47User-agent: Yeti
48Disallow: /
49
50User-agent: Applebot
51Disallow: /accounts/login/*?next=
52Disallow: /ajax/
53Disallow: /client_error/
54Disallow: /direct/
55Disallow: /logging/
56Disallow: /publicapi/
57Disallow: /qp/batch_fetch_web/
58Disallow: /query/
59Allow: /places/c/
60
61User-agent: Bingbot
62Disallow: /*/c/
63Disallow: /*/comments/
64Disallow: /*/liked_by/
65Disallow: /accounts/login/*?next=
66Disallow: /ajax/
67Disallow: /client_error/
68Disallow: /direct/
69Disallow: /logging/
70Disallow: /publicapi/
71Disallow: /qp/batch_fetch_web/
72Disallow: /query/
73Allow: /places/c/
74
75User-agent: deepcrawl
76Disallow: /accounts/login/*?next=
77Disallow: /ajax/
78Disallow: /client_error/
79Disallow: /direct/
80Disallow: /logging/
81Disallow: /publicapi/
82Disallow: /qp/batch_fetch_web/
83Disallow: /query/
84Allow: /places/c/
85
86User-agent: Discordbot
87Disallow: /accounts/login/*?next=
88Disallow: /ajax/
89Disallow: /client_error/
90Disallow: /direct/
91Disallow: /logging/
92Disallow: /publicapi/
93Disallow: /qp/batch_fetch_web/
94Disallow: /query/
95Allow: /places/c/
96
97User-agent: DuckDuckBot
98Disallow: /accounts/login/*?next=
99Disallow: /ajax/
100Disallow: /client_error/
101Disallow: /direct/
102Disallow: /logging/
103Disallow: /publicapi/
104Disallow: /qp/batch_fetch_web/
105Disallow: /query/
106Allow: /places/c/
107
108User-agent: facebookexternalhit
109Disallow: /accounts/login/*?next=
110Disallow: /ajax/
111Disallow: /client_error/
112Disallow: /direct/
113Disallow: /logging/
114Disallow: /publicapi/
115Disallow: /qp/batch_fetch_web/
116Disallow: /query/
117Allow: /places/c/
118
119User-agent: Googlebot
120Disallow: /*/c/
121Disallow: /*/comments/
122Disallow: /*/liked_by/
123Disallow: /accounts/login/*?next=
124Disallow: /ajax/
125Disallow: /client_error/
126Disallow: /direct/
127Disallow: /logging/
128Disallow: /publicapi/
129Disallow: /qp/batch_fetch_web/
130Disallow: /query/
131Allow: /places/c/
132
133User-agent: Googlebot-Image
134Disallow: /accounts/login/*?next=
135Disallow: /ajax/
136Disallow: /client_error/
137Disallow: /direct/
138Disallow: /logging/
139Disallow: /publicapi/
140Disallow: /qp/batch_fetch_web/
141Disallow: /query/
142Allow: /places/c/
143
144User-agent: Googlebot-Video
145Disallow: /accounts/login/*?next=
146Disallow: /ajax/
147Disallow: /client_error/
148Disallow: /direct/
149Disallow: /logging/
150Disallow: /publicapi/
151Disallow: /qp/batch_fetch_web/
152Disallow: /query/
153Allow: /places/c/
154
155User-agent: Googlebot-News
156Disallow: /accounts/login/*?next=
157Disallow: /ajax/
158Disallow: /client_error/
159Disallow: /direct/
160Disallow: /logging/
161Disallow: /publicapi/
162Disallow: /qp/batch_fetch_web/
163Disallow: /query/
164Allow: /places/c/
165
166User-agent: Google-InspectionTool
167Disallow: /accounts/login/*?next=
168Disallow: /ajax/
169Disallow: /client_error/
170Disallow: /direct/
171Disallow: /logging/
172Disallow: /publicapi/
173Disallow: /qp/batch_fetch_web/
174Disallow: /query/
175Allow: /places/c/
176
177User-agent: lumar
178Disallow: /accounts/login/*?next=
179Disallow: /ajax/
180Disallow: /client_error/
181Disallow: /direct/
182Disallow: /logging/
183Disallow: /publicapi/
184Disallow: /qp/batch_fetch_web/
185Disallow: /query/
186Allow: /places/c/
187
188User-agent: LinkedInBot
189Disallow: /accounts/login/*?next=
190Disallow: /ajax/
191Disallow: /client_error/
192Disallow: /direct/
193Disallow: /logging/
194Disallow: /publicapi/
195Disallow: /qp/batch_fetch_web/
196Disallow: /query/
197Allow: /places/c/
198
199User-agent: msnbot
200Disallow: /accounts/login/*?next=
201Disallow: /ajax/
202Disallow: /client_error/
203Disallow: /direct/
204Disallow: /logging/
205Disallow: /publicapi/
206Disallow: /qp/batch_fetch_web/
207Disallow: /query/
208Allow: /places/c/
209
210User-agent: Pinterestbot
211Disallow: /accounts/login/*?next=
212Disallow: /ajax/
213Disallow: /client_error/
214Disallow: /direct/
215Disallow: /logging/
216Disallow: /publicapi/
217Disallow: /qp/batch_fetch_web/
218Disallow: /query/
219Allow: /places/c/
220
221User-agent: Screaming Frog SEO Spider
222Disallow: /accounts/login/*?next=
223Disallow: /ajax/
224Disallow: /client_error/
225Disallow: /direct/
226Disallow: /logging/
227Disallow: /publicapi/
228Disallow: /qp/batch_fetch_web/
229Disallow: /query/
230Allow: /places/c/
231
232User-agent: seznambot
233Disallow: /accounts/login/*?next=
234Disallow: /ajax/
235Disallow: /client_error/
236Disallow: /direct/
237Disallow: /logging/
238Disallow: /publicapi/
239Disallow: /qp/batch_fetch_web/
240Disallow: /query/
241Allow: /places/c/
242
243User-agent: Slurp
244Disallow: /accounts/login/*?next=
245Disallow: /ajax/
246Disallow: /client_error/
247Disallow: /direct/
248Disallow: /logging/
249Disallow: /publicapi/
250Disallow: /qp/batch_fetch_web/
251Disallow: /query/
252Allow: /places/c/
253
254User-agent: teoma
255Disallow: /accounts/login/*?next=
256Disallow: /ajax/
257Disallow: /client_error/
258Disallow: /direct/
259Disallow: /logging/
260Disallow: /publicapi/
261Disallow: /qp/batch_fetch_web/
262Disallow: /query/
263Allow: /places/c/
264
265User-agent: TelegramBot
266Disallow: /accounts/login/*?next=
267Disallow: /ajax/
268Disallow: /client_error/
269Disallow: /direct/
270Disallow: /logging/
271Disallow: /publicapi/
272Disallow: /qp/batch_fetch_web/
273Disallow: /query/
274Allow: /places/c/
275
276User-agent: Twitterbot
277Disallow: /accounts/login/*?next=
278Disallow: /ajax/
279Disallow: /client_error/
280Disallow: /direct/
281Disallow: /logging/
282Disallow: /publicapi/
283Disallow: /qp/batch_fetch_web/
284Disallow: /query/
285Allow: /places/c/
286
287User-agent: *
288Disallow: /
289
290Sitemap: https://www.instagram.com/sitemap/ig_places_sitemap.xml.gz
291Sitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap.xml.gz
292Sitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap_non_media_eligible.xml.gz
293Sitemap: https://www.instagram.com/sitemap/dim_ig_logged_out_web_keyword_ent.xml.gz
294