NeuralCrawl

Instagram / robots.txt snapshot

← back to instagram.com · fetched 2026-11-05T19:53:47Z (1d ago) · HTTP 200 · 6392 bytes · sha256 3a62d0ddf2d87aa7 · raw

final URL: https://www.instagram.com/robots.txt

1# Notice: Collection of data on Instagram through automated means is
2# prohibited unless you have express written permission from Instagram
3# and may only be conducted for the limited purpose contained in said
4# permission.
5# All authorized user-agents listed on this page must comply with Meta’s
6# Automated Data Collection Terms available at:
7# https://www.facebook.com/legal/automated_data_collection_terms
8User-agent: __usage_ai-training
9Disallow: /
10
11User-agent: Amazonbot
12Disallow: /
13
14User-agent: Applebot-Extended
15Disallow: /
16
17User-agent: Brightbot
18Disallow: /
19
20User-agent: ClaudeBot
21Disallow: /
22
23User-agent: Google-Extended
24Disallow: /
25
26User-agent: GPTBot
27Disallow: /
28
29User-agent: PerplexityBot
30Disallow: /
31
32User-agent: PetalBot
33Disallow: /
34
35User-agent: Scrapy
36Disallow: /
37
38User-agent: uptimerobot
39Disallow: /
40
41User-agent: viberbot
42Disallow: /
43
44User-agent: YaK
45Disallow: /
46
47User-agent: Yandex
48Disallow: /
49
50User-agent: Yeti
51Disallow: /
52
53User-agent: Applebot
54Disallow: /accounts/login/*?next=
55Disallow: /ajax/
56Disallow: /client_error/
57Disallow: /direct/
58Disallow: /logging/
59Disallow: /publicapi/
60Disallow: /qp/batch_fetch_web/
61Disallow: /query/
62Allow: /places/c/
63
64User-agent: Bingbot
65Disallow: /*/c/
66Disallow: /*/comments/
67Disallow: /*/liked_by/
68Disallow: /accounts/login/*?next=
69Disallow: /ajax/
70Disallow: /client_error/
71Disallow: /direct/
72Disallow: /logging/
73Disallow: /publicapi/
74Disallow: /qp/batch_fetch_web/
75Disallow: /query/
76Allow: /places/c/
77
78User-agent: deepcrawl
79Disallow: /accounts/login/*?next=
80Disallow: /ajax/
81Disallow: /client_error/
82Disallow: /direct/
83Disallow: /logging/
84Disallow: /publicapi/
85Disallow: /qp/batch_fetch_web/
86Disallow: /query/
87Allow: /places/c/
88
89User-agent: Discordbot
90Disallow: /accounts/login/*?next=
91Disallow: /ajax/
92Disallow: /client_error/
93Disallow: /direct/
94Disallow: /logging/
95Disallow: /publicapi/
96Disallow: /qp/batch_fetch_web/
97Disallow: /query/
98Allow: /places/c/
99
100User-agent: DuckDuckBot
101Disallow: /accounts/login/*?next=
102Disallow: /ajax/
103Disallow: /client_error/
104Disallow: /direct/
105Disallow: /logging/
106Disallow: /publicapi/
107Disallow: /qp/batch_fetch_web/
108Disallow: /query/
109Allow: /places/c/
110
111User-agent: facebookexternalhit
112Disallow: /accounts/login/*?next=
113Disallow: /ajax/
114Disallow: /client_error/
115Disallow: /direct/
116Disallow: /logging/
117Disallow: /publicapi/
118Disallow: /qp/batch_fetch_web/
119Disallow: /query/
120Allow: /places/c/
121
122User-agent: Googlebot
123Disallow: /*/c/
124Disallow: /*/comments/
125Disallow: /*/liked_by/
126Disallow: /accounts/login/*?next=
127Disallow: /ajax/
128Disallow: /client_error/
129Disallow: /direct/
130Disallow: /logging/
131Disallow: /publicapi/
132Disallow: /qp/batch_fetch_web/
133Disallow: /query/
134Allow: /places/c/
135
136User-agent: Googlebot-Image
137Disallow: /accounts/login/*?next=
138Disallow: /ajax/
139Disallow: /client_error/
140Disallow: /direct/
141Disallow: /logging/
142Disallow: /publicapi/
143Disallow: /qp/batch_fetch_web/
144Disallow: /query/
145Allow: /places/c/
146
147User-agent: Googlebot-Video
148Disallow: /accounts/login/*?next=
149Disallow: /ajax/
150Disallow: /client_error/
151Disallow: /direct/
152Disallow: /logging/
153Disallow: /publicapi/
154Disallow: /qp/batch_fetch_web/
155Disallow: /query/
156Allow: /places/c/
157
158User-agent: Googlebot-News
159Disallow: /accounts/login/*?next=
160Disallow: /ajax/
161Disallow: /client_error/
162Disallow: /direct/
163Disallow: /logging/
164Disallow: /publicapi/
165Disallow: /qp/batch_fetch_web/
166Disallow: /query/
167Allow: /places/c/
168
169User-agent: Google-InspectionTool
170Disallow: /accounts/login/*?next=
171Disallow: /ajax/
172Disallow: /client_error/
173Disallow: /direct/
174Disallow: /logging/
175Disallow: /publicapi/
176Disallow: /qp/batch_fetch_web/
177Disallow: /query/
178Allow: /places/c/
179
180User-agent: lumar
181Disallow: /accounts/login/*?next=
182Disallow: /ajax/
183Disallow: /client_error/
184Disallow: /direct/
185Disallow: /logging/
186Disallow: /publicapi/
187Disallow: /qp/batch_fetch_web/
188Disallow: /query/
189Allow: /places/c/
190
191User-agent: LinkedInBot
192Disallow: /accounts/login/*?next=
193Disallow: /ajax/
194Disallow: /client_error/
195Disallow: /direct/
196Disallow: /logging/
197Disallow: /publicapi/
198Disallow: /qp/batch_fetch_web/
199Disallow: /query/
200Allow: /places/c/
201
202User-agent: msnbot
203Disallow: /accounts/login/*?next=
204Disallow: /ajax/
205Disallow: /client_error/
206Disallow: /direct/
207Disallow: /logging/
208Disallow: /publicapi/
209Disallow: /qp/batch_fetch_web/
210Disallow: /query/
211Allow: /places/c/
212
213User-agent: Pinterestbot
214Disallow: /accounts/login/*?next=
215Disallow: /ajax/
216Disallow: /client_error/
217Disallow: /direct/
218Disallow: /logging/
219Disallow: /publicapi/
220Disallow: /qp/batch_fetch_web/
221Disallow: /query/
222Allow: /places/c/
223
224User-agent: Screaming Frog SEO Spider
225Disallow: /accounts/login/*?next=
226Disallow: /ajax/
227Disallow: /client_error/
228Disallow: /direct/
229Disallow: /logging/
230Disallow: /publicapi/
231Disallow: /qp/batch_fetch_web/
232Disallow: /query/
233Allow: /places/c/
234
235User-agent: seznambot
236Disallow: /accounts/login/*?next=
237Disallow: /ajax/
238Disallow: /client_error/
239Disallow: /direct/
240Disallow: /logging/
241Disallow: /publicapi/
242Disallow: /qp/batch_fetch_web/
243Disallow: /query/
244Allow: /places/c/
245
246User-agent: Slurp
247Disallow: /accounts/login/*?next=
248Disallow: /ajax/
249Disallow: /client_error/
250Disallow: /direct/
251Disallow: /logging/
252Disallow: /publicapi/
253Disallow: /qp/batch_fetch_web/
254Disallow: /query/
255Allow: /places/c/
256
257User-agent: teoma
258Disallow: /accounts/login/*?next=
259Disallow: /ajax/
260Disallow: /client_error/
261Disallow: /direct/
262Disallow: /logging/
263Disallow: /publicapi/
264Disallow: /qp/batch_fetch_web/
265Disallow: /query/
266Allow: /places/c/
267
268User-agent: TelegramBot
269Disallow: /accounts/login/*?next=
270Disallow: /ajax/
271Disallow: /client_error/
272Disallow: /direct/
273Disallow: /logging/
274Disallow: /publicapi/
275Disallow: /qp/batch_fetch_web/
276Disallow: /query/
277Allow: /places/c/
278
279User-agent: Twitterbot
280Disallow: /accounts/login/*?next=
281Disallow: /ajax/
282Disallow: /client_error/
283Disallow: /direct/
284Disallow: /logging/
285Disallow: /publicapi/
286Disallow: /qp/batch_fetch_web/
287Disallow: /query/
288Allow: /places/c/
289
290User-agent: *
291Disallow: /
292
293Sitemap: https://www.instagram.com/sitemap/ig_places_sitemap.xml.gz
294Sitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap.xml.gz
295Sitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap_non_media_eligible.xml.gz
296Sitemap: https://www.instagram.com/sitemap/dim_ig_logged_out_web_keyword_ent.xml.gz
297Sitemap: https://www.instagram.com/sitemap/ig_seo_profile_sitemap_v2.xml.gz
298Sitemap: https://www.instagram.com/sitemap/ig_seo_historical_reels_mvp_video_rows.xml.gz
299