NeuralCrawl

Shutterstock / robots.txt snapshot

← back to shutterstock.com · fetched 2026-08-06T19:27:21Z (3mo ago) · HTTP 200 · 9159 bytes · sha256 3345fd3ff5638ed9 · raw

final URL: https://www.shutterstock.com/robots.txt

1User-agent: *
2Allow: */search/login
3Allow: */login$
4Allow: /login.mhtml$
5Disallow: */login
6Disallow: */base/logout
7Allow: */search/account
8Disallow: */account
9Disallow: /subscribe_success
10Allow: /download_history.mhtml$
11Disallow: /download
12Disallow: /checkout
13Disallow: /account_purchase_history.mhtml
14Disallow: /contests
15Allow: */search/portfolio
16Disallow: */portfolio
17Disallow: */editor/template
18Disallow: */editor/design
19Disallow: */editor/search
20Disallow: */video/cart
21Disallow: */video/checkout
22Disallow: */music/cart
23Disallow: */music/user
24Disallow: */music/orders
25Disallow: /web/checkout
26Disallow: /sstk/api/footage/wasabi/*
27Disallow: /studioapi/wasabi/*
28Disallow: /sstk/api/*
29Disallow: /studioapi/*
30Disallow: /napi/*
31Disallow: /api/*
32Disallow: /feed.mhtml
33Disallow: /nuance-chat.html
34Disallow: /sw/*
35Disallow: /base/login
36Allow: */search/collections
37Allow: */editorial/collections
38Disallow: */collections
39Disallow: */generate/*
40Allow: */generate/$
41Disallow: /image-*chatId=*
42Disallow: /image-*tool=*
43Disallow: /image-*mode=*
44Disallow: /blog/?s=*
45Disallow: /cart
46Disallow: */search/ris/*
47Disallow: */search/models/*
48Disallow: /search/personalized/
49Disallow: /*/search/personalized/
50Disallow: /search/similar/
51Disallow: /*/search/similar/
52
53# Editor Images
54Disallow: */editor/image/*
55Allow: */editor/image/$
56
57# Disallowed bots
58User-agent: CCBot
59Disallow: /
60
61User-agent: GPTBot
62Disallow: /
63
64User-agent: ClaudeBot
65Disallow: /
66
67User-agent: meta-externalagent
68Disallow: /
69
70# AI discovery / search bots: allow by default, block only exceptions
71User-agent: OAI-SearchBot
72Allow: */search/login
73Allow: */login$
74Allow: /login.mhtml$
75Disallow: */login
76Disallow: */base/logout
77Allow: */search/account
78Disallow: */account
79Disallow: /subscribe_success
80Allow: /download_history.mhtml$
81Disallow: /download
82Disallow: /checkout
83Disallow: /account_purchase_history.mhtml
84Disallow: /contests
85Allow: */search/portfolio
86Disallow: */portfolio
87Disallow: */editor/template
88Disallow: */editor/design
89Disallow: */editor/search
90Disallow: */video/cart
91Disallow: */video/checkout
92Disallow: */music/cart
93Disallow: */music/user
94Disallow: */music/orders
95Disallow: /web/checkout
96Disallow: /sstk/api/footage/wasabi/*
97Disallow: /studioapi/wasabi/*
98Disallow: /sstk/api/*
99Disallow: /studioapi/*
100Disallow: /napi/*
101Disallow: /api/*
102Disallow: /feed.mhtml
103Disallow: /nuance-chat.html
104Disallow: /sw/*
105Disallow: /base/login
106Allow: */search/collections
107Allow: */editorial/collections
108Disallow: */collections
109Disallow: */generate/*
110Allow: */generate/$
111Disallow: /image-*chatId=*
112Disallow: /image-*tool=*
113Disallow: /image-*mode=*
114Disallow: /blog/?s=*
115Disallow: /cart
116Disallow: */search/ris/*
117Disallow: */search/models/*
118Disallow: /search/personalized/
119Disallow: /*/search/personalized/
120Disallow: /search/similar/
121Disallow: /*/search/similar/
122Disallow: */editor/image/*
123Allow: */editor/image/$
124
125User-agent: PerplexityBot
126Allow: */search/login
127Allow: */login$
128Allow: /login.mhtml$
129Disallow: */login
130Disallow: */base/logout
131Allow: */search/account
132Disallow: */account
133Disallow: /subscribe_success
134Allow: /download_history.mhtml$
135Disallow: /download
136Disallow: /checkout
137Disallow: /account_purchase_history.mhtml
138Disallow: /contests
139Allow: */search/portfolio
140Disallow: */portfolio
141Disallow: */editor/template
142Disallow: */editor/design
143Disallow: */editor/search
144Disallow: */video/cart
145Disallow: */video/checkout
146Disallow: */music/cart
147Disallow: */music/user
148Disallow: */music/orders
149Disallow: /web/checkout
150Disallow: /sstk/api/footage/wasabi/*
151Disallow: /studioapi/wasabi/*
152Disallow: /sstk/api/*
153Disallow: /studioapi/*
154Disallow: /napi/*
155Disallow: /api/*
156Disallow: /feed.mhtml
157Disallow: /nuance-chat.html
158Disallow: /sw/*
159Disallow: /base/login
160Allow: */search/collections
161Allow: */editorial/collections
162Disallow: */collections
163Disallow: */generate/*
164Allow: */generate/$
165Disallow: /image-*chatId=*
166Disallow: /image-*tool=*
167Disallow: /image-*mode=*
168Disallow: /blog/?s=*
169Disallow: /cart
170Disallow: */search/ris/*
171Disallow: */search/models/*
172Disallow: /search/personalized/
173Disallow: /*/search/personalized/
174Disallow: /search/similar/
175Disallow: /*/search/similar/
176Disallow: */editor/image/*
177Allow: */editor/image/$
178
179User-agent: Claude-SearchBot
180Allow: */search/login
181Allow: */login$
182Allow: /login.mhtml$
183Disallow: */login
184Disallow: */base/logout
185Allow: */search/account
186Disallow: */account
187Disallow: /subscribe_success
188Allow: /download_history.mhtml$
189Disallow: /download
190Disallow: /checkout
191Disallow: /account_purchase_history.mhtml
192Disallow: /contests
193Allow: */search/portfolio
194Disallow: */portfolio
195Disallow: */editor/template
196Disallow: */editor/design
197Disallow: */editor/search
198Disallow: */video/cart
199Disallow: */video/checkout
200Disallow: */music/cart
201Disallow: */music/user
202Disallow: */music/orders
203Disallow: /web/checkout
204Disallow: /sstk/api/footage/wasabi/*
205Disallow: /studioapi/wasabi/*
206Disallow: /sstk/api/*
207Disallow: /studioapi/*
208Disallow: /napi/*
209Disallow: /api/*
210Disallow: /feed.mhtml
211Disallow: /nuance-chat.html
212Disallow: /sw/*
213Disallow: /base/login
214Allow: */search/collections
215Allow: */editorial/collections
216Disallow: */collections
217Disallow: */generate/*
218Allow: */generate/$
219Disallow: /image-*chatId=*
220Disallow: /image-*tool=*
221Disallow: /image-*mode=*
222Disallow: /blog/?s=*
223Disallow: /cart
224Disallow: */search/ris/*
225Disallow: */search/models/*
226Disallow: /search/personalized/
227Disallow: /*/search/personalized/
228Disallow: /search/similar/
229Disallow: /*/search/similar/
230Disallow: */editor/image/*
231Allow: */editor/image/$
232
233User-agent: adidxbot
234Allow: *
235
236User-agent: bingbot
237Allow: */image-illustration/*.jpg
238Allow: */image-illustration/*.webp
239Allow: */image-photo/*.jpg
240Allow: */image-photo/*.webp
241Allow: */image-vector/*.jpg
242Allow: */image-vector/*.webp
243Allow: */search/login
244Allow: */login$
245Allow: /login.mhtml$
246Disallow: */login
247Disallow: */base/logout
248Allow: */search/account
249Disallow: */account
250Disallow: /subscribe_success
251Allow: /download_history.mhtml$
252Disallow: /download
253Disallow: /checkout
254Disallow: /account_purchase_history.mhtml
255Disallow: /contests
256Allow: */search/portfolio
257Disallow: */portfolio
258Disallow: */editor/template
259Disallow: */editor/design
260Disallow: */editor/search
261Disallow: */video/cart
262Disallow: */video/checkout
263Disallow: */music/cart
264Disallow: */music/user
265Disallow: */music/orders
266Disallow: /web/checkout
267Disallow: /sstk/api/footage/wasabi/*
268Disallow: /studioapi/wasabi/*
269Disallow: /sstk/api/*
270Disallow: /studioapi/*
271Disallow: /napi/*
272Disallow: /api/*
273Disallow: /feed.mhtml
274Disallow: /nuance-chat.html
275Disallow: /sw/*
276Disallow: /base/login
277Allow: */search/collections
278Allow: */editorial/collections
279Disallow: */collections
280Disallow: */generate/*
281Allow: */generate/$
282Disallow: */search/ris/*
283Disallow: */search/models/*
284Disallow: */editor/image/*
285Allow: */editor/image/$
286Disallow: /image-*chatId=*
287Disallow: /image-*tool=*
288Disallow: /image-*mode=*
289Disallow: /blog/?s=*
290Disallow: /cart
291Disallow: /sitemaps
292Disallow: /sitemaps/*
293
294User-agent: Yandex
295Allow: */image-illustration/*.jpg
296Allow: */image-illustration/*.webp
297Allow: */image-photo/*.jpg
298Allow: */image-photo/*.webp
299Allow: */image-vector/*.jpg
300Allow: */image-vector/*.webp
301Disallow: */image-editorial/*
302Disallow: */music/track-*
303Disallow: */image-illustration/*
304Disallow: */image-photo/*
305Disallow: */image-vector/*
306Disallow: */image-generated/*
307Disallow: */video/clip-*
308Allow: */search/login
309Allow: */login$
310Allow: /login.mhtml$
311Disallow: */login
312Disallow: */base/logout
313Allow: */search/account
314Disallow: */account
315Disallow: /subscribe_success
316Allow: /download_history.mhtml$
317Disallow: /download
318Disallow: /checkout
319Disallow: /account_purchase_history.mhtml
320Disallow: /contests
321Allow: */search/portfolio
322Disallow: */portfolio
323Disallow: */editor/template
324Disallow: */editor/design
325Disallow: */editor/search
326Disallow: */video/cart
327Disallow: */video/checkout
328Disallow: */music/cart
329Disallow: */music/user
330Disallow: */music/orders
331Disallow: /web/checkout
332Disallow: /sstk/api/footage/wasabi/*
333Disallow: /studioapi/wasabi/*
334Disallow: /sstk/api/*
335Disallow: /studioapi/*
336Disallow: /napi/*
337Disallow: /api/*
338Disallow: /feed.mhtml
339Disallow: /nuance-chat.html
340Disallow: /sw/*
341Disallow: /base/login
342Allow: */search/collections
343Allow: */editorial/collections
344Disallow: */collections
345Disallow: */generate/*
346Allow: */generate/$
347Disallow: */search/ris/*
348Disallow: */search/models/*
349Disallow: */editor/image/*
350Allow: */editor/image/$
351Disallow: /image-*chatId=*
352Disallow: /image-*tool=*
353Disallow: /image-*mode=*
354Disallow: /blog/?s=*
355Disallow: /cart
356
357Sitemap: https://www.shutterstock.com/sitemaps
358Sitemap: https://www.shutterstock.com/blog/sitemap_index.xml
359Sitemap: https://www.shutterstock.com/blog/india/sitemap_index.xml
360Sitemap: https://www.shutterstock.com/de/blog/sitemap_index.xml
361Sitemap: https://www.shutterstock.com/es/blog/sitemap_index.xml
362Sitemap: https://www.shutterstock.com/fr/blog/sitemap_index.xml
363Sitemap: https://www.shutterstock.com/ja/blog/sitemap_index.xml
364Sitemap: https://www.shutterstock.com/ko/blog/sitemap_index.xml
365Sitemap: https://www.shutterstock.com/th/blog/sitemap_index.xml
366Sitemap: https://www.shutterstock.com/pt/blog/sitemap_index.xml
367Sitemap: https://www.shutterstock.com/zh-Hant/blog/sitemap_index.xml