NeuralCrawl

Pond5 / robots.txt snapshot

← back to pond5.com · fetched 2026-06-23T12:35:20Z (1mo ago) · HTTP 200 · 2187 bytes · sha256 236cb8eefc18526f · raw

final URL: manual:file

1User-agent: *
2
3Disallow: *page=artists
4Disallow: /*/mightlike
5Disallow: /artist-directory
6Disallow: *membership=1
7Disallow: *page=thundercloud
8Disallow: *page=lightbox_terms
9
10Disallow: /ajax/recent/items/add
11Disallow: /ajax/user/*
12Disallow: /ajax/user-account/*
13Disallow: /ajax/collections-bar
14Disallow: /ajax/dashboard/*
15Disallow: /ajax/cart/*
16Disallow: /ajax/bin/*
17Disallow: /ajax/item/*
18Disallow: /ajax/search-filters/*
19Disallow: /ajax/translation*
20Disallow: /ajax/search/suggestion*
21
22Disallow: /*/ajax/recent/items/add
23Disallow: /*/ajax/user/*
24Disallow: /*/ajax/user-account/*
25Disallow: /*/ajax/collections-bar
26Disallow: /*/ajax/dashboard/*
27Disallow: /*/ajax/cart/*
28Disallow: /*/ajax/bin/*
29Disallow: /*/ajax/item/*
30Disallow: /*/ajax/search-filters/*
31Disallow: /*/ajax/translation*
32
33Disallow: /stock-images/photos/item/
34Disallow: /stock-images/illustrations/item/
35Disallow: /stock-images/psd/item/
36Disallow: /*/stock-images/photos/item/
37Disallow: /*/stock-images/illustrations/item/
38Disallow: /*/stock-images/psd/item/
39
40User-agent: Slurp
41Crawl-delay: 1
42
43User-agent: rogerbot
44Disallow: /
45
46user-agent: sitebot
47disallow: /
48
49user-agent: coccoc
50disallow: /
51
52User-agent: facebookexternalhit/*
53Crawl-delay: 1
54
55User-agent: facebookexternalhit/1.1
56Crawl-delay: 1
57
58User-agent: facebookexternalhit*
59Allow: *
60
61User-agent: baiduspider
62Crawl-delay: 1
63
64User-agent: JikeSpider
65Disallow: /
66
67User-Agent: twengabot
68Disallow: /
69
70User-agent: Blekkobot
71Crawl-delay: 2
72
73User-agent: Linguee
74Disallow: /
75
76User-agent: sistrix
77Disallow: /
78
79User-agent: YioopBot
80Disallow: /
81
82User-agent: BLEXBot
83Disallow: /
84
85User-agent: CCBot
86Disallow: /
87
88User-agent: GPTBot
89Disallow: /
90
91User-agent: meta-externalagent
92Disallow: /
93
94Sitemap: https://sitemap.pond5.com/en-sitemaps-index-static-pages-1.xml.gz
95Sitemap: https://sitemap.pond5.com/en-sitemaps-index-artists-1.xml.gz
96Sitemap: https://sitemap.pond5.com/en-sitemaps-index-collections-1.xml.gz
97Sitemap: https://sitemap.pond5.com/en-sitemaps-index-searches-1.xml.gz
98Sitemap: https://sitemap.pond5.com/en-sitemaps-index-pd-1.xml.gz
99Sitemap: https://sitemap.pond5.com/en-sitemaps-index-detail-pages-1.xml.gz
100Sitemap: https://sitemap.pond5.com/categories-sitemap-index.xml.gz