NeuralCrawl

Discogs / robots.txt snapshot

← back to discogs.com · fetched 2026-06-27T10:56:08Z (1mo ago) · HTTP 200 · 3232 bytes · sha256 4ffa78a87bc3217e · raw

final URL: manual:file

1User-agent: *
2Allow: /ads.txt
3Allow: /*/_graphs/history
4Disallow: */sell/cart
5Disallow: */sell/post
6Disallow: */sell/order/
7Disallow: */sell/orders
8Disallow: */sell/mplistrss
9Disallow: /submissions
10Disallow: /*/submissions
11Disallow: /submission
12Disallow: */contributions
13Disallow: /my$
14Disallow: /sell/mywants$
15Disallow: /shop/mywants$
16Disallow: /browse/my$
17Disallow: /recent/my$
18Disallow: */messages/
19Disallow: /mygroups$
20Disallow: /mywantlist$
21Disallow: /mycontributions$
22Disallow: /mycollection$
23Disallow: /marketplace/mywants$
24Disallow: */release/add
25Disallow: /release/edit
26Disallow: /release/copy
27Disallow: /update
28Disallow: /*/update$
29Disallow: /login
30Disallow: /users/
31Disallow: /logout
32Disallow: /redir
33Disallow: /history
34Disallow: /*/history
35Disallow: /*/images$
36Disallow: */wantlist$
37Disallow: */wantlist?user=
38Disallow: */report
39Disallow: /rss
40Disallow: /groups/report
41Disallow: /artist2
42Disallow: /sell/unlist
43Disallow: /release/recs
44Disallow: /projects
45Disallow: /data$
46Disallow: /pt_BR/sell/item
47Disallow: /pt_BR/sell/list?
48Disallow: /pt_BR/seller/
49Disallow: /ko/sell/item
50Disallow: /ko/sell/list?
51Disallow: /ko/seller/
52Disallow: /master/stats/
53Disallow: /release/stats/
54
55User-agent: AhrefsBot
56Disallow: /
57User-agent: Applebot
58Crawl-delay: 2
59
60User-agent: GoogleOther
61Disallow: */sell/list
62Disallow: */mplistrss
63Disallow: */marketplace
64Disallow: */seller/*
65Disallow: */sell/cart
66Disallow: */sell/post
67Disallow: */sell/order/
68Disallow: */sell/orders
69Disallow: */sell/mplistrss
70Disallow: /submissions
71Disallow: /*/submissions
72Disallow: /submission
73Disallow: */contributions
74Disallow: /my$
75Disallow: /sell/mywants$
76Disallow: /shop/mywants$
77Disallow: /browse/my$
78Disallow: /recent/my$
79Disallow: */messages/
80Disallow: /mygroups$
81Disallow: /mywantlist$
82Disallow: /mycontributions$
83Disallow: /mycollection$
84Disallow: /marketplace/mywants$
85Disallow: */release/add
86Disallow: /release/edit
87Disallow: /release/copy
88Disallow: /update
89Disallow: /*/update$
90Disallow: /login
91Disallow: */login
92Disallow: /users/
93Disallow: /logout
94Disallow: /redir
95Disallow: /history
96Disallow: /*/history
97Disallow: /*/images$
98Disallow: */wantlist$
99Disallow: */wantlist?user=
100Disallow: */report
101Disallow: /rss
102Disallow: /groups/report
103Disallow: /artist2
104Disallow: /sell/unlist
105Disallow: /release/recs
106Disallow: /projects
107Disallow: /data$
108Disallow: /pt_BR/sell/item
109Disallow: /ko/sell/item
110Disallow: /master/stats/
111Disallow: /release/stats/
112
113User-agent: Sogou web spider
114Disallow: /search
115Disallow: /es/search
116Disallow: /de/search
117Disallow: /fr/search
118Disallow: /it/search
119Disallow: /ja/search
120Disallow: /ko/search
121Disallow: /pt_BR/search
122Disallow: /ru/search
123
124Sitemap: https://www.discogs.com/sitemap_artist.xml
125Sitemap: https://www.discogs.com/sitemap_release.xml
126Sitemap: https://www.discogs.com/sitemap_master_release.xml
127Sitemap: https://www.discogs.com/sitemap_footer.xml
128Sitemap: https://www.discogs.com/sitemap_mp_master.xml
129Sitemap: https://www.discogs.com/company/sitemap.xml
130Sitemap: https://www.discogs.com/digs/sitemap.xml
131Sitemap: https://www.discogs.com/selling/sitemap.xml
132Sitemap: https://www.discogs.com/record-stores/sitemap.xml
133Sitemap: https://www.discogs.com/media-kit/sitemap.xml
134Sitemap: https://www.discogs.com/music/sitemap.xml