NeuralCrawl

Scribd / robots.txt snapshot

← back to scribd.com · fetched 2026-06-20T11:49:23Z (7h ago) · HTTP 200 · 3546 bytes · sha256 3467b2a622d186e4 · raw

final URL: manual:file

1User-agent: *
2Disallow: /*/d/
3Disallow: /*/followers
4Disallow: /*/following
5Disallow: /*/pingback$
6Disallow: /*/info$
7Disallow: /*/subscribers$
8Disallow: /*/friends$
9Disallow: /*/*/data$
10Disallow: /*?format=json
11Disallow: /account/
12Disallow: /account-settings
13Disallow: /analytics/
14Disallow: /api?
15Disallow: /api/
16Disallow: /archive/
17Disallow: /blocks?
18Disallow: /blocks/
19Disallow: /book_annotations/
20Disallow: /book-preview/*/recommenders
21Disallow: /book-preview/*/shadow_loader
22Disallow: /cpds2/
23Disallow: /cpspa/
24Disallow: /csrf_token
25Disallow: /doc-page/animated-hero-banner-recommender-props
26Disallow: /doc-page/download-receipt-modal-props/
27Disallow: /doc-page/embed-modal-props/
28Disallow: /doc-page/exclusive-free-trial-props/
29Disallow: /doc-page/flag-document-modal-props/
30Disallow: /doc-page/paused-user-download-modal-props
31Disallow: /doc-page/pmp-login-join-modal-props/
32Disallow: /doc-page/recommenders/
33Disallow: /doc-page/seo-roadblock-props/
34Disallow: /doc_landing
35Disallow: /doc/protected/
36Disallow: /docinfo/
37Disallow: /document_auxillary_content/
38Disallow: /document_downloads/
39Disallow: /documents/*/comments?
40Disallow: /documents/search
41Disallow: /embeds/
42Disallow: /errors/browser-unsupported
43Disallow: /events/
44Disallow: /explorespa/
45Disallow: /feeds/user_rss/
46Disallow: /fullscreen/
47Disallow: /full/
48Disallow: /home
49Disallow: /job_game/
50Disallow: /jobs/botrace_bot/
51Disallow: /jobs/botrace_matches/
52Disallow: /jobs/botrace_my_bots
53Disallow: /jobs/botrace_rankings
54Disallow: /jobs/botrace_upload_bot
55Disallow: /language?
56Disallow: /listen/
57Disallow: /listen_spa/ # TODO: Remove this after the migration is complete
58Disallow: /audiobooks_spa/ # TODO: Remove this after the migration is complete
59Disallow: /login?
60Disallow: /lp-spa
61Disallow: /next/
62Disallow: /notifications/
63Disallow: /oauth/login
64Disallow: /oauth/signup
65Disallow: /options/exposed_to_client.js
66Disallow: /payments/
67Disallow: /profiles/content.json
68Disallow: /profiles/more_reviews
69Disallow: /profiles/timeline?
70Disallow: /promo_code
71Disallow: /ratings.json
72Disallow: /ratings$
73Disallow: /read/
74Disallow: /sbd/
75Disallow: /search-4gen
76Disallow: /search-collections?
77Disallow: /search-people?
78Disallow: /search?
79Disallow: /sectionized/
80Disallow: /ssi/geoip_cc
81Disallow: /store_purchase
82Disallow: /upload-document?
83Disallow: /upload/auto
84Disallow: /upload/throttled
85Disallow: /upload/upload_progress/
86Disallow: /user_flags/
87Disallow: /user-document-collections/
88Disallow: /viewer/
89Disallow: /word/access_denied/
90Disallow: /word/add_to_favorites/
91Disallow: /word/comment_login/
92Disallow: /word/document_edit/
93Disallow: /word/document_stats/
94Disallow: /word/document_url/
95Disallow: /word/download/
96Disallow: /word/embed/
97Disallow: /your-account
98
99User-agent: Bytespider
100Disallow: /
101User-agent: CCBot
102Disallow: /
103User-agent: Claude-Web
104Disallow: /
105User-agent: ClaudeBot
106Disallow: /
107User-agent: Diffbot
108Disallow: /
109User-agent: ImagesiftBot
110Disallow: /
111User-agent: omgili
112Disallow: /
113User-agent: AI2Bot
114Disallow: /
115
116Sitemap: https://sitemaps.scribd.com/docs_de_index.xml
117Sitemap: https://sitemaps.scribd.com/docs_en_index.xml
118Sitemap: https://sitemaps.scribd.com/docs_es_index.xml
119Sitemap: https://sitemaps.scribd.com/docs_fr_index.xml
120Sitemap: https://sitemaps.scribd.com/docs_id_index.xml
121Sitemap: https://sitemaps.scribd.com/docs_other_index.xml
122Sitemap: https://sitemaps.scribd.com/docs_pt_index.xml
123Sitemap: https://sitemaps.scribd.com/docs_ro_index.xml
124Sitemap: https://sitemaps.scribd.com/docs_ru_index.xml
125
126Sitemap: https://sitemaps.scribd.com/users_index.xml