NeuralCrawl

MongoDB / robots.txt snapshot

← back to mongodb.com · fetched 2026-06-20T01:10:31Z (15h ago) · HTTP 200 · 2823 bytes · sha256 817d348c26b8dd8d · raw

final URL: https://www.mongodb.com/robots.txt

1# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
2
3User-agent: mauibot
4Disallow: /community/forums/
5
6User-agent: semrushbot
7Disallow: /community/forums/
8
9User-agent: ahrefsbot
10Disallow: /community/forums/
11
12User-agent: blexbot
13Disallow: /community/forums/
14
15User-agent: seo spider
16Disallow: /community/forums/
17
18User-agent: bingbot
19Disallow: /community/forums/auth/
20Disallow: /community/forums/assets/browser-update*.js
21Disallow: /community/forums/email/
22Disallow: /community/forums/session
23Disallow: /community/forums/session/
24Disallow: /community/forums/admin
25Disallow: /community/forums/admin/
26Disallow: /community/forums/user-api-key
27Disallow: /community/forums/user-api-key/
28Disallow: /community/forums/*?api_key*
29Disallow: /community/forums/*?*api_key*
30
31User-agent: *
32# Directories
33Disallow: /misc/
34Disallow: /profiles/
35Disallow: /u/
36Disallow: /community/forums/auth/
37Disallow: /community/forums/email/
38Disallow: /community/forums/session
39Disallow: /community/forums/session/
40Disallow: /community/forums/admin
41Disallow: /community/forums/admin/
42Disallow: /community/forums/user-api-key
43Disallow: /community/forums/user-api-key/
44Disallow: /community/forums/g/
45Disallow: /community/forums/u/
46Disallow: /community/forums/user_avatar/
47
48# Files
49Disallow: /admin/*
50Disallow: /data/*
51Disallow: /comment/reply/
52Disallow: /filter/tips/
53Disallow: /node/add/
54Disallow: /search/
55Disallow: /search
56Disallow: /user/register/
57Disallow: /user/password/
58Disallow: /user/login/
59Disallow: /user/logout/
60Disallow: /lp/p/*
61Disallow: /lp/s/*
62Disallow: /lp/d/*
63Disallow: /thank-you/*
64Disallow: /link/*
65Disallow: /community/forums/assets/browser-update*.js
66Disallow: /docs/*.epub
67Disallow: /docs/*.pdf
68Disallow: /docs/atlas/app-services/
69
70# No access for quicktabs in the URL
71Disallow: /*?quicktabs_*
72Disallow: /*&quicktabs_*
73Disallow: /*?qt-view_*
74Disallow: /*&qt-view_*
75# No crawling field collections or taxonomy
76Disallow: /field-collection/*
77Disallow: /tags/*
78Disallow: /taxonomy/term/*
79# Paths (no clean URLs)
80Disallow: /?q=admin/
81Disallow: /?q=comment/reply/
82Disallow: /?q=filter/tips/
83Disallow: /?q=node/add/
84Disallow: /?q=search/
85Disallow: /?q=user/password/
86Disallow: /?q=user/register/
87Disallow: /?q=user/login/
88Disallow: /?q=user/logout/
89# Specific pages
90Disallow: /commercialsupport/
91Disallow: /partners/referral-federal
92Disallow: /customer-evaluation-downloads-development-versions
93Disallow: /community/forums/*?api_key*
94Disallow: /community/forums/*?*api_key*
95Disallow: /docs/meta/
96Disallow: /docs/search/
97Disallow: /docs/openapi/
98
99# Prevent exposing future docs
100Disallow: /docs-qa*
101
102# Prevent exposing devhub UAT
103Disallow: /devcenter-uat*
104
105Disallow: /lp/internal/cheq-ppc-invalid-users*
106
107Disallow: /.well-known/security.txt
108
109Sitemap: https://www.mongodb.com/sitemap-index.xml