NeuralCrawl

MongoDB / robots.txt snapshot

← back to mongodb.com · fetched 2026-07-08T20:50:19Z (29d ago) · HTTP 200 · 2859 bytes · sha256 79b3e7543ec8c0e5 · raw

final URL: https://www.mongodb.com/robots.txt

1# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
2
3User-agent: mauibot
4Disallow: /community/forums/
5
6User-agent: semrushbot
7Disallow: /community/forums/
8
9User-agent: ahrefsbot
10Disallow: /community/forums/
11
12User-agent: blexbot
13Disallow: /community/forums/
14
15User-agent: seo spider
16Disallow: /community/forums/
17
18User-agent: bingbot
19Disallow: /community/forums/auth/
20Disallow: /community/forums/assets/browser-update*.js
21Disallow: /community/forums/email/
22Disallow: /community/forums/session
23Disallow: /community/forums/session/
24Disallow: /community/forums/admin
25Disallow: /community/forums/admin/
26Disallow: /community/forums/user-api-key
27Disallow: /community/forums/user-api-key/
28Disallow: /community/forums/*?api_key*
29Disallow: /community/forums/*?*api_key*
30
31User-agent: *
32# Directories
33Disallow: /company/blog/search
34Disallow: /misc/
35Disallow: /profiles/
36Disallow: /u/
37Disallow: /community/forums/auth/
38Disallow: /community/forums/email/
39Disallow: /community/forums/session
40Disallow: /community/forums/session/
41Disallow: /community/forums/admin
42Disallow: /community/forums/admin/
43Disallow: /community/forums/user-api-key
44Disallow: /community/forums/user-api-key/
45Disallow: /community/forums/g/
46Disallow: /community/forums/u/
47Disallow: /community/forums/user_avatar/
48
49# Files
50Disallow: /admin/*
51Disallow: /data/*
52Disallow: /comment/reply/
53Disallow: /filter/tips/
54Disallow: /node/add/
55Disallow: /search/
56Disallow: /search
57Disallow: /user/register/
58Disallow: /user/password/
59Disallow: /user/login/
60Disallow: /user/logout/
61Disallow: /lp/p/*
62Disallow: /lp/s/*
63Disallow: /lp/d/*
64Disallow: /thank-you/*
65Disallow: /link/*
66Disallow: /community/forums/assets/browser-update*.js
67Disallow: /docs/*.epub
68Disallow: /docs/*.pdf
69Disallow: /docs/atlas/app-services/
70
71# No access for quicktabs in the URL
72Disallow: /*?quicktabs_*
73Disallow: /*&quicktabs_*
74Disallow: /*?qt-view_*
75Disallow: /*&qt-view_*
76# No crawling field collections or taxonomy
77Disallow: /field-collection/*
78Disallow: /tags/*
79Disallow: /taxonomy/term/*
80# Paths (no clean URLs)
81Disallow: /?q=admin/
82Disallow: /?q=comment/reply/
83Disallow: /?q=filter/tips/
84Disallow: /?q=node/add/
85Disallow: /?q=search/
86Disallow: /?q=user/password/
87Disallow: /?q=user/register/
88Disallow: /?q=user/login/
89Disallow: /?q=user/logout/
90# Specific pages
91Disallow: /commercialsupport/
92Disallow: /partners/referral-federal
93Disallow: /customer-evaluation-downloads-development-versions
94Disallow: /community/forums/*?api_key*
95Disallow: /community/forums/*?*api_key*
96Disallow: /docs/meta/
97Disallow: /docs/site-search/
98Disallow: /docs/openapi/
99
100# Prevent exposing future docs
101Disallow: /docs-qa*
102
103# Prevent exposing devhub UAT
104Disallow: /devcenter-uat*
105
106Disallow: /lp/internal/cheq-ppc-invalid-users*
107
108Disallow: /.well-known/security.txt
109
110Sitemap: https://www.mongodb.com/sitemap-index.xml