GitLab / robots.txt snapshot

← back to gitlab.com · fetched 2026-06-20T01:10:31Z (18h ago) · HTTP 200 · 3230 bytes · sha256 9e1e837e03b726ee · raw

final URL: https://gitlab.com/robots.txt

1	# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
2	#
3	# To ban all spiders from the entire site uncomment the next two lines:
4	# User-Agent: *
5	# Disallow: /
6
7	# Add a 1 second delay between successive requests to the same server, limits resources used by crawler
8	# Only some crawlers respect this setting, e.g. Googlebot does not
9	# Crawl-delay: 1
10
11	# Based on details in https://gitlab.com/gitlab-org/gitlab/blob/master/config/routes.rb,
12	# https://gitlab.com/gitlab-org/gitlab/blob/master/spec/routing, and using application
13
14	# Global routes
15	User-Agent: *
16	Disallow: /autocomplete/users
17	Disallow: /autocomplete/projects
18	Disallow: /search
19	Disallow: /admin
20	Disallow: /profile
21	Disallow: /dashboard
22	Disallow: /users
23	Disallow: /api/v*
24	Disallow: /help
25	Disallow: /s/
26	Disallow: /-/profile
27	Disallow: /-/profile/
28	Disallow: /-/user_settings/
29	Disallow: /-/ide/
30	Disallow: /-/experiment
31	# Restrict allowed routes to avoid very ugly search results
32	Allow: /users/sign_in
33	Allow: /users/sign_up
34	Allow: /users/*/snippets
35
36	# Generic resource routes like new, edit, raw
37	# This will block routes like:
38	# - /projects/new
39	# - /gitlab-org/gitlab-foss/issues/123/-/edit
40	User-Agent: *
41	Disallow: /*/new
42	Disallow: /*/edit
43	Disallow: /*/raw
44	Disallow: /*/realtime_changes
45
46	# Group details
47	User-Agent: *
48	Disallow: /groups/*/-/analytics
49	Disallow: /groups/*/-/analytics/
50	Disallow: /groups/*/-/insights/
51	Disallow: /groups/*/-/issues_analytics
52	Disallow: /groups/*/-/contribution_analytics
53	Disallow: /groups/*/-/group_members
54	Disallow: /groups/*/-/saml/
55	Disallow: /groups/*/-/saml_group_links
56	Disallow: /groups/*/-/settings/
57	Disallow: /groups/*/-/billings
58	Disallow: /groups/*/-/hooks
59	Disallow: /groups/*/-/projects
60
61	# Project details
62	User-Agent: *
63	Disallow: //.git$
64	Disallow: //.git/*
65	Disallow: /*/archive/
66	Disallow: //repository/archive
67	Disallow: /*/activity
68	Disallow: /*/-/project_members
69	Disallow: /*/-/blame/
70	Disallow: /*/-/branches
71	Disallow: /*/-/commits/
72	Disallow: /*/-/commit
73	Disallow: //commit/.patch
74	Disallow: //commit/.diff
75	Disallow: /*/-/compare/
76	Disallow: /*/-/network/
77	Disallow: /*/path_locks
78	Disallow: //merge_requests/.patch
79	Disallow: //merge_requests/.diff
80	Disallow: //merge_requests//diffs
81	Disallow: /*/services
82	Disallow: /*/uploads/
83	Disallow: /*/-/import
84	Disallow: /*/-/requirements_management/
85	Disallow: /*/-/pipelines
86	Disallow: /*/-/pipeline_schedules
87	Disallow: /*/-/jobs
88	Disallow: /*/-/ci/
89	Disallow: /*/-/quality/
90	Disallow: /*/-/licenses
91	Disallow: /*/-/security/
92	Disallow: /*/-/dependencies
93	Disallow: /*/-/audit_events
94	Disallow: /*/-/on_demand_scans
95	Disallow: /*/-/feature_flags
96	Disallow: /*/-/ml/
97	Disallow: /*/-/environments
98	Disallow: /*/-/clusters
99	Disallow: /*/-/terraform
100	Disallow: /*/-/terraform_module_registry
101	Disallow: //-//configuration
102	Disallow: /*/-/error_tracking
103	Disallow: /*/-/metrics
104	Disallow: /*/-/alert_management
105	Disallow: /*/-/incidents
106	Disallow: /*/-/oncall_schedules
107	Disallow: /*/-/escalation_policies
108	Disallow: //-//service_desk
109	Disallow: /*/-/analytics
110	Disallow: /*/-/analytics/
111	Disallow: /*/-/value_stream_analytics
112	Disallow: /*/-/graphs/
113	Disallow: /*/insights/
114	Disallow: /*/-/pipelines/
115	Disallow: /*/-/settings/
116	Disallow: /*/-/hooks
117	Disallow: /*/-/usage_quotas