NeuralCrawl

Semrush / robots.txt snapshot

← back to semrush.com · fetched 2026-07-16T11:40:45Z (21d ago) · HTTP 200 · 3617 bytes · sha256 4644f05541a3e5d6 · raw

final URL: https://www.semrush.com/robots.txt

1User-agent: 008
2Disallow: /
3
4User-agent: SiteAuditBot
5Crawl-delay: 1
6Allow: /
7
8User-agent: Semrushbot-SI
9Allow: /
10
11User-agent: Yahoo Pipes 2.0
12Disallow: /
13
14User-agent: Facebot
15Disallow: /archive/graphs.php
16Allow: /academy/certificates/*
17
18User-agent: facebookexternalhit
19Allow: /academy/certificates/*
20
21User-agent: Bingbot
22Disallow: /archive/graphs.php
23
24User-agent: BingPreview
25Disallow: /archive/graphs.php
26
27User-agent: LinkedInBot
28Allow: /academy/certificates/*
29
30User-agent: Twitterbot
31Allow: /blog/*
32Allow: /ranking-factors/*
33Allow: /academy/certificates/*
34
35User-agent: *
36Disallow: /blog/*?img=*
37Disallow: /archive/graphs.php
38Disallow: /forrester-marketing-silos/*
39Disallow: /partner/
40Disallow: /blog/search/*
41Disallow: /analytics/seomagic/
42Disallow: /my_reports/reports/*
43Disallow: /archive/
44Disallow: /admin/
45Disallow: /users/
46Disallow: /clients/
47Disallow: /custom_report/
48Disallow: /limit_hits/
49Disallow: /payment/
50Disallow: /no_cookies/
51Disallow: /double_activation/
52Disallow: /projects/?*
53Disallow: /?*/
54Disallow: /activation_successful*
55Disallow: /api.html?*
56Disallow: /partners/
57Disallow: /partner/
58Disallow: */dvd-mode/
59Disallow: /unsubscribe/
60Disallow: /unsubscribe/*
61Disallow: /demonstration/
62Disallow: /demonstration/*
63Allow: /projects/
64Disallow: */custom-report/
65Disallow: */request-demo/
66Allow: */info/kdt/
67Disallow: */pleasereg/
68Disallow: */custom-report/
69Disallow: */custom-report
70Allow: */sensor/*
71Disallow: /swa/api/*
72Disallow: /redirect?url=*
73Disallow: /sso/
74Disallow: /_/empty_blank/
75Disallow: /_/toolkit_blank/
76Disallow: /_/site_info_blank/
77Disallow: /seo-old/*
78Disallow: /projects2/*
79Disallow: /free-tools/plagiarism-checker/?q=
80Allow: /news/*?page=*
81
82# Community rules
83Allow: /blog/*
84Disallow: /blog/search/*
85Disallow: /blog/*utm_source=*
86Disallow: /blog-manager/*
87Allow: /user/*
88
89# Disallow technical signup pages
90Allow: /login/$
91Allow: /signup/$
92Disallow: /login/*
93Disallow: /signup/*
94
95# Solutions
96Disallow: /solutions/$
97
98# Enterprise
99Allow: /enterprise/seo/
100
101#landing
102Disallow: /landing/
103Disallow: /lp/
104Allow: /lp/affiliate-program/*
105Allow: /lp/for-agencies/en/
106Allow: /lp/cannes2026/*
107Allow: /lp/enterprise-aio/*
108Allow: /lp/enterprise-crawler/*
109Allow: /lp/enterprise-partner-program/*
110Allow: /lp/seasonal-ai/en/
111Allow: /lp/black-friday/*
112Disallow: /lp/black-friday-aff/*
113Allow: /lp/cyber-week/*
114Allow: /lp/journeys/*
115Allow: /lp/insights/*
116Allow: /lp/growth-quadrant/
117Allow: /lp/mfour/*
118Allow: /lp/semrush-circle/*
119Allow: /lp/semrush-one/*
120Allow: /seo/
121Allow: /lp/seo-consultation/*
122Allow: /content-marketing/
123Allow: /ppc/
124Allow: /competitive-research/
125Allow: /analytics/traffic/overview/
126Disallow: /listings-management/superpower-landing2/
127
128#academy
129Allow: /academy/*
130Disallow: /academy/*utm_source=*
131Disallow: /academy/*/results
132Disallow: /academy/certificates/*
133
134#websites
135Allow: /website/*
136Disallow: /website/search/*
137
138Disallow: /*/?*
139Crawl-delay: 20
140
141# Sitemap files
142#Blog
143Sitemap: https://www.semrush.com/sitemap.xml
144Sitemap: https://pt.semrush.com/blog/sitemap/
145Sitemap: https://es.semrush.com/blog/sitemap/
146Sitemap: https://it.semrush.com/blog/sitemap/
147Sitemap: https://fr.semrush.com/blog/sitemap/
148Sitemap: https://de.semrush.com/blog/sitemap/
149Sitemap: https://ja.semrush.com/blog/sitemap/
150#Academy
151Sitemap: https://www.semrush.com/academy/sitemap.xml
152Sitemap: https://es.semrush.com/academy/sitemap.xml
153Sitemap: https://pt.semrush.com/academy/sitemap.xml
154Sitemap: https://it.semrush.com/academy/sitemap.xml
155Sitemap: https://fr.semrush.com/academy/sitemap.xml
156Sitemap: https://de.semrush.com/academy/sitemap.xml
157#Free tools:
158Sitemap: https://www.semrush.com/free-tools/sitemap.xml