NeuralCrawl

Semrush / robots.txt snapshot

← back to semrush.com · fetched 2026-06-20T14:56:27Z (7h ago) · HTTP 200 · 3693 bytes · sha256 6be6a3b2ccc3896a · raw

final URL: https://www.semrush.com/robots.txt

1User-agent: 008
2Disallow: /
3
4User-agent: SiteAuditBot
5Crawl-delay: 1
6Allow: /
7
8User-agent: Semrushbot-SI
9Allow: /
10
11User-agent: Yahoo Pipes 2.0
12Disallow: /
13
14User-agent: Facebot
15Disallow: /archive/graphs.php
16Allow: /academy/certificates/*
17
18User-agent: facebookexternalhit
19Allow: /academy/certificates/*
20
21User-agent: Bingbot
22Disallow: /archive/graphs.php
23
24User-agent: BingPreview
25Disallow: /archive/graphs.php
26
27User-agent: LinkedInBot
28Allow: /academy/certificates/*
29
30User-agent: Twitterbot
31Allow: /blog/*
32Allow: /ranking-factors/*
33Allow: /academy/certificates/*
34
35User-agent: *
36Disallow: /blog/*?img=*
37Disallow: /archive/graphs.php
38Disallow: /forrester-marketing-silos/*
39Disallow: /partner/
40Disallow: /blog/search/*
41Disallow: /analytics/seomagic/
42Disallow: /my_reports/reports/*
43Disallow: /archive/
44Disallow: /admin/
45Disallow: /users/
46Disallow: /clients/
47Disallow: /custom_report/
48Disallow: /limit_hits/
49Disallow: /payment/
50Disallow: /no_cookies/
51Disallow: /double_activation/
52Disallow: /projects/?*
53Disallow: /?*/
54Disallow: /activation_successful*
55Disallow: /api.html?*
56Disallow: /partners/
57Disallow: /partner/
58Disallow: */dvd-mode/
59Disallow: /unsubscribe/
60Disallow: /unsubscribe/*
61Disallow: /demonstration/
62Disallow: /demonstration/*
63Allow: /projects/
64Disallow: */custom-report/
65Disallow: */request-demo/
66Allow: */info/kdt/
67Disallow: */pleasereg/
68Disallow: */custom-report/
69Disallow: */custom-report
70Allow: */sensor/*
71Disallow: /swa/api/*
72Disallow: /redirect?url=*
73Disallow: /sso/
74Disallow: /_/empty_blank/
75Disallow: /_/toolkit_blank/
76Disallow: /_/site_info_blank/
77Disallow: /seo-old/*
78Disallow: /projects2/*
79Disallow: /free-tools/plagiarism-checker/?q=
80Allow: /news/*?page=*
81
82# Community rules
83Allow: /blog/*
84Disallow: /blog/search/*
85Disallow: /blog/*utm_source=*
86Disallow: /blog-manager/*
87Allow: /user/*
88
89# Features new pages
90Disallow: /features/index2/
91Disallow: /features/for-*
92
93# Disallow technical signup pages
94Allow: /login/$
95Allow: /signup/$
96Disallow: /login/*
97Disallow: /signup/*
98
99# Solutions
100Disallow: /solutions/$
101
102# Enterprise
103Allow: /enterprise/seo/
104
105#landing
106Disallow: /landing/
107Disallow: /lp/
108Allow: /lp/affiliate-program/*
109Allow: /lp/for-agencies/en/
110Allow: /lp/cannes2026/*
111Allow: /lp/enterprise-aio/*
112Allow: /lp/enterprise-crawler/*
113Allow: /lp/enterprise-partner-program/*
114Allow: /lp/seasonal-ai/en/
115Allow: /lp/black-friday/*
116Disallow: /lp/black-friday-aff/*
117Allow: /lp/cyber-week/*
118Allow: /lp/journeys/*
119Allow: /lp/insights/*
120Allow: /lp/growth-quadrant/
121Allow: /lp/mfour/*
122Allow: /lp/semrush-circle/*
123Allow: /lp/semrush-one/*
124Allow: /seo/
125Allow: /lp/seo-consultation/*
126Allow: /content-marketing/
127Allow: /ppc/
128Allow: /competitive-research/
129Allow: /analytics/traffic/overview/
130Disallow: /listings-management/superpower-landing2/
131
132#academy
133Allow: /academy/*
134Disallow: /academy/*utm_source=*
135Disallow: /academy/*/results
136Disallow: /academy/certificates/*
137
138#websites
139Allow: /website/*
140Disallow: /website/search/*
141
142Disallow: /*/?*
143Crawl-delay: 20
144
145# Sitemap files
146#Blog
147Sitemap: https://www.semrush.com/sitemap.xml
148Sitemap: https://pt.semrush.com/blog/sitemap/
149Sitemap: https://es.semrush.com/blog/sitemap/
150Sitemap: https://it.semrush.com/blog/sitemap/
151Sitemap: https://fr.semrush.com/blog/sitemap/
152Sitemap: https://de.semrush.com/blog/sitemap/
153Sitemap: https://ja.semrush.com/blog/sitemap/
154#Academy
155Sitemap: https://www.semrush.com/academy/sitemap.xml
156Sitemap: https://es.semrush.com/academy/sitemap.xml
157Sitemap: https://pt.semrush.com/academy/sitemap.xml
158Sitemap: https://it.semrush.com/academy/sitemap.xml
159Sitemap: https://fr.semrush.com/academy/sitemap.xml
160Sitemap: https://de.semrush.com/academy/sitemap.xml
161#Free tools:
162Sitemap: https://www.semrush.com/free-tools/sitemap.xml