NeuralCrawl

Semrush / robots.txt snapshot

← back to semrush.com · fetched 2026-09-28T18:34:49Z (1mo ago) · HTTP 200 · 3599 bytes · sha256 65bac391e0e7a745 · raw

final URL: https://www.semrush.com/robots.txt

1User-agent: 008
2Disallow: /
3
4User-agent: SiteAuditBot
5Crawl-delay: 1
6Allow: /
7
8User-agent: Semrushbot-SI
9Allow: /
10
11User-agent: Yahoo Pipes 2.0
12Disallow: /
13
14User-agent: Facebot
15Disallow: /archive/graphs.php
16Allow: /academy/certificates/*
17
18User-agent: facebookexternalhit
19Allow: /academy/certificates/*
20
21User-agent: Bingbot
22Disallow: /archive/graphs.php
23
24User-agent: BingPreview
25Disallow: /archive/graphs.php
26
27User-agent: LinkedInBot
28Allow: /academy/certificates/*
29
30User-agent: Twitterbot
31Allow: /blog/*
32Allow: /ranking-factors/*
33Allow: /academy/certificates/*
34
35User-agent: *
36Disallow: /blog/*?img=*
37Disallow: /archive/graphs.php
38Disallow: /forrester-marketing-silos/*
39Disallow: /partner/
40Disallow: /blog/search/*
41Disallow: /analytics/seomagic/
42Disallow: /my_reports/reports/*
43Disallow: /archive/
44Disallow: /admin/
45Disallow: /users/
46Disallow: /clients/
47Disallow: /custom_report/
48Disallow: /limit_hits/
49Disallow: /payment/
50Disallow: /no_cookies/
51Disallow: /double_activation/
52Disallow: /projects/?*
53Disallow: /?*/
54Disallow: /activation_successful*
55Disallow: /api.html?*
56Disallow: /partners/
57Disallow: /partner/
58Disallow: */dvd-mode/
59Disallow: /unsubscribe/
60Disallow: /unsubscribe/*
61Disallow: /demonstration/
62Disallow: /demonstration/*
63Allow: /projects/
64Disallow: */custom-report/
65Disallow: */request-demo/
66Allow: */info/kdt/
67Disallow: */pleasereg/
68Disallow: */custom-report/
69Disallow: */custom-report
70Allow: */sensor/*
71Disallow: /swa/api/*
72Disallow: /redirect?url=*
73Disallow: /sso/
74Disallow: /_/semrush_regular_1_blank/
75Disallow: /seo-old/*
76Disallow: /projects2/*
77Disallow: /free-tools/plagiarism-checker/?q=
78Allow: /news/*?page=*
79Allow: /analytics/keywordoverview/*
80
81# Community rules
82Allow: /blog/*
83Disallow: /blog/search/*
84Disallow: /blog/*utm_source=*
85Disallow: /blog-manager/*
86Allow: /user/*
87
88# Disallow technical signup pages
89Allow: /login/$
90Allow: /signup/$
91Disallow: /login/*
92Disallow: /signup/*
93
94# Enterprise
95Allow: /enterprise/seo/
96
97#landing
98Disallow: /landing/
99Disallow: /lp/
100Allow: /lp/affiliate-program/*
101Allow: /lp/for-agencies/en/
102Allow: /lp/cannes2026/*
103Allow: /lp/enterprise-aio/*
104Allow: /lp/enterprise-crawler/*
105Allow: /lp/enterprise-partner-program/*
106Allow: /lp/seasonal-ai/en/
107Allow: /lp/black-friday/*
108Disallow: /lp/black-friday-aff/*
109Allow: /lp/cyber-week/*
110Allow: /lp/journeys/*
111Allow: /lp/insights/*
112Allow: /lp/growth-quadrant/
113Allow: /lp/mfour/*
114Allow: /lp/semrush-circle/*
115Allow: /lp/semrush-one/*
116Allow: /seo/
117Allow: /lp/seo-consultation/*
118Allow: /content-marketing/
119Allow: /ppc/
120Allow: /competitive-research/
121Allow: /analytics/traffic/overview/
122Disallow: /listings-management/superpower-landing2/
123Allow: /semrush-questions/*
124
125#academy
126Allow: /academy/*
127Disallow: /academy/*utm_source=*
128Disallow: /academy/*/results
129Disallow: /academy/certificates/*
130
131#websites
132Allow: /website/*
133Disallow: /website/search/*
134
135Disallow: /*/?*
136Crawl-delay: 20
137
138# Sitemap files
139#Blog
140Sitemap: https://www.semrush.com/sitemap.xml
141Sitemap: https://pt.semrush.com/blog/sitemap/
142Sitemap: https://es.semrush.com/blog/sitemap/
143Sitemap: https://it.semrush.com/blog/sitemap/
144Sitemap: https://fr.semrush.com/blog/sitemap/
145Sitemap: https://de.semrush.com/blog/sitemap/
146Sitemap: https://ja.semrush.com/blog/sitemap/
147#Academy
148Sitemap: https://www.semrush.com/academy/sitemap.xml
149Sitemap: https://es.semrush.com/academy/sitemap.xml
150Sitemap: https://pt.semrush.com/academy/sitemap.xml
151Sitemap: https://it.semrush.com/academy/sitemap.xml
152Sitemap: https://fr.semrush.com/academy/sitemap.xml
153Sitemap: https://de.semrush.com/academy/sitemap.xml
154#Free tools:
155Sitemap: https://www.semrush.com/free-tools/sitemap.xml