NeuralCrawl

Semrush / robots.txt snapshot

← back to semrush.com · fetched 2026-08-19T17:42:33Z (2mo ago) · HTTP 200 · 3645 bytes · sha256 09df5a37281fc62f · raw

final URL: https://www.semrush.com/robots.txt

1User-agent: 008
2Disallow: /
3
4User-agent: SiteAuditBot
5Crawl-delay: 1
6Allow: /
7
8User-agent: Semrushbot-SI
9Allow: /
10
11User-agent: Yahoo Pipes 2.0
12Disallow: /
13
14User-agent: Facebot
15Disallow: /archive/graphs.php
16Allow: /academy/certificates/*
17
18User-agent: facebookexternalhit
19Allow: /academy/certificates/*
20
21User-agent: Bingbot
22Disallow: /archive/graphs.php
23
24User-agent: BingPreview
25Disallow: /archive/graphs.php
26
27User-agent: LinkedInBot
28Allow: /academy/certificates/*
29
30User-agent: Twitterbot
31Allow: /blog/*
32Allow: /ranking-factors/*
33Allow: /academy/certificates/*
34
35User-agent: *
36Disallow: /blog/*?img=*
37Disallow: /archive/graphs.php
38Disallow: /forrester-marketing-silos/*
39Disallow: /partner/
40Disallow: /blog/search/*
41Disallow: /analytics/seomagic/
42Disallow: /my_reports/reports/*
43Disallow: /archive/
44Disallow: /admin/
45Disallow: /users/
46Disallow: /clients/
47Disallow: /custom_report/
48Disallow: /limit_hits/
49Disallow: /payment/
50Disallow: /no_cookies/
51Disallow: /double_activation/
52Disallow: /projects/?*
53Disallow: /?*/
54Disallow: /activation_successful*
55Disallow: /api.html?*
56Disallow: /partners/
57Disallow: /partner/
58Disallow: */dvd-mode/
59Disallow: /unsubscribe/
60Disallow: /unsubscribe/*
61Disallow: /demonstration/
62Disallow: /demonstration/*
63Allow: /projects/
64Disallow: */custom-report/
65Disallow: */request-demo/
66Allow: */info/kdt/
67Disallow: */pleasereg/
68Disallow: */custom-report/
69Disallow: */custom-report
70Allow: */sensor/*
71Disallow: /swa/api/*
72Disallow: /redirect?url=*
73Disallow: /sso/
74Disallow: /_/empty_blank/
75Disallow: /_/toolkit_blank/
76Disallow: /_/site_info_blank/
77Disallow: /seo-old/*
78Disallow: /projects2/*
79Disallow: /free-tools/plagiarism-checker/?q=
80Allow: /news/*?page=*
81Allow: /analytics/keywordoverview/*
82
83# Community rules
84Allow: /blog/*
85Disallow: /blog/search/*
86Disallow: /blog/*utm_source=*
87Disallow: /blog-manager/*
88Allow: /user/*
89
90# Disallow technical signup pages
91Allow: /login/$
92Allow: /signup/$
93Disallow: /login/*
94Disallow: /signup/*
95
96# Enterprise
97Allow: /enterprise/seo/
98
99#landing
100Disallow: /landing/
101Disallow: /lp/
102Allow: /lp/affiliate-program/*
103Allow: /lp/for-agencies/en/
104Allow: /lp/cannes2026/*
105Allow: /lp/enterprise-aio/*
106Allow: /lp/enterprise-crawler/*
107Allow: /lp/enterprise-partner-program/*
108Allow: /lp/seasonal-ai/en/
109Allow: /lp/black-friday/*
110Disallow: /lp/black-friday-aff/*
111Allow: /lp/cyber-week/*
112Allow: /lp/journeys/*
113Allow: /lp/insights/*
114Allow: /lp/growth-quadrant/
115Allow: /lp/mfour/*
116Allow: /lp/semrush-circle/*
117Allow: /lp/semrush-one/*
118Allow: /seo/
119Allow: /lp/seo-consultation/*
120Allow: /content-marketing/
121Allow: /ppc/
122Allow: /competitive-research/
123Allow: /analytics/traffic/overview/
124Disallow: /listings-management/superpower-landing2/
125Allow: /semrush-questions/*
126
127#academy
128Allow: /academy/*
129Disallow: /academy/*utm_source=*
130Disallow: /academy/*/results
131Disallow: /academy/certificates/*
132
133#websites
134Allow: /website/*
135Disallow: /website/search/*
136
137Disallow: /*/?*
138Crawl-delay: 20
139
140# Sitemap files
141#Blog
142Sitemap: https://www.semrush.com/sitemap.xml
143Sitemap: https://pt.semrush.com/blog/sitemap/
144Sitemap: https://es.semrush.com/blog/sitemap/
145Sitemap: https://it.semrush.com/blog/sitemap/
146Sitemap: https://fr.semrush.com/blog/sitemap/
147Sitemap: https://de.semrush.com/blog/sitemap/
148Sitemap: https://ja.semrush.com/blog/sitemap/
149#Academy
150Sitemap: https://www.semrush.com/academy/sitemap.xml
151Sitemap: https://es.semrush.com/academy/sitemap.xml
152Sitemap: https://pt.semrush.com/academy/sitemap.xml
153Sitemap: https://it.semrush.com/academy/sitemap.xml
154Sitemap: https://fr.semrush.com/academy/sitemap.xml
155Sitemap: https://de.semrush.com/academy/sitemap.xml
156#Free tools:
157Sitemap: https://www.semrush.com/free-tools/sitemap.xml