NeuralCrawl

Semrush / robots.txt snapshot

← back to semrush.com · fetched 2026-07-16T17:46:03Z (21d ago) · HTTP 200 · 3581 bytes · sha256 d783244848b9a1f7 · raw

final URL: https://www.semrush.com/robots.txt

1User-agent: 008
2Disallow: /
3
4User-agent: SiteAuditBot
5Crawl-delay: 1
6Allow: /
7
8User-agent: Semrushbot-SI
9Allow: /
10
11User-agent: Yahoo Pipes 2.0
12Disallow: /
13
14User-agent: Facebot
15Disallow: /archive/graphs.php
16Allow: /academy/certificates/*
17
18User-agent: facebookexternalhit
19Allow: /academy/certificates/*
20
21User-agent: Bingbot
22Disallow: /archive/graphs.php
23
24User-agent: BingPreview
25Disallow: /archive/graphs.php
26
27User-agent: LinkedInBot
28Allow: /academy/certificates/*
29
30User-agent: Twitterbot
31Allow: /blog/*
32Allow: /ranking-factors/*
33Allow: /academy/certificates/*
34
35User-agent: *
36Disallow: /blog/*?img=*
37Disallow: /archive/graphs.php
38Disallow: /forrester-marketing-silos/*
39Disallow: /partner/
40Disallow: /blog/search/*
41Disallow: /analytics/seomagic/
42Disallow: /my_reports/reports/*
43Disallow: /archive/
44Disallow: /admin/
45Disallow: /users/
46Disallow: /clients/
47Disallow: /custom_report/
48Disallow: /limit_hits/
49Disallow: /payment/
50Disallow: /no_cookies/
51Disallow: /double_activation/
52Disallow: /projects/?*
53Disallow: /?*/
54Disallow: /activation_successful*
55Disallow: /api.html?*
56Disallow: /partners/
57Disallow: /partner/
58Disallow: */dvd-mode/
59Disallow: /unsubscribe/
60Disallow: /unsubscribe/*
61Disallow: /demonstration/
62Disallow: /demonstration/*
63Allow: /projects/
64Disallow: */custom-report/
65Disallow: */request-demo/
66Allow: */info/kdt/
67Disallow: */pleasereg/
68Disallow: */custom-report/
69Disallow: */custom-report
70Allow: */sensor/*
71Disallow: /swa/api/*
72Disallow: /redirect?url=*
73Disallow: /sso/
74Disallow: /_/empty_blank/
75Disallow: /_/toolkit_blank/
76Disallow: /_/site_info_blank/
77Disallow: /seo-old/*
78Disallow: /projects2/*
79Disallow: /free-tools/plagiarism-checker/?q=
80Allow: /news/*?page=*
81
82# Community rules
83Allow: /blog/*
84Disallow: /blog/search/*
85Disallow: /blog/*utm_source=*
86Disallow: /blog-manager/*
87Allow: /user/*
88
89# Disallow technical signup pages
90Allow: /login/$
91Allow: /signup/$
92Disallow: /login/*
93Disallow: /signup/*
94
95# Enterprise
96Allow: /enterprise/seo/
97
98#landing
99Disallow: /landing/
100Disallow: /lp/
101Allow: /lp/affiliate-program/*
102Allow: /lp/for-agencies/en/
103Allow: /lp/cannes2026/*
104Allow: /lp/enterprise-aio/*
105Allow: /lp/enterprise-crawler/*
106Allow: /lp/enterprise-partner-program/*
107Allow: /lp/seasonal-ai/en/
108Allow: /lp/black-friday/*
109Disallow: /lp/black-friday-aff/*
110Allow: /lp/cyber-week/*
111Allow: /lp/journeys/*
112Allow: /lp/insights/*
113Allow: /lp/growth-quadrant/
114Allow: /lp/mfour/*
115Allow: /lp/semrush-circle/*
116Allow: /lp/semrush-one/*
117Allow: /seo/
118Allow: /lp/seo-consultation/*
119Allow: /content-marketing/
120Allow: /ppc/
121Allow: /competitive-research/
122Allow: /analytics/traffic/overview/
123Disallow: /listings-management/superpower-landing2/
124
125#academy
126Allow: /academy/*
127Disallow: /academy/*utm_source=*
128Disallow: /academy/*/results
129Disallow: /academy/certificates/*
130
131#websites
132Allow: /website/*
133Disallow: /website/search/*
134
135Disallow: /*/?*
136Crawl-delay: 20
137
138# Sitemap files
139#Blog
140Sitemap: https://www.semrush.com/sitemap.xml
141Sitemap: https://pt.semrush.com/blog/sitemap/
142Sitemap: https://es.semrush.com/blog/sitemap/
143Sitemap: https://it.semrush.com/blog/sitemap/
144Sitemap: https://fr.semrush.com/blog/sitemap/
145Sitemap: https://de.semrush.com/blog/sitemap/
146Sitemap: https://ja.semrush.com/blog/sitemap/
147#Academy
148Sitemap: https://www.semrush.com/academy/sitemap.xml
149Sitemap: https://es.semrush.com/academy/sitemap.xml
150Sitemap: https://pt.semrush.com/academy/sitemap.xml
151Sitemap: https://it.semrush.com/academy/sitemap.xml
152Sitemap: https://fr.semrush.com/academy/sitemap.xml
153Sitemap: https://de.semrush.com/academy/sitemap.xml
154#Free tools:
155Sitemap: https://www.semrush.com/free-tools/sitemap.xml