Fortra / robots.txt snapshot

← back to fortra.com · fetched 2026-09-18T15:27:37Z (20h ago) · HTTP 200 · 2743 bytes · sha256 00087f009f00614d · raw

final URL: https://www.fortra.com/robots.txt

1	Sitemap: https://www.fortra.com/sitemap.xml
2
3	User-agent: *
4	# CSS, JS, Images
5	Allow: /core/*.css$
6	Allow: /core/*.css?
7	Allow: /core/*.js$
8	Allow: /core/*.js?
9	Allow: /core/*.gif
10	Allow: /core/*.jpg
11	Allow: /core/*.jpeg
12	Allow: /core/*.png
13	Allow: /core/*.svg
14	Allow: /profiles/*.css$
15	Allow: /profiles/*.css?
16	Allow: /profiles/*.js$
17	Allow: /profiles/*.js?
18	Allow: /profiles/*.gif
19	Allow: /profiles/*.jpg
20	Allow: /profiles/*.jpeg
21	Allow: /profiles/*.png
22	Allow: /profiles/*.svg
23	# Directories
24	Disallow: /core/
25	Disallow: /profiles/
26	# Files
27	Disallow: /README.md
28	Disallow: /composer/Metapackage/README.txt
29	Disallow: /composer/Plugin/ProjectMessage/README.md
30	Disallow: /composer/Plugin/Scaffold/README.md
31	Disallow: /composer/Plugin/VendorHardening/README.txt
32	Disallow: /composer/Template/README.txt
33	Disallow: /modules/README.txt
34	Disallow: /sites/README.txt
35	Disallow: /themes/README.txt
36	Disallow: /web.config
37	# Paths (clean URLs)
38	Disallow: /admin/
39	Disallow: /comment/reply/
40	Disallow: /filter/tips
41	Disallow: /node/add/
42	Disallow: /search/
43	Disallow: /user/register
44	Disallow: /user/password
45	Disallow: /user/login
46	Disallow: /user/logout
47	Disallow: /media/oembed
48	Disallow: /*/media/oembed
49	# Paths (no clean URLs)
50	Disallow: /index.php/admin/
51	Disallow: /index.php/comment/reply/
52	Disallow: /index.php/filter/tips
53	Disallow: /index.php/node/add/
54	Disallow: /index.php/search/
55	Disallow: /index.php/user/password
56	Disallow: /index.php/user/register
57	Disallow: /index.php/user/login
58	Disallow: /index.php/user/logout
59	Disallow: /index.php/media/oembed
60	Disallow: /index.php/*/media/oembed
61
62	# === AI Crawler ===
63
64	# OpenAI
65	User-agent: GPTBot
66	Allow: /
67
68	User-agent: ChatGPT-User
69	Allow: /
70
71	# Anthropic
72	User-agent: ClaudeBot
73	Allow: /
74
75	# Google AI training
76	User-agent: Google-Extended
77	Allow: /
78
79	# Google general AI crawling
80	User-agent: GoogleOther
81	Allow: /
82
83	# Perplexity
84	User-agent: PerplexityBot
85	Allow: /
86
87	# Amazon (no clear training split → allow)
88	User-agent: Amazonbot
89	Allow: /
90
91	# Apple (no clear training split → allow)
92	User-agent: Applebot-Extended
93	Allow: /
94
95	# ByteDance / TikTok (no clear training split → allow)
96	User-agent: Bytespider
97	Allow: /
98
99	User-agent: TikTokSpider
100	Allow: /
101
102	# SEMrush
103	User-agent: SiteAuditBot
104	Crawl-delay: 5
105
106	User-agent: SemrushBot
107	Crawl-delay: 5
108
109	# Fallback
110	User-agent: *
111	Allow: /
112
113
114	#
115	# Upstream Robots Additions
116	#
117	# Block specific parameters
118	Disallow: /*?__hsfp
119	Disallow: /*?f=
120	Disallow: /*?wtime
121
122	# Allow specific parameters
123	Allow: /*?page=
124
125	# Resources URLs
126	Disallow: /*/resources?
127	Disallow: /*/resources$
128	Disallow: /resources?
129	Disallow: /resources$
130	Allow: /resources/
131	Allow: /*/resources/
132
133	# Search URLs
134	Disallow: /*/search
135	Disallow: /search
136
137	# Blog URLs
138	Disallow: /*/blog?
139	Disallow: /*/blog$
140	Disallow: /blog?
141	Disallow: /blog$
142	Allow: /*/blog/
143	Allow: /blog/