Fortra / robots.txt snapshot

← back to fortra.com · fetched 2026-07-01T00:16:40Z (1mo ago) · HTTP 200 · 2662 bytes · sha256 36469889d2aae694 · raw

final URL: https://www.fortra.com/robots.txt

1	Sitemap: https://www.fortra.com/sitemap.xml
2
3	User-agent: *
4	# CSS, JS, Images
5	Allow: /core/*.css$
6	Allow: /core/*.css?
7	Allow: /core/*.js$
8	Allow: /core/*.js?
9	Allow: /core/*.gif
10	Allow: /core/*.jpg
11	Allow: /core/*.jpeg
12	Allow: /core/*.png
13	Allow: /core/*.svg
14	Allow: /profiles/*.css$
15	Allow: /profiles/*.css?
16	Allow: /profiles/*.js$
17	Allow: /profiles/*.js?
18	Allow: /profiles/*.gif
19	Allow: /profiles/*.jpg
20	Allow: /profiles/*.jpeg
21	Allow: /profiles/*.png
22	Allow: /profiles/*.svg
23	# Directories
24	Disallow: /core/
25	Disallow: /profiles/
26	# Files
27	Disallow: /README.md
28	Disallow: /composer/Metapackage/README.txt
29	Disallow: /composer/Plugin/ProjectMessage/README.md
30	Disallow: /composer/Plugin/Scaffold/README.md
31	Disallow: /composer/Plugin/VendorHardening/README.txt
32	Disallow: /composer/Template/README.txt
33	Disallow: /modules/README.txt
34	Disallow: /sites/README.txt
35	Disallow: /themes/README.txt
36	Disallow: /web.config
37	# Paths (clean URLs)
38	Disallow: /admin/
39	Disallow: /comment/reply/
40	Disallow: /filter/tips
41	Disallow: /node/add/
42	Disallow: /search/
43	Disallow: /user/register
44	Disallow: /user/password
45	Disallow: /user/login
46	Disallow: /user/logout
47	Disallow: /media/oembed
48	Disallow: /*/media/oembed
49	# Paths (no clean URLs)
50	Disallow: /index.php/admin/
51	Disallow: /index.php/comment/reply/
52	Disallow: /index.php/filter/tips
53	Disallow: /index.php/node/add/
54	Disallow: /index.php/search/
55	Disallow: /index.php/user/password
56	Disallow: /index.php/user/register
57	Disallow: /index.php/user/login
58	Disallow: /index.php/user/logout
59	Disallow: /index.php/media/oembed
60	Disallow: /index.php/*/media/oembed
61
62	# === AI Crawler ===
63
64	# OpenAI
65	User-agent: GPTBot
66	Disallow: /
67
68	User-agent: ChatGPT-User
69	Allow: /
70
71	# Anthropic
72	User-agent: ClaudeBot
73	Disallow: /
74
75	# Google AI training
76	User-agent: Google-Extended
77	Disallow: /
78
79	# Google general AI crawling
80	User-agent: GoogleOther
81	Allow: /
82
83	# Perplexity
84	User-agent: PerplexityBot
85	Allow: /
86
87	# Amazon (no clear training split → allow)
88	User-agent: Amazonbot
89	Allow: /
90
91	# Apple (no clear training split → allow)
92	User-agent: Applebot-Extended
93	Allow: /
94
95	# ByteDance / TikTok (no clear training split → allow)
96	User-agent: Bytespider
97	Allow: /
98
99	User-agent: TikTokSpider
100	Allow: /
101
102	# Fallback
103	User-agent: *
104	Allow: /
105
106
107	#
108	# Upstream Robots Additions
109	#
110	# Block specific parameters
111	Disallow: /*?__hsfp
112	Disallow: /*?f=
113	Disallow: /*?wtime
114
115	# Allow specific parameters
116	Allow: /*?page=
117
118	# Resources URLs
119	Disallow: /*/resources?
120	Disallow: /*/resources$
121	Disallow: /resources?
122	Disallow: /resources$
123	Allow: /resources/
124	Allow: /*/resources/
125
126	# Search URLs
127	Disallow: /*/search
128	Disallow: /search
129
130	# Blog URLs
131	Disallow: /*/blog?
132	Disallow: /*/blog$
133	Disallow: /blog?
134	Disallow: /blog$
135	Allow: /*/blog/
136	Allow: /blog/