NeuralCrawl

Zscaler / robots.txt snapshot

← back to zscaler.com · fetched 2026-08-04T12:42:22Z (11h ago) · HTTP 200 · 4757 bytes · sha256 ec50a33a4eff6bd4 · raw

final URL: https://www.zscaler.com/robots.txt

1
2 User-agent: *
3 Allow: /
4 # ======================================================================
5 # AI/LLM Friendly Content Locations
6 # See https://llmstxt.org for the llms.txt specification
7 #
8 # - /llms.txt - Curated overview for AI/LLMs (markdown)
9 # - /llms-full.txt - Full expanded content for larger context windows
10 # - /answers.json - Structured QA/answer feed for AI engines
11 # ======================================================================
12
13 # Allow AI search engines and user-triggered LLM crawlers
14 User-agent: OAI-SearchBot
15 User-agent: ChatGPT-User
16 User-agent: PerplexityBot
17 User-agent: Claude-SearchBot
18 User-agent: Claude-User
19 User-agent: ClaudeBot
20 User-agent: GPTBot
21 User-agent: Google-Extended
22 User-agent: Cohere-ai
23 User-agent: Applebot-Extended
24 Allow: /
25
26
27
28 Allow: /*/resources?topic=*
29 Allow: /*/blogs?type=*
30 Allow: /*/blogs?topic=*
31 Allow: /*/resources?type=*
32 Allow: /*/zpedia?type=*
33 Allow: /*/resources/webinars?type=*
34 Allow: /*/resources/webinars?topic=*
35 Allow: /*/careers/search?office=*
36
37 Allow: /resources?topic=*
38 Allow: /blogs?type=*
39 Allow: /blogs?topic=*
40 Allow: /resources?type=*
41 Allow: /zpedia?type=*
42 Allow: /resources/webinars?type=*
43 Allow: /resources/webinars?topic=*
44 Allow: /careers/search?office=*
45
46
47 # Zscaler: leftovers
48 Disallow: *_archive.html
49 Disallow: /community-api-webhook.php
50
51 # Next settings
52 Disallow: /preview_*
53
54 # Zscaler: Disallow email-signature page
55 Disallow: /email-signature
56 Disallow: /*/email-signature
57
58 # Query strings
59 Disallow: /*?*
60
61 # Allow _next images
62 Allow: /_next/image?url=*
63
64 # To be remove after proxy
65 # CSS, JS, Images
66 Allow: /core/*.css$
67 Allow: /core/*.css?
68 Allow: /core/*.js$
69 Allow: /core/*.js?
70 Allow: /core/*.gif
71 Allow: /core/*.jpg
72 Allow: /core/*.jpeg
73 Allow: /core/*.png
74 Allow: /core/*.svg
75 Allow: /profiles/*.css$
76 Allow: /profiles/*.css?
77 Allow: /profiles/*.js$
78 Allow: /profiles/*.js?
79 Allow: /profiles/*.gif
80 Allow: /profiles/*.jpg
81 Allow: /profiles/*.jpeg
82 Allow: /profiles/*.png
83 Allow: /profiles/*.svg
84 # Directories
85 Disallow: /core/
86 Disallow: /profiles/
87 # Files
88 Disallow: /README.txt
89 Disallow: /web.config
90 # Paths (clean URLs)
91 Disallow: /admin/
92 Disallow: /comment/reply/
93 Disallow: /filter/tips
94 Disallow: /node/add/
95 Disallow: /search/
96 Disallow: /user/register/
97 Disallow: /user/password/
98 Disallow: /user/login/
99 Disallow: /user/logout/
100 # Paths (no clean URLs)
101 Disallow: /index.php/admin/
102 Disallow: /index.php/comment/reply/
103 Disallow: /index.php/filter/tips
104 Disallow: /index.php/node/add/
105 Disallow: /index.php/search/
106 Disallow: /index.php/user/password/
107 Disallow: /index.php/user/register/
108 Disallow: /index.php/user/login/
109 Disallow: /index.php/user/logout/
110
111 # Zscaler: CSS, JS, Images
112 Allow: /themes/*.css$
113 Allow: /themes/*.css?
114 Allow: /themes/*.js$
115 Allow: /themes/*.js?
116 Allow: /themes/*.gif
117 Allow: /themes/*.jpg
118 Allow: /themes/*.jpeg
119 Allow: /themes/*.png
120 Allow: /themes/*.svg
121 Allow: /themes/*.woff2
122
123 # Zscaler: Directories
124 Disallow: /includes/
125 Disallow: /misc/
126 Disallow: /modules/
127 Disallow: /profiles/
128 Disallow: /scripts/
129 Disallow: /themes/
130
131 # Zscaler: Files
132 Disallow: /CHANGELOG.txt
133 Disallow: /cron.php
134 Disallow: /INSTALL.mysql.txt
135 Disallow: /INSTALL.pgsql.txt
136 Disallow: /INSTALL.sqlite.txt
137 Disallow: /install.php
138 Disallow: /INSTALL.txt
139 Disallow: /LICENSE.txt
140 Disallow: /MAINTAINERS.txt
141 Disallow: /update.php
142 Disallow: /UPGRADE.txt
143 Disallow: /xmlrpc.php
144
145 # Zscaler: Paths (clean URLs)
146 Disallow: /admin/
147 Disallow: /comment/reply/
148 Disallow: /filter/tips/
149 Disallow: /node/add/
150 Disallow: /search/
151 Disallow: /user/register/
152 Disallow: /user/password/
153 Disallow: /user/login/
154 Disallow: /user/logout/
155
156 # Query strings
157 Disallow: /*?*
158
159 Disallow: /en$
160 Disallow: /en/*
161
162 # Zscaler: Disallow cdn URLs
163 Disallow: /cdn/*
164
165 # Zscaler: GWT Error
166 Disallow: /drupal
167 Disallow: /drupal/*
168
169 # Zscaler: leftovers
170 Disallow: *_archive.html
171 Disallow: /community-api-webhook.php
172
173 # Disallow for proxy pages of m7
174 Disallow: /media/oembed
175 Disallow: /*/media/oembed
176 Disallow: /index.php/media/oembed
177 Disallow: /index.php/*/media/oembed
178
179 # Disallow IndexNowKey
180 Disallow: /indexNowKey.txt
181
182 # To be remove after proxy
183 # CSS, JS, Images ends
184
185 Sitemap: https://www.zscaler.com/sitemap_index.xml
186