NeuralCrawl

Zscaler / robots.txt snapshot

← back to zscaler.com · fetched 2026-09-23T10:50:20Z (1mo ago) · HTTP 200 · 4701 bytes · sha256 c6d5c8049edafe10 · raw

final URL: https://www.zscaler.com/robots.txt

1
2 User-agent: *
3 Allow: /
4 # ======================================================================
5 # AI/LLM Friendly Content Locations
6 # See https://llmstxt.org for the llms.txt specification
7 #
8 # - /llms.txt - Curated overview for AI/LLMs (markdown)
9 # - /llms-full.txt - Full expanded content for larger context windows
10 # - /answers.json - Structured QA/answer feed for AI engines
11 # ======================================================================
12
13 # Allow AI search engines and user-triggered LLM crawlers
14 User-agent: OAI-SearchBot
15 User-agent: ChatGPT-User
16 User-agent: PerplexityBot
17 User-agent: Claude-SearchBot
18 User-agent: Claude-User
19 User-agent: ClaudeBot
20 User-agent: GPTBot
21 User-agent: Google-Extended
22 User-agent: Cohere-ai
23 User-agent: Applebot-Extended
24 Allow: /
25
26
27
28 Allow: /*/resources?topic=*
29 Allow: /*/blogs?type=*
30 Allow: /*/blogs?topic=*
31 Allow: /*/resources?type=*
32 Allow: /*/zpedia?type=*
33 Allow: /*/resources/webinars?type=*
34 Allow: /*/resources/webinars?topic=*
35 Allow: /*/careers/search?office=*
36
37 Allow: /resources?topic=*
38 Allow: /blogs?type=*
39 Allow: /blogs?topic=*
40 Allow: /resources?type=*
41 Allow: /zpedia?type=*
42 Allow: /resources/webinars?type=*
43 Allow: /resources/webinars?topic=*
44 Allow: /careers/search?office=*
45
46
47 # Zscaler: leftovers
48 Disallow: *_archive.html
49 Disallow: /community-api-webhook.php
50
51 # Next settings
52 Disallow: /preview_*
53
54 # Zscaler: Disallow email-signature page
55 Disallow: /email-signature
56 Disallow: /*/email-signature
57
58 # Allow _next images
59 Allow: /_next/image?url=*
60
61 # To be remove after proxy
62 # CSS, JS, Images
63 Allow: /core/*.css$
64 Allow: /core/*.css?
65 Allow: /core/*.js$
66 Allow: /core/*.js?
67 Allow: /core/*.gif
68 Allow: /core/*.jpg
69 Allow: /core/*.jpeg
70 Allow: /core/*.png
71 Allow: /core/*.svg
72 Allow: /profiles/*.css$
73 Allow: /profiles/*.css?
74 Allow: /profiles/*.js$
75 Allow: /profiles/*.js?
76 Allow: /profiles/*.gif
77 Allow: /profiles/*.jpg
78 Allow: /profiles/*.jpeg
79 Allow: /profiles/*.png
80 Allow: /profiles/*.svg
81 # Directories
82 Disallow: /core/
83 Disallow: /profiles/
84 # Files
85 Disallow: /README.txt
86 Disallow: /web.config
87 # Paths (clean URLs)
88 Disallow: /admin/
89 Disallow: /comment/reply/
90 Disallow: /filter/tips
91 Disallow: /node/add/
92 Disallow: /search/
93 Disallow: /user/register/
94 Disallow: /user/password/
95 Disallow: /user/login/
96 Disallow: /user/logout/
97 # Paths (no clean URLs)
98 Disallow: /index.php/admin/
99 Disallow: /index.php/comment/reply/
100 Disallow: /index.php/filter/tips
101 Disallow: /index.php/node/add/
102 Disallow: /index.php/search/
103 Disallow: /index.php/user/password/
104 Disallow: /index.php/user/register/
105 Disallow: /index.php/user/login/
106 Disallow: /index.php/user/logout/
107
108 # Zscaler: CSS, JS, Images
109 Allow: /themes/*.css$
110 Allow: /themes/*.css?
111 Allow: /themes/*.js$
112 Allow: /themes/*.js?
113 Allow: /themes/*.gif
114 Allow: /themes/*.jpg
115 Allow: /themes/*.jpeg
116 Allow: /themes/*.png
117 Allow: /themes/*.svg
118 Allow: /themes/*.woff2
119
120 # Zscaler: Directories
121 Disallow: /includes/
122 Disallow: /misc/
123 Disallow: /modules/
124 Disallow: /profiles/
125 Disallow: /scripts/
126 Disallow: /themes/
127
128 # Zscaler: Files
129 Disallow: /CHANGELOG.txt
130 Disallow: /cron.php
131 Disallow: /INSTALL.mysql.txt
132 Disallow: /INSTALL.pgsql.txt
133 Disallow: /INSTALL.sqlite.txt
134 Disallow: /install.php
135 Disallow: /INSTALL.txt
136 Disallow: /LICENSE.txt
137 Disallow: /MAINTAINERS.txt
138 Disallow: /update.php
139 Disallow: /UPGRADE.txt
140 Disallow: /xmlrpc.php
141
142 # Zscaler: Paths (clean URLs)
143 Disallow: /admin/
144 Disallow: /comment/reply/
145 Disallow: /filter/tips/
146 Disallow: /node/add/
147 Disallow: /search/
148 Disallow: /user/register/
149 Disallow: /user/password/
150 Disallow: /user/login/
151 Disallow: /user/logout/
152 Disallow: /en$
153 Disallow: /en/*
154
155 # Zscaler: Disallow cdn URLs
156 Disallow: /cdn/*
157
158 # Zscaler: GWT Error
159 Disallow: /drupal
160 Disallow: /drupal/*
161
162 # Zscaler: leftovers
163 Disallow: *_archive.html
164 Disallow: /community-api-webhook.php
165
166 # Disallow for proxy pages of m7
167 Disallow: /media/oembed
168 Disallow: /*/media/oembed
169 Disallow: /index.php/media/oembed
170 Disallow: /index.php/*/media/oembed
171
172 # Disallow IndexNowKey
173 Disallow: /8199ac63-be6c-4cba-9bd9-45500a02d64f.txt
174
175 # To be remove after proxy
176 # CSS, JS, Images ends
177
178 Sitemap: https://www.zscaler.com/sitemap_index.xml
179