NeuralCrawl

Zscaler / robots.txt snapshot

← back to zscaler.com · fetched 2026-06-20T01:10:31Z (15h ago) · HTTP 200 · 3932 bytes · sha256 14498d61ca7d020f · raw

final URL: https://www.zscaler.com/robots.txt

1
2 User-agent: *
3 Crawl-delay: 10
4
5 Allow: /*/resources?topic=*
6 Allow: /*/blogs?type=*
7 Allow: /*/blogs?topic=*
8 Allow: /*/resources?type=*
9 Allow: /*/zpedia?type=*
10 Allow: /*/resources/webinars?type=*
11 Allow: /*/resources/webinars?topic=*
12 Allow: /*/careers/search?office=*
13
14 Allow: /resources?topic=*
15 Allow: /blogs?type=*
16 Allow: /blogs?topic=*
17 Allow: /resources?type=*
18 Allow: /zpedia?type=*
19 Allow: /resources/webinars?type=*
20 Allow: /resources/webinars?topic=*
21 Allow: /careers/search?office=*
22
23
24 # Zscaler: leftovers
25 Disallow: *_archive.html
26 Disallow: /community-api-webhook.php
27
28 # Next settings
29 Disallow: /preview_*
30
31 # Zscaler: Disallow email-signature page
32 Disallow: /email-signature
33 Disallow: /*/email-signature
34
35 # Query strings
36 Disallow: /*?*
37
38 # Allow _next images
39 Allow: /_next/image?url=*
40
41 # To be remove after proxy
42 # CSS, JS, Images
43 Allow: /core/*.css$
44 Allow: /core/*.css?
45 Allow: /core/*.js$
46 Allow: /core/*.js?
47 Allow: /core/*.gif
48 Allow: /core/*.jpg
49 Allow: /core/*.jpeg
50 Allow: /core/*.png
51 Allow: /core/*.svg
52 Allow: /profiles/*.css$
53 Allow: /profiles/*.css?
54 Allow: /profiles/*.js$
55 Allow: /profiles/*.js?
56 Allow: /profiles/*.gif
57 Allow: /profiles/*.jpg
58 Allow: /profiles/*.jpeg
59 Allow: /profiles/*.png
60 Allow: /profiles/*.svg
61 # Directories
62 Disallow: /core/
63 Disallow: /profiles/
64 # Files
65 Disallow: /README.txt
66 Disallow: /web.config
67 # Paths (clean URLs)
68 Disallow: /admin/
69 Disallow: /comment/reply/
70 Disallow: /filter/tips
71 Disallow: /node/add/
72 Disallow: /search/
73 Disallow: /user/register/
74 Disallow: /user/password/
75 Disallow: /user/login/
76 Disallow: /user/logout/
77 # Paths (no clean URLs)
78 Disallow: /index.php/admin/
79 Disallow: /index.php/comment/reply/
80 Disallow: /index.php/filter/tips
81 Disallow: /index.php/node/add/
82 Disallow: /index.php/search/
83 Disallow: /index.php/user/password/
84 Disallow: /index.php/user/register/
85 Disallow: /index.php/user/login/
86 Disallow: /index.php/user/logout/
87
88 # Zscaler: CSS, JS, Images
89 Allow: /themes/*.css$
90 Allow: /themes/*.css?
91 Allow: /themes/*.js$
92 Allow: /themes/*.js?
93 Allow: /themes/*.gif
94 Allow: /themes/*.jpg
95 Allow: /themes/*.jpeg
96 Allow: /themes/*.png
97 Allow: /themes/*.svg
98 Allow: /themes/*.woff2
99
100 # Zscaler: Directories
101 Disallow: /includes/
102 Disallow: /misc/
103 Disallow: /modules/
104 Disallow: /profiles/
105 Disallow: /scripts/
106 Disallow: /themes/
107
108 # Zscaler: Files
109 Disallow: /CHANGELOG.txt
110 Disallow: /cron.php
111 Disallow: /INSTALL.mysql.txt
112 Disallow: /INSTALL.pgsql.txt
113 Disallow: /INSTALL.sqlite.txt
114 Disallow: /install.php
115 Disallow: /INSTALL.txt
116 Disallow: /LICENSE.txt
117 Disallow: /MAINTAINERS.txt
118 Disallow: /update.php
119 Disallow: /UPGRADE.txt
120 Disallow: /xmlrpc.php
121
122 # Zscaler: Paths (clean URLs)
123 Disallow: /admin/
124 Disallow: /comment/reply/
125 Disallow: /filter/tips/
126 Disallow: /node/add/
127 Disallow: /search/
128 Disallow: /user/register/
129 Disallow: /user/password/
130 Disallow: /user/login/
131 Disallow: /user/logout/
132
133 # Query strings
134 Disallow: /*?*
135
136 Disallow: /en$
137 Disallow: /en/*
138
139 # Zscaler: Disallow cdn URLs
140 Disallow: /cdn/*
141
142 # Zscaler: GWT Error
143 Disallow: /drupal
144 Disallow: /drupal/*
145
146 # Zscaler: leftovers
147 Disallow: *_archive.html
148 Disallow: /community-api-webhook.php
149
150 # Disallow for proxy pages of m7
151 Disallow: /media/oembed
152 Disallow: /*/media/oembed
153 Disallow: /index.php/media/oembed
154 Disallow: /index.php/*/media/oembed
155
156 # Disallow IndexNowKey
157 Disallow: /indexNowKey.txt
158
159 # To be remove after proxy
160 # CSS, JS, Images ends
161
162 Sitemap: https://www.zscaler.com/sitemap_index.xml
163