NeuralCrawl

Scale AI / robots.txt

← back to scale.com · change detected 2mo ago (2026-08-25T19:41:43Z) · modified +256 −0

What changed

Diff old (2026-08-21T00:12:49Z) → new (2026-08-25T19:41:43Z)

@@ -10,6 +10,262 @@
1010  Disallow: /*?*utm_source=*
1111  Disallow: /*?*_hstc=*
1212  Disallow: /*?*
13+ Disallow: /retail/solution-brief/
14+ Disallow: /_makeswift/
15+ Disallow: /404
16+ Disallow: /corp/
17+ Disallow: /corp-api/
18+ Disallow: /internal/
19+ Disallow: /scaleai/
20+ Disallow: /&spoof=*
21+ Disallow: /events/transformx-2021/linkedin-auth
22+ Disallow: /events/transformx-2021/ticket-image
23+ Disallow: /events/transformx-2021/ticket/*
24+ Disallow: /events/transformx/linkedin-auth
25+ Disallow: /3d-sensor-fusion/*
26+ Disallow: /careers/legal
27+ Disallow: /careers/legal/*
28+ Disallow: /claims-intelligence
29+ Disallow: /content-understanding
30+ Disallow: /data-labeling
31+ Disallow: /styleguide
32+ Disallow: /federal-video-frame
33+ Disallow: /animations
34+ Disallow: /research
35+ Disallow: /research/*
36+
37+ # Twitterbot
38+ User-agent: Twitterbot
39+ Allow: /
40+ Disallow: /studio
41+ Disallow: /studio/*
42+ Disallow: /draft
43+ Disallow: /cdn-cgi/l/email-protection
44+ Disallow: /feedback/
45+ Disallow: /retail/solution-brief/
46+ Disallow: /_makeswift/
47+ Disallow: /404
48+ Disallow: /corp/
49+ Disallow: /corp-api/
50+ Disallow: /internal/
51+ Disallow: /scaleai/
52+ Disallow: /&spoof=*
53+ Disallow: /events/transformx-2021/linkedin-auth
54+ Disallow: /events/transformx-2021/ticket-image
55+ Disallow: /events/transformx-2021/ticket/*
56+ Disallow: /events/transformx/linkedin-auth
57+ Disallow: /3d-sensor-fusion/*
58+ Disallow: /careers/legal
59+ Disallow: /careers/legal/*
60+ Disallow: /claims-intelligence
61+ Disallow: /content-understanding
62+ Disallow: /data-labeling
63+ Disallow: /styleguide
64+ Disallow: /federal-video-frame
65+ Disallow: /animations
66+ Disallow: /research
67+ Disallow: /research/*
68+
69+ # facebookexternalhit
70+ User-agent: facebookexternalhit
71+ Allow: /
72+ Disallow: /studio
73+ Disallow: /studio/*
74+ Disallow: /draft
75+ Disallow: /cdn-cgi/l/email-protection
76+ Disallow: /feedback/
77+ Disallow: /retail/solution-brief/
78+ Disallow: /_makeswift/
79+ Disallow: /404
80+ Disallow: /corp/
81+ Disallow: /corp-api/
82+ Disallow: /internal/
83+ Disallow: /scaleai/
84+ Disallow: /&spoof=*
85+ Disallow: /events/transformx-2021/linkedin-auth
86+ Disallow: /events/transformx-2021/ticket-image
87+ Disallow: /events/transformx-2021/ticket/*
88+ Disallow: /events/transformx/linkedin-auth
89+ Disallow: /3d-sensor-fusion/*
90+ Disallow: /careers/legal
91+ Disallow: /careers/legal/*
92+ Disallow: /claims-intelligence
93+ Disallow: /content-understanding
94+ Disallow: /data-labeling
95+ Disallow: /styleguide
96+ Disallow: /federal-video-frame
97+ Disallow: /animations
98+ Disallow: /research
99+ Disallow: /research/*
100+
101+ # LinkedInBot
102+ User-agent: LinkedInBot
103+ Allow: /
104+ Disallow: /studio
105+ Disallow: /studio/*
106+ Disallow: /draft
107+ Disallow: /cdn-cgi/l/email-protection
108+ Disallow: /feedback/
109+ Disallow: /retail/solution-brief/
110+ Disallow: /_makeswift/
111+ Disallow: /404
112+ Disallow: /corp/
113+ Disallow: /corp-api/
114+ Disallow: /internal/
115+ Disallow: /scaleai/
116+ Disallow: /&spoof=*
117+ Disallow: /events/transformx-2021/linkedin-auth
118+ Disallow: /events/transformx-2021/ticket-image
119+ Disallow: /events/transformx-2021/ticket/*
120+ Disallow: /events/transformx/linkedin-auth
121+ Disallow: /3d-sensor-fusion/*
122+ Disallow: /careers/legal
123+ Disallow: /careers/legal/*
124+ Disallow: /claims-intelligence
125+ Disallow: /content-understanding
126+ Disallow: /data-labeling
127+ Disallow: /styleguide
128+ Disallow: /federal-video-frame
129+ Disallow: /animations
130+ Disallow: /research
131+ Disallow: /research/*
132+
133+ # Discordbot
134+ User-agent: Discordbot
135+ Allow: /
136+ Disallow: /studio
137+ Disallow: /studio/*
138+ Disallow: /draft
139+ Disallow: /cdn-cgi/l/email-protection
140+ Disallow: /feedback/
141+ Disallow: /retail/solution-brief/
142+ Disallow: /_makeswift/
143+ Disallow: /404
144+ Disallow: /corp/
145+ Disallow: /corp-api/
146+ Disallow: /internal/
147+ Disallow: /scaleai/
148+ Disallow: /&spoof=*
149+ Disallow: /events/transformx-2021/linkedin-auth
150+ Disallow: /events/transformx-2021/ticket-image
151+ Disallow: /events/transformx-2021/ticket/*
152+ Disallow: /events/transformx/linkedin-auth
153+ Disallow: /3d-sensor-fusion/*
154+ Disallow: /careers/legal
155+ Disallow: /careers/legal/*
156+ Disallow: /claims-intelligence
157+ Disallow: /content-understanding
158+ Disallow: /data-labeling
159+ Disallow: /styleguide
160+ Disallow: /federal-video-frame
161+ Disallow: /animations
162+ Disallow: /research
163+ Disallow: /research/*
164+
165+ # Slackbot
166+ User-agent: Slackbot
167+ Allow: /
168+ Disallow: /studio
169+ Disallow: /studio/*
170+ Disallow: /draft
171+ Disallow: /cdn-cgi/l/email-protection
172+ Disallow: /feedback/
173+ Disallow: /retail/solution-brief/
174+ Disallow: /_makeswift/
175+ Disallow: /404
176+ Disallow: /corp/
177+ Disallow: /corp-api/
178+ Disallow: /internal/
179+ Disallow: /scaleai/
180+ Disallow: /&spoof=*
181+ Disallow: /events/transformx-2021/linkedin-auth
182+ Disallow: /events/transformx-2021/ticket-image
183+ Disallow: /events/transformx-2021/ticket/*
184+ Disallow: /events/transformx/linkedin-auth
185+ Disallow: /3d-sensor-fusion/*
186+ Disallow: /careers/legal
187+ Disallow: /careers/legal/*
188+ Disallow: /claims-intelligence
189+ Disallow: /content-understanding
190+ Disallow: /data-labeling
191+ Disallow: /styleguide
192+ Disallow: /federal-video-frame
193+ Disallow: /animations
194+ Disallow: /research
195+ Disallow: /research/*
196+
197+ # Slackbot-LinkExpanding
198+ User-agent: Slackbot-LinkExpanding
199+ Allow: /
200+ Disallow: /studio
201+ Disallow: /studio/*
202+ Disallow: /draft
203+ Disallow: /cdn-cgi/l/email-protection
204+ Disallow: /feedback/
205+ Disallow: /retail/solution-brief/
206+ Disallow: /_makeswift/
207+ Disallow: /404
208+ Disallow: /corp/
209+ Disallow: /corp-api/
210+ Disallow: /internal/
211+ Disallow: /scaleai/
212+ Disallow: /&spoof=*
213+ Disallow: /events/transformx-2021/linkedin-auth
214+ Disallow: /events/transformx-2021/ticket-image
215+ Disallow: /events/transformx-2021/ticket/*
216+ Disallow: /events/transformx/linkedin-auth
217+ Disallow: /3d-sensor-fusion/*
218+ Disallow: /careers/legal
219+ Disallow: /careers/legal/*
220+ Disallow: /claims-intelligence
221+ Disallow: /content-understanding
222+ Disallow: /data-labeling
223+ Disallow: /styleguide
224+ Disallow: /federal-video-frame
225+ Disallow: /animations
226+ Disallow: /research
227+ Disallow: /research/*
228+
229+ # WhatsApp
230+ User-agent: WhatsApp
231+ Allow: /
232+ Disallow: /studio
233+ Disallow: /studio/*
234+ Disallow: /draft
235+ Disallow: /cdn-cgi/l/email-protection
236+ Disallow: /feedback/
237+ Disallow: /retail/solution-brief/
238+ Disallow: /_makeswift/
239+ Disallow: /404
240+ Disallow: /corp/
241+ Disallow: /corp-api/
242+ Disallow: /internal/
243+ Disallow: /scaleai/
244+ Disallow: /&spoof=*
245+ Disallow: /events/transformx-2021/linkedin-auth
246+ Disallow: /events/transformx-2021/ticket-image
247+ Disallow: /events/transformx-2021/ticket/*
248+ Disallow: /events/transformx/linkedin-auth
249+ Disallow: /3d-sensor-fusion/*
250+ Disallow: /careers/legal
251+ Disallow: /careers/legal/*
252+ Disallow: /claims-intelligence
253+ Disallow: /content-understanding
254+ Disallow: /data-labeling
255+ Disallow: /styleguide
256+ Disallow: /federal-video-frame
257+ Disallow: /animations
258+ Disallow: /research
259+ Disallow: /research/*
260+
261+ # TelegramBot
262+ User-agent: TelegramBot
263+ Allow: /
264+ Disallow: /studio
265+ Disallow: /studio/*
266+ Disallow: /draft
267+ Disallow: /cdn-cgi/l/email-protection
268+ Disallow: /feedback/
13269  Disallow: /retail/solution-brief/
14270  Disallow: /_makeswift/
15271  Disallow: /404