NeuralCrawl

Scale AI / robots.txt snapshot

← back to scale.com · fetched 2026-10-27T22:52:01Z (10d ago) · HTTP 200 · 8007 bytes · sha256 5309b2e80a6bc532 · raw

final URL: https://scale.com/robots.txt

1# *
2User-agent: *
3Allow: /
4Allow: /api/og
5Disallow: /studio
6Disallow: /studio/*
7Disallow: /draft
8Disallow: /cdn-cgi/l/email-protection
9Disallow: /feedback/
10Disallow: /*?*utm_source=*
11Disallow: /*?*_hstc=*
12Disallow: /*?*
13Disallow: /retail/solution-brief/
14Disallow: /_makeswift/
15Disallow: /404
16Disallow: /corp/
17Disallow: /corp-api/
18Disallow: /internal/
19Disallow: /scaleai/
20Disallow: /&spoof=*
21Disallow: /events/transformx-2021/linkedin-auth
22Disallow: /events/transformx-2021/ticket-image
23Disallow: /events/transformx-2021/ticket/*
24Disallow: /events/transformx/linkedin-auth
25Disallow: /3d-sensor-fusion/*
26Disallow: /careers/legal
27Disallow: /careers/legal/*
28Disallow: /claims-intelligence
29Disallow: /content-understanding
30Disallow: /data-labeling
31Disallow: /styleguide
32Disallow: /federal-video-frame
33Disallow: /animations
34Disallow: /research
35Disallow: /research/*
36
37# Twitterbot
38User-agent: Twitterbot
39Allow: /
40Disallow: /studio
41Disallow: /studio/*
42Disallow: /draft
43Disallow: /cdn-cgi/l/email-protection
44Disallow: /feedback/
45Disallow: /retail/solution-brief/
46Disallow: /_makeswift/
47Disallow: /404
48Disallow: /corp/
49Disallow: /corp-api/
50Disallow: /internal/
51Disallow: /scaleai/
52Disallow: /&spoof=*
53Disallow: /events/transformx-2021/linkedin-auth
54Disallow: /events/transformx-2021/ticket-image
55Disallow: /events/transformx-2021/ticket/*
56Disallow: /events/transformx/linkedin-auth
57Disallow: /3d-sensor-fusion/*
58Disallow: /careers/legal
59Disallow: /careers/legal/*
60Disallow: /claims-intelligence
61Disallow: /content-understanding
62Disallow: /data-labeling
63Disallow: /styleguide
64Disallow: /federal-video-frame
65Disallow: /animations
66Disallow: /research
67Disallow: /research/*
68
69# facebookexternalhit
70User-agent: facebookexternalhit
71Allow: /
72Disallow: /studio
73Disallow: /studio/*
74Disallow: /draft
75Disallow: /cdn-cgi/l/email-protection
76Disallow: /feedback/
77Disallow: /retail/solution-brief/
78Disallow: /_makeswift/
79Disallow: /404
80Disallow: /corp/
81Disallow: /corp-api/
82Disallow: /internal/
83Disallow: /scaleai/
84Disallow: /&spoof=*
85Disallow: /events/transformx-2021/linkedin-auth
86Disallow: /events/transformx-2021/ticket-image
87Disallow: /events/transformx-2021/ticket/*
88Disallow: /events/transformx/linkedin-auth
89Disallow: /3d-sensor-fusion/*
90Disallow: /careers/legal
91Disallow: /careers/legal/*
92Disallow: /claims-intelligence
93Disallow: /content-understanding
94Disallow: /data-labeling
95Disallow: /styleguide
96Disallow: /federal-video-frame
97Disallow: /animations
98Disallow: /research
99Disallow: /research/*
100
101# LinkedInBot
102User-agent: LinkedInBot
103Allow: /
104Disallow: /studio
105Disallow: /studio/*
106Disallow: /draft
107Disallow: /cdn-cgi/l/email-protection
108Disallow: /feedback/
109Disallow: /retail/solution-brief/
110Disallow: /_makeswift/
111Disallow: /404
112Disallow: /corp/
113Disallow: /corp-api/
114Disallow: /internal/
115Disallow: /scaleai/
116Disallow: /&spoof=*
117Disallow: /events/transformx-2021/linkedin-auth
118Disallow: /events/transformx-2021/ticket-image
119Disallow: /events/transformx-2021/ticket/*
120Disallow: /events/transformx/linkedin-auth
121Disallow: /3d-sensor-fusion/*
122Disallow: /careers/legal
123Disallow: /careers/legal/*
124Disallow: /claims-intelligence
125Disallow: /content-understanding
126Disallow: /data-labeling
127Disallow: /styleguide
128Disallow: /federal-video-frame
129Disallow: /animations
130Disallow: /research
131Disallow: /research/*
132
133# Discordbot
134User-agent: Discordbot
135Allow: /
136Disallow: /studio
137Disallow: /studio/*
138Disallow: /draft
139Disallow: /cdn-cgi/l/email-protection
140Disallow: /feedback/
141Disallow: /retail/solution-brief/
142Disallow: /_makeswift/
143Disallow: /404
144Disallow: /corp/
145Disallow: /corp-api/
146Disallow: /internal/
147Disallow: /scaleai/
148Disallow: /&spoof=*
149Disallow: /events/transformx-2021/linkedin-auth
150Disallow: /events/transformx-2021/ticket-image
151Disallow: /events/transformx-2021/ticket/*
152Disallow: /events/transformx/linkedin-auth
153Disallow: /3d-sensor-fusion/*
154Disallow: /careers/legal
155Disallow: /careers/legal/*
156Disallow: /claims-intelligence
157Disallow: /content-understanding
158Disallow: /data-labeling
159Disallow: /styleguide
160Disallow: /federal-video-frame
161Disallow: /animations
162Disallow: /research
163Disallow: /research/*
164
165# Slackbot
166User-agent: Slackbot
167Allow: /
168Disallow: /studio
169Disallow: /studio/*
170Disallow: /draft
171Disallow: /cdn-cgi/l/email-protection
172Disallow: /feedback/
173Disallow: /retail/solution-brief/
174Disallow: /_makeswift/
175Disallow: /404
176Disallow: /corp/
177Disallow: /corp-api/
178Disallow: /internal/
179Disallow: /scaleai/
180Disallow: /&spoof=*
181Disallow: /events/transformx-2021/linkedin-auth
182Disallow: /events/transformx-2021/ticket-image
183Disallow: /events/transformx-2021/ticket/*
184Disallow: /events/transformx/linkedin-auth
185Disallow: /3d-sensor-fusion/*
186Disallow: /careers/legal
187Disallow: /careers/legal/*
188Disallow: /claims-intelligence
189Disallow: /content-understanding
190Disallow: /data-labeling
191Disallow: /styleguide
192Disallow: /federal-video-frame
193Disallow: /animations
194Disallow: /research
195Disallow: /research/*
196
197# Slackbot-LinkExpanding
198User-agent: Slackbot-LinkExpanding
199Allow: /
200Disallow: /studio
201Disallow: /studio/*
202Disallow: /draft
203Disallow: /cdn-cgi/l/email-protection
204Disallow: /feedback/
205Disallow: /retail/solution-brief/
206Disallow: /_makeswift/
207Disallow: /404
208Disallow: /corp/
209Disallow: /corp-api/
210Disallow: /internal/
211Disallow: /scaleai/
212Disallow: /&spoof=*
213Disallow: /events/transformx-2021/linkedin-auth
214Disallow: /events/transformx-2021/ticket-image
215Disallow: /events/transformx-2021/ticket/*
216Disallow: /events/transformx/linkedin-auth
217Disallow: /3d-sensor-fusion/*
218Disallow: /careers/legal
219Disallow: /careers/legal/*
220Disallow: /claims-intelligence
221Disallow: /content-understanding
222Disallow: /data-labeling
223Disallow: /styleguide
224Disallow: /federal-video-frame
225Disallow: /animations
226Disallow: /research
227Disallow: /research/*
228
229# WhatsApp
230User-agent: WhatsApp
231Allow: /
232Disallow: /studio
233Disallow: /studio/*
234Disallow: /draft
235Disallow: /cdn-cgi/l/email-protection
236Disallow: /feedback/
237Disallow: /retail/solution-brief/
238Disallow: /_makeswift/
239Disallow: /404
240Disallow: /corp/
241Disallow: /corp-api/
242Disallow: /internal/
243Disallow: /scaleai/
244Disallow: /&spoof=*
245Disallow: /events/transformx-2021/linkedin-auth
246Disallow: /events/transformx-2021/ticket-image
247Disallow: /events/transformx-2021/ticket/*
248Disallow: /events/transformx/linkedin-auth
249Disallow: /3d-sensor-fusion/*
250Disallow: /careers/legal
251Disallow: /careers/legal/*
252Disallow: /claims-intelligence
253Disallow: /content-understanding
254Disallow: /data-labeling
255Disallow: /styleguide
256Disallow: /federal-video-frame
257Disallow: /animations
258Disallow: /research
259Disallow: /research/*
260
261# TelegramBot
262User-agent: TelegramBot
263Allow: /
264Disallow: /studio
265Disallow: /studio/*
266Disallow: /draft
267Disallow: /cdn-cgi/l/email-protection
268Disallow: /feedback/
269Disallow: /retail/solution-brief/
270Disallow: /_makeswift/
271Disallow: /404
272Disallow: /corp/
273Disallow: /corp-api/
274Disallow: /internal/
275Disallow: /scaleai/
276Disallow: /&spoof=*
277Disallow: /events/transformx-2021/linkedin-auth
278Disallow: /events/transformx-2021/ticket-image
279Disallow: /events/transformx-2021/ticket/*
280Disallow: /events/transformx/linkedin-auth
281Disallow: /3d-sensor-fusion/*
282Disallow: /careers/legal
283Disallow: /careers/legal/*
284Disallow: /claims-intelligence
285Disallow: /content-understanding
286Disallow: /data-labeling
287Disallow: /styleguide
288Disallow: /federal-video-frame
289Disallow: /animations
290Disallow: /research
291Disallow: /research/*
292
293# Host
294Host: https://scale.com
295
296# Sitemaps
297Sitemap: https://scale.com/sitemap.xml
298Sitemap: https://scale.com/careers/sitemap.xml
299Sitemap: https://scale.com/blog/sitemap.xml
300Sitemap: https://scale.com/blog/category/sitemap.xml
301Sitemap: https://scale.com/blog/author/sitemap.xml
302Sitemap: https://scale.com/customers/sitemap.xml
303Sitemap: https://scale.com/guides/sitemap.xml
304Sitemap: https://scale.com/partnership/sitemap.xml
305Sitemap: https://scale.com/government-innovation/sitemap.xml
306Sitemap: https://scale.com/legal/sitemap.xml
307Sitemap: https://scale.com/events/sitemap.xml
308Sitemap: https://scale.com/docs/sitemap.xml
309Sitemap: https://brand.scale.com/sitemap.xml
310Sitemap: https://labs.scale.com/sitemap.xml