NeuralCrawl

Indeed / robots.txt snapshot

← back to indeed.com · fetched 2026-06-30T18:10:20Z (18d ago) · HTTP 200 · 12493 bytes · sha256 776e04c63e7f30d7 · raw

final URL: https://www.indeed.com/robots.txt

1User-agent: *
2Allow: /
3Disallow: /*rt=nc
4Disallow: /*&alid=
5Disallow: /*&calert=
6Disallow: /*&iafilter=
7Disallow: /*&mna=
8Disallow: /*?rss
9Disallow: /*&serpstart=
10Disallow: /*/vaclk
11Disallow: /addlLoc/
12Disallow: /ads/
13Disallow: /advanced_search
14Disallow: /alert
15Disallow: /api/fetch/mc-anon
16Disallow: /api/getrecjobs
17Disallow: /applystart
18Disallow: /cmp/_/
19Disallow: /cmp/_c/claim/*
20Disallow: /cmp/_rpc/
21Disallow: /cmp/addlink
22Disallow: /cmp/addvideo
23Disallow: /cmp/Login*
24Disallow: /cmp/*/analytics
25Disallow: /cmp/*/company-questions
26Disallow: /cmp/*/people
27Disallow: /cmp/*/write-review
28Disallow: /community/
29Disallow: /company/*
30Disallow: /conversion/
31Disallow: /cookiemigrator/
32Disallow: /create-resume/lp/
33Disallow: /cdn-cgi/
34Disallow: /%E4%BB%95%E4%BA%8B?
35Disallow: /%E5%B7%A5%E4%BD%9C/
36Disallow: /%E6%B1%82%E4%BA%BA/JP/
37Disallow: /%E6%B1%82%E4%BA%BA/title/
38Disallow: /%E8%81%8C%E4%BD%8D%E6%98%BE%E7%A4%BA?
39Disallow: /%EC%B7%A8%EC%97%85/KR/
40Disallow: /%EC%B7%A8%EC%97%85/title/
41Disallow: /empleo/
42Disallow: /emploi/
43Disallow: /emplois/FR/
44Disallow: /emplois/title/
45Disallow: /emprego/
46Disallow: /empregos/BR/
47Disallow: /empregos/title
48Disallow: /forum/profile
49Disallow: /g/
50Disallow: /graphql
51Disallow: /imgping
52Disallow: /ita?
53Disallow: /ita/v1/publisher
54Disallow: /ja/clk?
55Disallow: /ja/imp.gif
56Disallow: /job/
57Disallow: /Job/
58Disallow: /jobb/
59Disallow: /jobroll
60Disallow: /jobs/AE/
61Disallow: /jobs/AQ/
62Disallow: /Jobs/AT/
63Disallow: /jobs/AU/
64Disallow: /jobs/BE/
65Disallow: /jobs/BH/
66Disallow: /jobs/CA/
67Disallow: /jobs/CZ/
68Disallow: /jobs/DE/
69Disallow: /Jobs/DE/
70Disallow: /jobs/DK/
71Disallow: /jobs/FI/
72Disallow: /jobs/GB/
73Disallow: /jobs/GR/
74Disallow: /jobs/HK/
75Disallow: /jobs/HU/
76Disallow: /jobs/ID/
77Disallow: /jobs/IE/
78Disallow: /jobs/IL/
79Disallow: /jobs/IN/
80Disallow: /jobs/KW/
81Disallow: /jobs/LU/
82Disallow: /jobs/MY/
83Disallow: /jobs/NO/
84Disallow: /jobs/NZ/
85Disallow: /jobs/OM/
86Disallow: /jobs/PE/
87Disallow: /jobs/PH/
88Disallow: /jobs/PK/
89Disallow: /jobs/PT/
90Disallow: /jobs/QA/
91Disallow: /jobs/RO/
92Disallow: /jobs/RU/
93Disallow: /jobs/SA/
94Disallow: /jobs/SG/
95Disallow: /jobs/title
96Disallow: /Jobs/title
97Disallow: /jobs/TR/
98Disallow: /jobs/TW/
99Disallow: /jobs/US/
100Disallow: /jobs/VE/
101Disallow: /jobs/ZA/
102Disallow: /jobtrends/trends/log/*
103Disallow: /jwidget.js*
104Disallow: /me/*/pdf
105Disallow: /m/basecamp/analytics.js
106Disallow: /m/jobalerts?
107Disallow: /m/moreLoc
108Disallow: /m/newjobs
109Disallow: /m/recommended
110Disallow: /m/rpc/
111Disallow: /m/viewjob?
112Disallow: /my/
113Disallow: /*oc=1
114Disallow: /ofertas/ES/
115Disallow: /ofertas/title/
116Disallow: /offerta-lavoro
117Disallow: /offerte-lavoro/IT/
118Disallow: /offerte-lavoro/title
119Disallow: /pagead/
120Disallow: /poka%C5%BCprac%C4%99?
121Disallow: /praca/
122Disallow: /preferences
123Disallow: /*radius=
124Disallow: /rc/
125Disallow: /rdr/
126Disallow: /recommendedjobs
127Disallow: /record-page-tags
128Disallow: /resumes/account
129Disallow: /resumes/advanced?
130Disallow: /resumes/alert
131Disallow: /resumes*radius=
132Disallow: /resumes*rb=
133Disallow: /resumes*res=
134Disallow: /resumes/rpc/
135Disallow: /resumes*sort=
136Disallow: /resumes*start=
137Disallow: /rpc/
138Disallow: /r/*/pdf
139Disallow: /rss
140Disallow: /setprefs
141Disallow: /*sid=
142Disallow: /*sp=0
143Disallow: /*&start=
144Disallow: /stc
145Disallow: /Stellen/CH/
146Disallow: /Stellen/title
147Disallow: /tellafriend
148Disallow: /tmn/ccs/
149Disallow: /tos/banner.js
150Disallow: /trabajo/
151Disallow: /uie/clk
152Disallow: /url
153Disallow: /vacature/
154Disallow: /vacature-bekijken?
155Disallow: /vacatures/NL/
156Disallow: /vacatures/title/
157Disallow: /ver-empleo?
158Disallow: /ver-emprego?
159Disallow: /ver-oferta?
160Disallow: /viewjob?
161Disallow: /visajobb?
162Disallow: /voir-emploi?
163Disallow: /Zeige-Job?
164Disallow: //applystart
165
166
167User-agent: Googlebot
168User-agent: Google-Extended
169User-agent: ChatGPT-User
170User-agent: Bingbot
171User-agent: Slurp
172User-agent: DuckDuckBot
173User-agent: YahooJP
174User-agent: PetalBot
175User-agent: PerplexityBot
176User-agent: DuckAssistBot
177User-agent: Perplexity-User
178User-agent: YouBot
179User-agent: Claude-User
180User-agent: Gemini-Deep-Research
181User-agent: Claude-SearchBot
182User-agent: OAI-SearchBot
183User-agent: Google-Agent
184User-agent: Google-CloudVertexBot
185User-agent: Meta-WebIndexer
186User-agent: Meta-ExternalFetcher
187User-agent: xAI-Grok
188User-agent: Grok-DeepSearch
189Allow: /
190Disallow: /*rt=nc
191Disallow: /*&alid=
192Disallow: /*&calert=
193Disallow: /*&iafilter=
194Disallow: /*&mna=
195Disallow: /*?rss
196Disallow: /*&serpstart=
197Disallow: /*/vaclk
198Disallow: /addlLoc/
199Disallow: /ads/
200Disallow: /advanced_search
201Disallow: /alert
202Disallow: /api/fetch/mc-anon
203Disallow: /api/getrecjobs
204Disallow: /applystart
205Disallow: /cmp/_/
206Disallow: /cmp/_c/claim/*
207Disallow: /cmp/_rpc/
208Disallow: /cmp/addlink
209Disallow: /cmp/addvideo
210Disallow: /cmp/Login*
211Disallow: /cmp/*/analytics
212Disallow: /cmp/*/company-questions
213Disallow: /cmp/*/people
214Disallow: /cmp/*/write-review
215Disallow: /community/
216Disallow: /company/*
217Disallow: /conversion/
218Disallow: /cookiemigrator/
219Disallow: /create-resume/lp/
220Disallow: /cdn-cgi/
221Disallow: /%E4%BB%95%E4%BA%8B?
222Disallow: /%E5%B7%A5%E4%BD%9C/
223Disallow: /%E6%B1%82%E4%BA%BA/JP/
224Disallow: /%E6%B1%82%E4%BA%BA/title/
225Disallow: /%E8%81%8C%E4%BD%8D%E6%98%BE%E7%A4%BA?
226Disallow: /%EC%B7%A8%EC%97%85/KR/
227Disallow: /%EC%B7%A8%EC%97%85/title/
228Disallow: /empleo/
229Disallow: /emploi/
230Disallow: /emplois/FR/
231Disallow: /emplois/title/
232Disallow: /emprego/
233Disallow: /empregos/BR/
234Disallow: /empregos/title
235Disallow: /forum/profile
236Disallow: /g/
237Disallow: /generate-question
238Disallow: /graphql
239Disallow: /imgping
240Disallow: /ita?
241Disallow: /ita/v1/publisher
242Disallow: /ja/clk?
243Disallow: /ja/imp.gif
244Disallow: /job/
245Disallow: /Job/
246Disallow: /jobb/
247Disallow: /jobroll
248Disallow: /jobs/AE/
249Disallow: /jobs/AQ/
250Disallow: /Jobs/AT/
251Disallow: /jobs/AU/
252Disallow: /jobs/BE/
253Disallow: /jobs/BH/
254Disallow: /jobs/CA/
255Disallow: /jobs/CZ/
256Disallow: /jobs/DE/
257Disallow: /Jobs/DE/
258Disallow: /jobs/DK/
259Disallow: /jobs/FI/
260Disallow: /jobs/GB/
261Disallow: /jobs/GR/
262Disallow: /jobs/HK/
263Disallow: /jobs/HU/
264Disallow: /jobs/ID/
265Disallow: /jobs/IE/
266Disallow: /jobs/IL/
267Disallow: /jobs/IN/
268Disallow: /jobs/KW/
269Disallow: /jobs/LU/
270Disallow: /jobs/MY/
271Disallow: /jobs/NO/
272Disallow: /jobs/NZ/
273Disallow: /jobs/OM/
274Disallow: /jobs/PE/
275Disallow: /jobs/PH/
276Disallow: /jobs/PK/
277Disallow: /jobs/PT/
278Disallow: /jobs/QA/
279Disallow: /jobs/RO/
280Disallow: /jobs/RU/
281Disallow: /jobs/SA/
282Disallow: /jobs/SG/
283Disallow: /jobs/title
284Disallow: /Jobs/title
285Disallow: /jobs/TR/
286Disallow: /jobs/TW/
287Disallow: /jobs/US/
288Disallow: /jobs/VE/
289Disallow: /jobs/ZA/
290Disallow: /jobtrends/trends/log/*
291Disallow: /jwidget.js*
292Disallow: /me/*/pdf
293Disallow: /m/basecamp/analytics.js
294Disallow: /m/jobalerts?
295Disallow: /m/moreLoc
296Disallow: /m/newjobs
297Disallow: /m/recommended
298Disallow: /m/rpc/
299Disallow: /my/
300Disallow: /*oc=1
301Disallow: /ofertas/ES/
302Disallow: /ofertas/title/
303Disallow: /offerte-lavoro/IT/
304Disallow: /offerte-lavoro/title
305Disallow: /pagead/
306Disallow: /poka%C5%BCprac%C4%99?
307Disallow: /praca/
308Disallow: /preferences
309Disallow: /*radius=
310Disallow: /rc/
311Disallow: /rdr/
312Disallow: /recommendedjobs
313Disallow: /record-page-tags
314Disallow: /resumes/account
315Disallow: /resumes/advanced?
316Disallow: /resumes/alert
317Disallow: /resumes*radius=
318Disallow: /resumes*rb=
319Disallow: /resumes*res=
320Disallow: /resumes/rpc/
321Disallow: /resumes*sort=
322Disallow: /resumes*start=
323Disallow: /rpc/
324Disallow: /r/*/pdf
325Disallow: /rss
326Disallow: /setprefs
327Disallow: /*sid=
328Disallow: /*sp=0
329Disallow: /*&start=
330Disallow: /stc
331Disallow: /Stellen/CH/
332Disallow: /Stellen/title
333Disallow: /tellafriend
334Disallow: /tmn/ccs/
335Disallow: /tos/banner.js
336Disallow: /trabajo/
337Disallow: /uie/clk
338Disallow: /url
339Disallow: /v1/alerts
340Disallow: /v1/impressions
341Disallow: /vacature/
342Disallow: /vacatures/NL/
343Disallow: /vacatures/title/
344Disallow: //applystart
345Disallow: /hire/o/ats-partners
346Disallow: /hire/ats-partners
347Disallow: /offers
348
349
350User-agent: Mediapartners-Google
351Allow: /
352
353
354User-agent: AdsBot-Google
355User-agent: AdsBot-Google-Mobile
356Allow: /create-resume/lp/
357Allow: /*sid=
358Disallow: /rpc/
359
360
361User-agent: adidxbot
362Allow: /*sid=
363
364
365User-agent: Googlebot-Image
366User-agent: msnbot-media
367Disallow: /trendgraph/*
368
369
370User-Agent: OmniExplorer_Bot
371User-agent: ia_archiver
372User-agent: 008
373User-agent: JobdiggerSpider
374User-agent: Cliqzbot
375User-agent: GoogleOther
376User-agent: omgili
377User-agent: omgilibot
378User-agent: Webzio
379User-agent: Webzio-Extended
380User-agent: Scrapy
381User-agent: MyCentralAIScraperBot
382Disallow: /
383
384
385User-agent: facebookexternalhit
386User-agent: WhatsApp
387User-agent: snapchat
388User-agent: TelegramBot
389User-agent: Twitterbot
390User-agent: Slackbot-LinkExpanding
391Allow: /
392
393
394User-agent: GPTBot
395User-agent: CCBot
396User-agent: anthropic-ai
397User-agent: FacebookBot
398User-agent: AmazonBot
399User-agent: Applebot-Extended
400User-agent: Bytespider
401User-agent: Baiduspider
402User-agent: cohere-training-data-crawler
403User-agent: FriendlyCrawler
404User-agent: img2dataset
405User-agent: Meta-ExternalAgent
406User-agent: ClaudeBot
407User-agent: DeepSeekBot
408User-agent: GrokBot
409User-agent: Diffbot
410User-agent: AI2Bot
411User-agent: PanguBot
412User-agent: Timpibot
413Disallow: /*rt=nc
414Disallow: /*&alid=
415Disallow: /*&calert=
416Disallow: /*&iafilter=
417Disallow: /*&mna=
418Disallow: /*?rss
419Disallow: /*&serpstart=
420Disallow: /*/vaclk
421Disallow: /addlLoc/
422Disallow: /ads/
423Disallow: /advanced_search
424Disallow: /alert
425Disallow: /api/fetch/mc-anon
426Disallow: /api/getrecjobs
427Disallow: /applystart
428Disallow: /community/
429Disallow: /company/*
430Disallow: /conversion/
431Disallow: /cookiemigrator/
432Disallow: /create-resume/lp/
433Disallow: /cdn-cgi/
434Disallow: /%E4%BB%95%E4%BA%8B?
435Disallow: /%E6%B1%82%E4%BA%BA/JP/
436Disallow: /%E6%B1%82%E4%BA%BA/title/
437Disallow: /%E8%81%8C%E4%BD%8D%E6%98%BE%E7%A4%BA?
438Disallow: /%EC%B7%A8%EC%97%85/KR/
439Disallow: /%EC%B7%A8%EC%97%85/title/
440Disallow: /empleo/
441Disallow: /emploi/
442Disallow: /emplois/FR/
443Disallow: /emplois/title/
444Disallow: /emprego/
445Disallow: /forum/profile
446Disallow: /g/
447Disallow: /graphql
448Disallow: /imgping
449Disallow: /ita?
450Disallow: /ita/v1/publisher
451Disallow: /ja/clk?
452Disallow: /ja/imp.gif
453Disallow: /job/
454Disallow: /Job/
455Disallow: /jobroll
456Disallow: /jobtrends/trends/log/*
457Disallow: /jwidget.js*
458Disallow: /me/*/pdf
459Disallow: /m/basecamp/analytics.js
460Disallow: /m/jobalerts?
461Disallow: /m/moreLoc
462Disallow: /m/newjobs
463Disallow: /m/recommended
464Disallow: /m/rpc/
465Disallow: /my/
466Disallow: /*oc=1
467Disallow: /pagead/
468Disallow: /poka%C5%BCprac%C4%99?
469Disallow: /preferences
470Disallow: /*radius=
471Disallow: /rc/
472Disallow: /rdr/
473Disallow: /recommendedjobs
474Disallow: /record-page-tags
475Disallow: /resumes/account
476Disallow: /resumes/advanced?
477Disallow: /resumes/alert
478Disallow: /resumes*radius=
479Disallow: /resumes*rb=
480Disallow: /resumes*res=
481Disallow: /resumes/rpc/
482Disallow: /resumes*sort=
483Disallow: /resumes*start=
484Disallow: /rpc/
485Disallow: /r/*/pdf
486Disallow: /rss
487Disallow: /setprefs
488Disallow: /*sid=
489Disallow: /*sp=0
490Disallow: /*&start=
491Disallow: /stc
492Disallow: /Stellen/CH/
493Disallow: /Stellen/title
494Disallow: /tellafriend
495Disallow: /tmn/ccs/
496Disallow: /tos/banner.js
497Disallow: /uie/clk
498Disallow: /url
499Disallow: /vacature/
500Disallow: /vacature-bekijken?
501Disallow: /ver-empleo?
502Disallow: /ver-emprego?
503Disallow: /ver-oferta?
504Disallow: /visajobb?
505Disallow: /voir-emploi?
506Disallow: /Zeige-Job?
507Disallow: /c/info/
508Disallow: /hire/how-to-hire/
509Disallow: /hire/interview-questions/
510Disallow: /recruiting/interviewfragen/
511Disallow: /recruiting/stellenbeschreibung/
512Disallow: /recrutement/description-du-poste/
513Disallow: /recrutement/questions-entretien-dembauche/
514Disallow: /personeel/functiebeschrijving/
515Disallow: /companies/
516Disallow: /cmp/
517Disallow: /career-advice/
518Disallow: /conselho-de-carreira/
519Disallow: /hire/job-description/
520Disallow: /karriere-guide/
521Disallow: /conseils-carrieres/
522Disallow: /conseils-carriere/
523Disallow: /orientacion-profesional/
524Disallow: /porady-zawodowe/
525Disallow: /guida-alla-carriera/
526Disallow: /職涯貼士/
527Disallow: /carrieregids/
528Disallow: /karriarrad/
529Disallow: /orientacion-laboral/
530Disallow: /career/
531Disallow: /*jobs.html
532Disallow: /jobs
533Disallow: /viewjob
534Disallow: /certifications/
535Disallow: /trabajo
536Disallow: /ofertas
537Disallow: /%D0%B2%D0%B0%D0%BA%D0%B0%D0%BD%D1%81%D0%B8%D0%B8
538Disallow: /emplois
539Disallow: /%E6%B1%82%E4%BA%BA
540Disallow: /m/jobs
541Disallow: /m/viewjob
542Disallow: /vacatures
543Disallow: /offerte-lavoro
544Disallow: /Vi%E1%BB%87c-l%C3%A0m
545Disallow: /Jobs
546Disallow: /%D9%88%D8%B8%D8%A7%D8%A6%D9%81
547Disallow: /%EC%B7%A8%EC%97%85
548Disallow: /Jobb
549Disallow: /empregos
550Disallow: /praca
551Disallow: /%E5%B7%A5%E4%BD%9C
552Disallow: /jobb
553Disallow: /%E0%B8%87%E0%B8%B2%E0%B8%99
554Disallow: /lowongan-kerja
555Disallow: /Stellen
556Disallow: /q-
557Disallow: /l-