NeuralCrawl

Indeed / robots.txt snapshot

← back to indeed.com · fetched 2026-10-07T21:56:40Z (30d ago) · HTTP 200 · 13724 bytes · sha256 3ec692c8e27e5626 · raw

final URL: https://www.indeed.com/robots.txt

1# Default rules for all unspecified bots. Note that these apply to Bingbot too.
2User-agent: *
3Allow: /
4Allow: /*&start=0&
5Allow: /*&start=10&
6Allow: /*&start=20&
7Allow: /*&start=30&
8Allow: /*&start=40&
9Allow: /*&start=50&
10Allow: /*&start=60&
11Allow: /*&start=70&
12Allow: /*&start=80&
13Allow: /*&start=90&
14Disallow: /*rt=nc
15Disallow: /*&alid=
16Disallow: /*&calert=
17Disallow: /*&iafilter=
18Disallow: /*&mna=
19Disallow: /*?rss
20Disallow: /*&serpstart=
21Disallow: /*/vaclk
22Disallow: /addlLoc/
23Disallow: /ads/
24Disallow: /advanced_search
25Disallow: /alert
26Disallow: /api/fetch/mc-anon
27Disallow: /api/getrecjobs
28Disallow: /applystart
29Disallow: /cmp/_/
30Disallow: /cmp/_c/claim/*
31Disallow: /cmp/_rpc/
32Disallow: /cmp/addlink
33Disallow: /cmp/addvideo
34Disallow: /cmp/Login*
35Disallow: /cmp/*/analytics
36Disallow: /cmp/*/company-questions
37Disallow: /cmp/*/people
38Disallow: /cmp/*/write-review
39Disallow: /community/
40Disallow: /company/*
41Disallow: /conversion/
42Disallow: /cookiemigrator/
43Disallow: /create-resume/lp/
44Disallow: /cdn-cgi/
45Disallow: /%E4%BB%95%E4%BA%8B?
46Disallow: /%E5%B7%A5%E4%BD%9C/
47Disallow: /%E6%B1%82%E4%BA%BA/JP/
48Disallow: /%E6%B1%82%E4%BA%BA/title/
49Disallow: /%E8%81%8C%E4%BD%8D%E6%98%BE%E7%A4%BA?
50Disallow: /%EC%B7%A8%EC%97%85/KR/
51Disallow: /%EC%B7%A8%EC%97%85/title/
52Disallow: /empleo/
53Disallow: /emploi/
54Disallow: /emplois/FR/
55Disallow: /emplois/title/
56Disallow: /emprego/
57Disallow: /empregos/BR/
58Disallow: /empregos/title
59Disallow: /forum/profile
60Disallow: /g/
61Disallow: /generate-question
62Disallow: /graphql
63Disallow: /imgping
64Disallow: /ita?
65Disallow: /ita/v1/publisher
66Disallow: /ja/clk?
67Disallow: /ja/imp.gif
68Disallow: /job/
69Disallow: /Job/
70Disallow: /jobb/
71Disallow: /jobroll
72Disallow: /jobs/AE/
73Disallow: /jobs/AQ/
74Disallow: /Jobs/AT/
75Disallow: /jobs/AU/
76Disallow: /jobs/BE/
77Disallow: /jobs/BH/
78Disallow: /jobs/CA/
79Disallow: /jobs/CZ/
80Disallow: /jobs/DE/
81Disallow: /Jobs/DE/
82Disallow: /jobs/DK/
83Disallow: /jobs/FI/
84Disallow: /jobs/GB/
85Disallow: /jobs/GR/
86Disallow: /jobs/HK/
87Disallow: /jobs/HU/
88Disallow: /jobs/ID/
89Disallow: /jobs/IE/
90Disallow: /jobs/IL/
91Disallow: /jobs/IN/
92Disallow: /jobs/KW/
93Disallow: /jobs/LU/
94Disallow: /jobs/MY/
95Disallow: /jobs/NO/
96Disallow: /jobs/NZ/
97Disallow: /jobs/OM/
98Disallow: /jobs/PE/
99Disallow: /jobs/PH/
100Disallow: /jobs/PK/
101Disallow: /jobs/PT/
102Disallow: /jobs/QA/
103Disallow: /jobs/RO/
104Disallow: /jobs/RU/
105Disallow: /jobs/SA/
106Disallow: /jobs/SG/
107Disallow: /jobs/title
108Disallow: /Jobs/title
109Disallow: /jobs/TR/
110Disallow: /jobs/TW/
111Disallow: /jobs/US/
112Disallow: /jobs/VE/
113Disallow: /jobs/ZA/
114Disallow: /jobtrends/trends/log/*
115Disallow: /jwidget.js*
116Disallow: /me/*/pdf
117Disallow: /m/basecamp/analytics.js
118Disallow: /m/jobalerts?
119Disallow: /m/moreLoc
120Disallow: /m/newjobs
121Disallow: /m/recommended
122Disallow: /m/rpc/
123Disallow: /m/viewjob
124Disallow: /my/
125Disallow: /*oc=1
126Disallow: /ofertas/ES/
127Disallow: /ofertas/title/
128Disallow: /offerta-lavoro
129Disallow: /offerte-lavoro/IT/
130Disallow: /offerte-lavoro/title
131Disallow: /pagead/
132Disallow: /poka%C5%BCprac%C4%99?
133Disallow: /praca/
134Disallow: /preferences
135Disallow: /promo/
136Disallow: /*radius=
137Disallow: /rc/
138Disallow: /rdr/
139Disallow: /recommendedjobs
140Disallow: /record-page-tags
141Disallow: /resumes/account
142Disallow: /resumes/advanced?
143Disallow: /resumes/alert
144Disallow: /resumes*radius=
145Disallow: /resumes*rb=
146Disallow: /resumes*res=
147Disallow: /resumes/rpc/
148Disallow: /resumes*sort=
149Disallow: /resumes*start=
150Disallow: /rpc/
151Disallow: /r/*/pdf
152Disallow: /rss
153Disallow: /setprefs
154Disallow: /*sid=
155Disallow: /*sp=0
156Disallow: /*&start=
157Disallow: /stc
158Disallow: /Stellen/CH/
159Disallow: /Stellen/title
160Disallow: /tellafriend
161Disallow: /tmn/ccs/
162Disallow: /tos/banner.js
163Disallow: /trabajo/
164Disallow: /uie/clk
165Disallow: /url
166Disallow: /v1/alerts
167Disallow: /v1/impressions
168Disallow: /vacature/
169Disallow: /vacature-bekijken
170Disallow: /vacatures/NL/
171Disallow: /vacatures/title/
172Disallow: /ver-empleo
173Disallow: /ver-emprego
174Disallow: /ver-oferta
175Disallow: /viewjob
176Disallow: /visajobb
177Disallow: /voir-emploi
178Disallow: /Zeige-Job
179Disallow: //applystart
180Disallow: /hire/o/ats-partners
181Disallow: /hire/ats-partners
182Disallow: /offers
183
184
185# Rules for traditional search engines and for retrieval/grounding bots.
186User-agent: Googlebot
187User-agent: Google-Extended
188User-agent: ChatGPT-User
189User-agent: Slurp
190User-agent: DuckDuckBot
191User-agent: YahooJP
192User-agent: PetalBot
193User-agent: PerplexityBot
194User-agent: DuckAssistBot
195User-agent: Perplexity-User
196User-agent: YouBot
197User-agent: Claude-User
198User-agent: Gemini-Deep-Research
199User-agent: Claude-SearchBot
200User-agent: OAI-SearchBot
201User-agent: Google-Agent
202User-agent: Google-CloudVertexBot
203User-agent: Meta-WebIndexer
204User-agent: Meta-ExternalFetcher
205User-agent: xAI-Grok
206User-agent: Grok-DeepSearch
207Allow: /
208Allow: /*&start=0&
209Allow: /*&start=10&
210Allow: /*&start=20&
211Allow: /*&start=30&
212Allow: /*&start=40&
213Allow: /*&start=50&
214Allow: /*&start=60&
215Allow: /*&start=70&
216Allow: /*&start=80&
217Allow: /*&start=90&
218Disallow: /*rt=nc
219Disallow: /*&alid=
220Disallow: /*&calert=
221Disallow: /*&iafilter=
222Disallow: /*&mna=
223Disallow: /*?rss
224Disallow: /*&serpstart=
225Disallow: /*/vaclk
226Disallow: /addlLoc/
227Disallow: /ads/
228Disallow: /advanced_search
229Disallow: /alert
230Disallow: /api/fetch/mc-anon
231Disallow: /api/getrecjobs
232Disallow: /applystart
233Disallow: /cmp/_/
234Disallow: /cmp/_c/claim/*
235Disallow: /cmp/_rpc/
236Disallow: /cmp/addlink
237Disallow: /cmp/addvideo
238Disallow: /cmp/Login*
239Disallow: /cmp/*/analytics
240Disallow: /cmp/*/company-questions
241Disallow: /cmp/*/people
242Disallow: /cmp/*/write-review
243Disallow: /community/
244Disallow: /company/*
245Disallow: /conversion/
246Disallow: /cookiemigrator/
247Disallow: /create-resume/lp/
248Disallow: /cdn-cgi/
249Disallow: /%E4%BB%95%E4%BA%8B?
250Disallow: /%E5%B7%A5%E4%BD%9C/
251Disallow: /%E6%B1%82%E4%BA%BA/JP/
252Disallow: /%E6%B1%82%E4%BA%BA/title/
253Disallow: /%E8%81%8C%E4%BD%8D%E6%98%BE%E7%A4%BA?
254Disallow: /%EC%B7%A8%EC%97%85/KR/
255Disallow: /%EC%B7%A8%EC%97%85/title/
256Disallow: /empleo/
257Disallow: /emploi/
258Disallow: /emplois/FR/
259Disallow: /emplois/title/
260Disallow: /emprego/
261Disallow: /empregos/BR/
262Disallow: /empregos/title
263Disallow: /forum/profile
264Disallow: /g/
265Disallow: /generate-question
266Disallow: /graphql
267Disallow: /imgping
268Disallow: /ita?
269Disallow: /ita/v1/publisher
270Disallow: /ja/clk?
271Disallow: /ja/imp.gif
272Disallow: /job/
273Disallow: /Job/
274Disallow: /jobb/
275Disallow: /jobroll
276Disallow: /jobs/AE/
277Disallow: /jobs/AQ/
278Disallow: /Jobs/AT/
279Disallow: /jobs/AU/
280Disallow: /jobs/BE/
281Disallow: /jobs/BH/
282Disallow: /jobs/CA/
283Disallow: /jobs/CZ/
284Disallow: /jobs/DE/
285Disallow: /Jobs/DE/
286Disallow: /jobs/DK/
287Disallow: /jobs/FI/
288Disallow: /jobs/GB/
289Disallow: /jobs/GR/
290Disallow: /jobs/HK/
291Disallow: /jobs/HU/
292Disallow: /jobs/ID/
293Disallow: /jobs/IE/
294Disallow: /jobs/IL/
295Disallow: /jobs/IN/
296Disallow: /jobs/KW/
297Disallow: /jobs/LU/
298Disallow: /jobs/MY/
299Disallow: /jobs/NO/
300Disallow: /jobs/NZ/
301Disallow: /jobs/OM/
302Disallow: /jobs/PE/
303Disallow: /jobs/PH/
304Disallow: /jobs/PK/
305Disallow: /jobs/PT/
306Disallow: /jobs/QA/
307Disallow: /jobs/RO/
308Disallow: /jobs/RU/
309Disallow: /jobs/SA/
310Disallow: /jobs/SG/
311Disallow: /jobs/title
312Disallow: /Jobs/title
313Disallow: /jobs/TR/
314Disallow: /jobs/TW/
315Disallow: /jobs/US/
316Disallow: /jobs/VE/
317Disallow: /jobs/ZA/
318Disallow: /jobtrends/trends/log/*
319Disallow: /jwidget.js*
320Disallow: /me/*/pdf
321Disallow: /m/basecamp/analytics.js
322Disallow: /m/jobalerts?
323Disallow: /m/moreLoc
324Disallow: /m/newjobs
325Disallow: /m/recommended
326Disallow: /m/rpc/
327Disallow: /my/
328Disallow: /*oc=1
329Disallow: /ofertas/ES/
330Disallow: /ofertas/title/
331Disallow: /offerte-lavoro/IT/
332Disallow: /offerte-lavoro/title
333Disallow: /pagead/
334Disallow: /poka%C5%BCprac%C4%99?
335Disallow: /praca/
336Disallow: /preferences
337Disallow: /promo/
338Disallow: /*radius=
339Disallow: /rc/
340Disallow: /rdr/
341Disallow: /recommendedjobs
342Disallow: /record-page-tags
343Disallow: /resumes/account
344Disallow: /resumes/advanced?
345Disallow: /resumes/alert
346Disallow: /resumes*radius=
347Disallow: /resumes*rb=
348Disallow: /resumes*res=
349Disallow: /resumes/rpc/
350Disallow: /resumes*sort=
351Disallow: /resumes*start=
352Disallow: /rpc/
353Disallow: /r/*/pdf
354Disallow: /rss
355Disallow: /setprefs
356Disallow: /*sid=
357Disallow: /*sp=0
358Disallow: /*&start=
359Disallow: /stc
360Disallow: /Stellen/CH/
361Disallow: /Stellen/title
362Disallow: /tellafriend
363Disallow: /tmn/ccs/
364Disallow: /tos/banner.js
365Disallow: /trabajo/
366Disallow: /uie/clk
367Disallow: /url
368Disallow: /v1/alerts
369Disallow: /v1/impressions
370Disallow: /vacature/
371Disallow: /vacatures/NL/
372Disallow: /vacatures/title/
373Disallow: //applystart
374Disallow: /hire/o/ats-partners
375Disallow: /hire/ats-partners
376Disallow: /offers
377
378
379User-agent: Mediapartners-Google
380Allow: /
381
382
383User-agent: AdsBot-Google
384User-agent: AdsBot-Google-Mobile
385Allow: /create-resume/lp/
386Allow: /*sid=
387Disallow: /rpc/
388
389
390User-agent: adidxbot
391Allow: /*sid=
392
393
394User-agent: Googlebot-Image
395User-agent: msnbot-media
396Disallow: /trendgraph/*
397
398
399# Full disallow for aggressive scrapers, legacy crawlers, and unwanted utility bots.
400User-Agent: OmniExplorer_Bot
401User-agent: ia_archiver
402User-agent: 008
403User-agent: JobdiggerSpider
404User-agent: Cliqzbot
405User-agent: GoogleOther
406User-agent: omgili
407User-agent: omgilibot
408User-agent: Webzio
409User-agent: Webzio-Extended
410User-agent: Scrapy
411User-agent: MyCentralAIScraperBot
412Disallow: /
413
414
415# Rules for Social Media / Link Preview bots.
416User-agent: facebookexternalhit
417User-agent: WhatsApp
418User-agent: snapchat
419User-agent: TelegramBot
420User-agent: Twitterbot
421User-agent: Slackbot-LinkExpanding
422Allow: /
423
424
425# Rules for Foundational (Training) bots.
426User-agent: GPTBot
427User-agent: CCBot
428User-agent: anthropic-ai
429User-agent: FacebookBot
430User-agent: AmazonBot
431User-agent: Applebot-Extended
432User-agent: Bytespider
433User-agent: Baiduspider
434User-agent: cohere-training-data-crawler
435User-agent: FriendlyCrawler
436User-agent: img2dataset
437User-agent: Meta-ExternalAgent
438User-agent: ClaudeBot
439User-agent: DeepSeekBot
440User-agent: GrokBot
441User-agent: Diffbot
442User-agent: AI2Bot
443User-agent: PanguBot
444User-agent: Timpibot
445Allow: /*&start=0&
446Allow: /*&start=10&
447Allow: /*&start=20&
448Allow: /*&start=30&
449Allow: /*&start=40&
450Allow: /*&start=50&
451Allow: /*&start=60&
452Allow: /*&start=70&
453Allow: /*&start=80&
454Allow: /*&start=90&
455Disallow: /*rt=nc
456Disallow: /*&alid=
457Disallow: /*&calert=
458Disallow: /*&iafilter=
459Disallow: /*&mna=
460Disallow: /*?rss
461Disallow: /*&serpstart=
462Disallow: /*/vaclk
463Disallow: /addlLoc/
464Disallow: /ads/
465Disallow: /advanced_search
466Disallow: /alert
467Disallow: /api/fetch/mc-anon
468Disallow: /api/getrecjobs
469Disallow: /applystart
470Disallow: /community/
471Disallow: /company/*
472Disallow: /conversion/
473Disallow: /cookiemigrator/
474Disallow: /create-resume/lp/
475Disallow: /cdn-cgi/
476Disallow: /%E4%BB%95%E4%BA%8B?
477Disallow: /%E6%B1%82%E4%BA%BA/JP/
478Disallow: /%E6%B1%82%E4%BA%BA/title/
479Disallow: /%E8%81%8C%E4%BD%8D%E6%98%BE%E7%A4%BA?
480Disallow: /%EC%B7%A8%EC%97%85/KR/
481Disallow: /%EC%B7%A8%EC%97%85/title/
482Disallow: /empleo/
483Disallow: /emploi/
484Disallow: /emplois/FR/
485Disallow: /emplois/title/
486Disallow: /emprego/
487Disallow: /forum/profile
488Disallow: /g/
489Disallow: /generate-question
490Disallow: /graphql
491Disallow: /imgping
492Disallow: /ita?
493Disallow: /ita/v1/publisher
494Disallow: /ja/clk?
495Disallow: /ja/imp.gif
496Disallow: /job/
497Disallow: /Job/
498Disallow: /jobroll
499Disallow: /jobtrends/trends/log/*
500Disallow: /jwidget.js*
501Disallow: /me/*/pdf
502Disallow: /m/basecamp/analytics.js
503Disallow: /m/jobalerts?
504Disallow: /m/moreLoc
505Disallow: /m/newjobs
506Disallow: /m/recommended
507Disallow: /m/rpc/
508Disallow: /my/
509Disallow: /*oc=1
510Disallow: /pagead/
511Disallow: /poka%C5%BCprac%C4%99?
512Disallow: /preferences
513Disallow: /*radius=
514Disallow: /rc/
515Disallow: /rdr/
516Disallow: /recommendedjobs
517Disallow: /record-page-tags
518Disallow: /resumes/account
519Disallow: /resumes/advanced?
520Disallow: /resumes/alert
521Disallow: /resumes*radius=
522Disallow: /resumes*rb=
523Disallow: /resumes*res=
524Disallow: /resumes/rpc/
525Disallow: /resumes*sort=
526Disallow: /resumes*start=
527Disallow: /rpc/
528Disallow: /r/*/pdf
529Disallow: /rss
530Disallow: /setprefs
531Disallow: /*sid=
532Disallow: /*sp=0
533Disallow: /*&start=
534Disallow: /stc
535Disallow: /Stellen/CH/
536Disallow: /Stellen/title
537Disallow: /tellafriend
538Disallow: /tmn/ccs/
539Disallow: /tos/banner.js
540Disallow: /uie/clk
541Disallow: /url
542Disallow: /v1/alerts
543Disallow: /v1/impressions
544Disallow: /vacature/
545Disallow: /vacature-bekijken
546Disallow: /ver-empleo
547Disallow: /ver-emprego
548Disallow: /ver-oferta
549Disallow: /visajobb
550Disallow: /voir-emploi
551Disallow: /Zeige-Job
552Disallow: /hire/o/ats-partners
553Disallow: /hire/ats-partners
554Disallow: /offers
555Disallow: /c/info/
556Disallow: /hire/how-to-hire/
557Disallow: /hire/interview-questions/
558Disallow: /recruiting/interviewfragen/
559Disallow: /recruiting/stellenbeschreibung/
560Disallow: /recrutement/description-du-poste/
561Disallow: /recrutement/questions-entretien-dembauche/
562Disallow: /personeel/functiebeschrijving/
563Disallow: /companies/
564Disallow: /cmp/
565Disallow: /career-advice/
566Disallow: /conselho-de-carreira/
567Disallow: /hire/job-description/
568Disallow: /karriere-guide/
569Disallow: /conseils-carrieres/
570Disallow: /conseils-carriere/
571Disallow: /orientacion-profesional/
572Disallow: /porady-zawodowe/
573Disallow: /guida-alla-carriera/
574Disallow: /職涯貼士/
575Disallow: /carrieregids/
576Disallow: /karriarrad/
577Disallow: /orientacion-laboral/
578Disallow: /career/
579Disallow: /*jobs.html
580Disallow: /jobs
581Disallow: /viewjob
582Disallow: /certifications/
583Disallow: /trabajo
584Disallow: /ofertas
585Disallow: /%D0%B2%D0%B0%D0%BA%D0%B0%D0%BD%D1%81%D0%B8%D0%B8
586Disallow: /emplois
587Disallow: /%E6%B1%82%E4%BA%BA
588Disallow: /m/jobs
589Disallow: /m/viewjob
590Disallow: /vacatures
591Disallow: /offerte-lavoro
592Disallow: /Vi%E1%BB%87c-l%C3%A0m
593Disallow: /Jobs
594Disallow: /%D9%88%D8%B8%D8%A7%D8%A6%D9%81
595Disallow: /%EC%B7%A8%EC%97%85
596Disallow: /Jobb
597Disallow: /empregos
598Disallow: /praca
599Disallow: /%E5%B7%A5%E4%BD%9C
600Disallow: /jobb
601Disallow: /%E0%B8%87%E0%B8%B2%E0%B8%99
602Disallow: /lowongan-kerja
603Disallow: /Stellen
604Disallow: /q-
605Disallow: /l-