NeuralCrawl

Indeed / robots.txt snapshot

← back to indeed.com · fetched 2026-09-18T15:27:37Z (20h ago) · HTTP 200 · 13688 bytes · sha256 65d576a8ba993bae · raw

final URL: https://www.indeed.com/robots.txt

1# Default rules for all unspecified bots. Note that these apply to Bingbot too.
2User-agent: *
3Allow: /
4Allow: /*&start=0&
5Allow: /*&start=10&
6Allow: /*&start=20&
7Allow: /*&start=30&
8Allow: /*&start=40&
9Allow: /*&start=50&
10Allow: /*&start=60&
11Allow: /*&start=70&
12Allow: /*&start=80&
13Allow: /*&start=90&
14Disallow: /*rt=nc
15Disallow: /*&alid=
16Disallow: /*&calert=
17Disallow: /*&iafilter=
18Disallow: /*&mna=
19Disallow: /*?rss
20Disallow: /*&serpstart=
21Disallow: /*/vaclk
22Disallow: /addlLoc/
23Disallow: /ads/
24Disallow: /advanced_search
25Disallow: /alert
26Disallow: /api/fetch/mc-anon
27Disallow: /api/getrecjobs
28Disallow: /applystart
29Disallow: /cmp/_/
30Disallow: /cmp/_c/claim/*
31Disallow: /cmp/_rpc/
32Disallow: /cmp/addlink
33Disallow: /cmp/addvideo
34Disallow: /cmp/Login*
35Disallow: /cmp/*/analytics
36Disallow: /cmp/*/company-questions
37Disallow: /cmp/*/people
38Disallow: /cmp/*/write-review
39Disallow: /community/
40Disallow: /company/*
41Disallow: /conversion/
42Disallow: /cookiemigrator/
43Disallow: /create-resume/lp/
44Disallow: /cdn-cgi/
45Disallow: /%E4%BB%95%E4%BA%8B?
46Disallow: /%E5%B7%A5%E4%BD%9C/
47Disallow: /%E6%B1%82%E4%BA%BA/JP/
48Disallow: /%E6%B1%82%E4%BA%BA/title/
49Disallow: /%E8%81%8C%E4%BD%8D%E6%98%BE%E7%A4%BA?
50Disallow: /%EC%B7%A8%EC%97%85/KR/
51Disallow: /%EC%B7%A8%EC%97%85/title/
52Disallow: /empleo/
53Disallow: /emploi/
54Disallow: /emplois/FR/
55Disallow: /emplois/title/
56Disallow: /emprego/
57Disallow: /empregos/BR/
58Disallow: /empregos/title
59Disallow: /forum/profile
60Disallow: /g/
61Disallow: /generate-question
62Disallow: /graphql
63Disallow: /imgping
64Disallow: /ita?
65Disallow: /ita/v1/publisher
66Disallow: /ja/clk?
67Disallow: /ja/imp.gif
68Disallow: /job/
69Disallow: /Job/
70Disallow: /jobb/
71Disallow: /jobroll
72Disallow: /jobs/AE/
73Disallow: /jobs/AQ/
74Disallow: /Jobs/AT/
75Disallow: /jobs/AU/
76Disallow: /jobs/BE/
77Disallow: /jobs/BH/
78Disallow: /jobs/CA/
79Disallow: /jobs/CZ/
80Disallow: /jobs/DE/
81Disallow: /Jobs/DE/
82Disallow: /jobs/DK/
83Disallow: /jobs/FI/
84Disallow: /jobs/GB/
85Disallow: /jobs/GR/
86Disallow: /jobs/HK/
87Disallow: /jobs/HU/
88Disallow: /jobs/ID/
89Disallow: /jobs/IE/
90Disallow: /jobs/IL/
91Disallow: /jobs/IN/
92Disallow: /jobs/KW/
93Disallow: /jobs/LU/
94Disallow: /jobs/MY/
95Disallow: /jobs/NO/
96Disallow: /jobs/NZ/
97Disallow: /jobs/OM/
98Disallow: /jobs/PE/
99Disallow: /jobs/PH/
100Disallow: /jobs/PK/
101Disallow: /jobs/PT/
102Disallow: /jobs/QA/
103Disallow: /jobs/RO/
104Disallow: /jobs/RU/
105Disallow: /jobs/SA/
106Disallow: /jobs/SG/
107Disallow: /jobs/title
108Disallow: /Jobs/title
109Disallow: /jobs/TR/
110Disallow: /jobs/TW/
111Disallow: /jobs/US/
112Disallow: /jobs/VE/
113Disallow: /jobs/ZA/
114Disallow: /jobtrends/trends/log/*
115Disallow: /jwidget.js*
116Disallow: /me/*/pdf
117Disallow: /m/basecamp/analytics.js
118Disallow: /m/jobalerts?
119Disallow: /m/moreLoc
120Disallow: /m/newjobs
121Disallow: /m/recommended
122Disallow: /m/rpc/
123Disallow: /m/viewjob
124Disallow: /my/
125Disallow: /*oc=1
126Disallow: /ofertas/ES/
127Disallow: /ofertas/title/
128Disallow: /offerta-lavoro
129Disallow: /offerte-lavoro/IT/
130Disallow: /offerte-lavoro/title
131Disallow: /pagead/
132Disallow: /poka%C5%BCprac%C4%99?
133Disallow: /praca/
134Disallow: /preferences
135Disallow: /*radius=
136Disallow: /rc/
137Disallow: /rdr/
138Disallow: /recommendedjobs
139Disallow: /record-page-tags
140Disallow: /resumes/account
141Disallow: /resumes/advanced?
142Disallow: /resumes/alert
143Disallow: /resumes*radius=
144Disallow: /resumes*rb=
145Disallow: /resumes*res=
146Disallow: /resumes/rpc/
147Disallow: /resumes*sort=
148Disallow: /resumes*start=
149Disallow: /rpc/
150Disallow: /r/*/pdf
151Disallow: /rss
152Disallow: /setprefs
153Disallow: /*sid=
154Disallow: /*sp=0
155Disallow: /*&start=
156Disallow: /stc
157Disallow: /Stellen/CH/
158Disallow: /Stellen/title
159Disallow: /tellafriend
160Disallow: /tmn/ccs/
161Disallow: /tos/banner.js
162Disallow: /trabajo/
163Disallow: /uie/clk
164Disallow: /url
165Disallow: /v1/alerts
166Disallow: /v1/impressions
167Disallow: /vacature/
168Disallow: /vacature-bekijken
169Disallow: /vacatures/NL/
170Disallow: /vacatures/title/
171Disallow: /ver-empleo
172Disallow: /ver-emprego
173Disallow: /ver-oferta
174Disallow: /viewjob
175Disallow: /visajobb
176Disallow: /voir-emploi
177Disallow: /Zeige-Job
178Disallow: //applystart
179Disallow: /hire/o/ats-partners
180Disallow: /hire/ats-partners
181Disallow: /offers
182
183
184# Rules for traditional search engines and for retrieval/grounding bots.
185User-agent: Googlebot
186User-agent: Google-Extended
187User-agent: ChatGPT-User
188User-agent: Slurp
189User-agent: DuckDuckBot
190User-agent: YahooJP
191User-agent: PetalBot
192User-agent: PerplexityBot
193User-agent: DuckAssistBot
194User-agent: Perplexity-User
195User-agent: YouBot
196User-agent: Claude-User
197User-agent: Gemini-Deep-Research
198User-agent: Claude-SearchBot
199User-agent: OAI-SearchBot
200User-agent: Google-Agent
201User-agent: Google-CloudVertexBot
202User-agent: Meta-WebIndexer
203User-agent: Meta-ExternalFetcher
204User-agent: xAI-Grok
205User-agent: Grok-DeepSearch
206Allow: /
207Allow: /*&start=0&
208Allow: /*&start=10&
209Allow: /*&start=20&
210Allow: /*&start=30&
211Allow: /*&start=40&
212Allow: /*&start=50&
213Allow: /*&start=60&
214Allow: /*&start=70&
215Allow: /*&start=80&
216Allow: /*&start=90&
217Disallow: /*rt=nc
218Disallow: /*&alid=
219Disallow: /*&calert=
220Disallow: /*&iafilter=
221Disallow: /*&mna=
222Disallow: /*?rss
223Disallow: /*&serpstart=
224Disallow: /*/vaclk
225Disallow: /addlLoc/
226Disallow: /ads/
227Disallow: /advanced_search
228Disallow: /alert
229Disallow: /api/fetch/mc-anon
230Disallow: /api/getrecjobs
231Disallow: /applystart
232Disallow: /cmp/_/
233Disallow: /cmp/_c/claim/*
234Disallow: /cmp/_rpc/
235Disallow: /cmp/addlink
236Disallow: /cmp/addvideo
237Disallow: /cmp/Login*
238Disallow: /cmp/*/analytics
239Disallow: /cmp/*/company-questions
240Disallow: /cmp/*/people
241Disallow: /cmp/*/write-review
242Disallow: /community/
243Disallow: /company/*
244Disallow: /conversion/
245Disallow: /cookiemigrator/
246Disallow: /create-resume/lp/
247Disallow: /cdn-cgi/
248Disallow: /%E4%BB%95%E4%BA%8B?
249Disallow: /%E5%B7%A5%E4%BD%9C/
250Disallow: /%E6%B1%82%E4%BA%BA/JP/
251Disallow: /%E6%B1%82%E4%BA%BA/title/
252Disallow: /%E8%81%8C%E4%BD%8D%E6%98%BE%E7%A4%BA?
253Disallow: /%EC%B7%A8%EC%97%85/KR/
254Disallow: /%EC%B7%A8%EC%97%85/title/
255Disallow: /empleo/
256Disallow: /emploi/
257Disallow: /emplois/FR/
258Disallow: /emplois/title/
259Disallow: /emprego/
260Disallow: /empregos/BR/
261Disallow: /empregos/title
262Disallow: /forum/profile
263Disallow: /g/
264Disallow: /generate-question
265Disallow: /graphql
266Disallow: /imgping
267Disallow: /ita?
268Disallow: /ita/v1/publisher
269Disallow: /ja/clk?
270Disallow: /ja/imp.gif
271Disallow: /job/
272Disallow: /Job/
273Disallow: /jobb/
274Disallow: /jobroll
275Disallow: /jobs/AE/
276Disallow: /jobs/AQ/
277Disallow: /Jobs/AT/
278Disallow: /jobs/AU/
279Disallow: /jobs/BE/
280Disallow: /jobs/BH/
281Disallow: /jobs/CA/
282Disallow: /jobs/CZ/
283Disallow: /jobs/DE/
284Disallow: /Jobs/DE/
285Disallow: /jobs/DK/
286Disallow: /jobs/FI/
287Disallow: /jobs/GB/
288Disallow: /jobs/GR/
289Disallow: /jobs/HK/
290Disallow: /jobs/HU/
291Disallow: /jobs/ID/
292Disallow: /jobs/IE/
293Disallow: /jobs/IL/
294Disallow: /jobs/IN/
295Disallow: /jobs/KW/
296Disallow: /jobs/LU/
297Disallow: /jobs/MY/
298Disallow: /jobs/NO/
299Disallow: /jobs/NZ/
300Disallow: /jobs/OM/
301Disallow: /jobs/PE/
302Disallow: /jobs/PH/
303Disallow: /jobs/PK/
304Disallow: /jobs/PT/
305Disallow: /jobs/QA/
306Disallow: /jobs/RO/
307Disallow: /jobs/RU/
308Disallow: /jobs/SA/
309Disallow: /jobs/SG/
310Disallow: /jobs/title
311Disallow: /Jobs/title
312Disallow: /jobs/TR/
313Disallow: /jobs/TW/
314Disallow: /jobs/US/
315Disallow: /jobs/VE/
316Disallow: /jobs/ZA/
317Disallow: /jobtrends/trends/log/*
318Disallow: /jwidget.js*
319Disallow: /me/*/pdf
320Disallow: /m/basecamp/analytics.js
321Disallow: /m/jobalerts?
322Disallow: /m/moreLoc
323Disallow: /m/newjobs
324Disallow: /m/recommended
325Disallow: /m/rpc/
326Disallow: /my/
327Disallow: /*oc=1
328Disallow: /ofertas/ES/
329Disallow: /ofertas/title/
330Disallow: /offerte-lavoro/IT/
331Disallow: /offerte-lavoro/title
332Disallow: /pagead/
333Disallow: /poka%C5%BCprac%C4%99?
334Disallow: /praca/
335Disallow: /preferences
336Disallow: /*radius=
337Disallow: /rc/
338Disallow: /rdr/
339Disallow: /recommendedjobs
340Disallow: /record-page-tags
341Disallow: /resumes/account
342Disallow: /resumes/advanced?
343Disallow: /resumes/alert
344Disallow: /resumes*radius=
345Disallow: /resumes*rb=
346Disallow: /resumes*res=
347Disallow: /resumes/rpc/
348Disallow: /resumes*sort=
349Disallow: /resumes*start=
350Disallow: /rpc/
351Disallow: /r/*/pdf
352Disallow: /rss
353Disallow: /setprefs
354Disallow: /*sid=
355Disallow: /*sp=0
356Disallow: /*&start=
357Disallow: /stc
358Disallow: /Stellen/CH/
359Disallow: /Stellen/title
360Disallow: /tellafriend
361Disallow: /tmn/ccs/
362Disallow: /tos/banner.js
363Disallow: /trabajo/
364Disallow: /uie/clk
365Disallow: /url
366Disallow: /v1/alerts
367Disallow: /v1/impressions
368Disallow: /vacature/
369Disallow: /vacatures/NL/
370Disallow: /vacatures/title/
371Disallow: //applystart
372Disallow: /hire/o/ats-partners
373Disallow: /hire/ats-partners
374Disallow: /offers
375
376
377User-agent: Mediapartners-Google
378Allow: /
379
380
381User-agent: AdsBot-Google
382User-agent: AdsBot-Google-Mobile
383Allow: /create-resume/lp/
384Allow: /*sid=
385Disallow: /rpc/
386
387
388User-agent: adidxbot
389Allow: /*sid=
390
391
392User-agent: Googlebot-Image
393User-agent: msnbot-media
394Disallow: /trendgraph/*
395
396
397# Full disallow for aggressive scrapers, legacy crawlers, and unwanted utility bots.
398User-Agent: OmniExplorer_Bot
399User-agent: ia_archiver
400User-agent: 008
401User-agent: JobdiggerSpider
402User-agent: Cliqzbot
403User-agent: GoogleOther
404User-agent: omgili
405User-agent: omgilibot
406User-agent: Webzio
407User-agent: Webzio-Extended
408User-agent: Scrapy
409User-agent: MyCentralAIScraperBot
410Disallow: /
411
412
413# Rules for Social Media / Link Preview bots.
414User-agent: facebookexternalhit
415User-agent: WhatsApp
416User-agent: snapchat
417User-agent: TelegramBot
418User-agent: Twitterbot
419User-agent: Slackbot-LinkExpanding
420Allow: /
421
422
423# Rules for Foundational (Training) bots.
424User-agent: GPTBot
425User-agent: CCBot
426User-agent: anthropic-ai
427User-agent: FacebookBot
428User-agent: AmazonBot
429User-agent: Applebot-Extended
430User-agent: Bytespider
431User-agent: Baiduspider
432User-agent: cohere-training-data-crawler
433User-agent: FriendlyCrawler
434User-agent: img2dataset
435User-agent: Meta-ExternalAgent
436User-agent: ClaudeBot
437User-agent: DeepSeekBot
438User-agent: GrokBot
439User-agent: Diffbot
440User-agent: AI2Bot
441User-agent: PanguBot
442User-agent: Timpibot
443Allow: /*&start=0&
444Allow: /*&start=10&
445Allow: /*&start=20&
446Allow: /*&start=30&
447Allow: /*&start=40&
448Allow: /*&start=50&
449Allow: /*&start=60&
450Allow: /*&start=70&
451Allow: /*&start=80&
452Allow: /*&start=90&
453Disallow: /*rt=nc
454Disallow: /*&alid=
455Disallow: /*&calert=
456Disallow: /*&iafilter=
457Disallow: /*&mna=
458Disallow: /*?rss
459Disallow: /*&serpstart=
460Disallow: /*/vaclk
461Disallow: /addlLoc/
462Disallow: /ads/
463Disallow: /advanced_search
464Disallow: /alert
465Disallow: /api/fetch/mc-anon
466Disallow: /api/getrecjobs
467Disallow: /applystart
468Disallow: /community/
469Disallow: /company/*
470Disallow: /conversion/
471Disallow: /cookiemigrator/
472Disallow: /create-resume/lp/
473Disallow: /cdn-cgi/
474Disallow: /%E4%BB%95%E4%BA%8B?
475Disallow: /%E6%B1%82%E4%BA%BA/JP/
476Disallow: /%E6%B1%82%E4%BA%BA/title/
477Disallow: /%E8%81%8C%E4%BD%8D%E6%98%BE%E7%A4%BA?
478Disallow: /%EC%B7%A8%EC%97%85/KR/
479Disallow: /%EC%B7%A8%EC%97%85/title/
480Disallow: /empleo/
481Disallow: /emploi/
482Disallow: /emplois/FR/
483Disallow: /emplois/title/
484Disallow: /emprego/
485Disallow: /forum/profile
486Disallow: /g/
487Disallow: /generate-question
488Disallow: /graphql
489Disallow: /imgping
490Disallow: /ita?
491Disallow: /ita/v1/publisher
492Disallow: /ja/clk?
493Disallow: /ja/imp.gif
494Disallow: /job/
495Disallow: /Job/
496Disallow: /jobroll
497Disallow: /jobtrends/trends/log/*
498Disallow: /jwidget.js*
499Disallow: /me/*/pdf
500Disallow: /m/basecamp/analytics.js
501Disallow: /m/jobalerts?
502Disallow: /m/moreLoc
503Disallow: /m/newjobs
504Disallow: /m/recommended
505Disallow: /m/rpc/
506Disallow: /my/
507Disallow: /*oc=1
508Disallow: /pagead/
509Disallow: /poka%C5%BCprac%C4%99?
510Disallow: /preferences
511Disallow: /*radius=
512Disallow: /rc/
513Disallow: /rdr/
514Disallow: /recommendedjobs
515Disallow: /record-page-tags
516Disallow: /resumes/account
517Disallow: /resumes/advanced?
518Disallow: /resumes/alert
519Disallow: /resumes*radius=
520Disallow: /resumes*rb=
521Disallow: /resumes*res=
522Disallow: /resumes/rpc/
523Disallow: /resumes*sort=
524Disallow: /resumes*start=
525Disallow: /rpc/
526Disallow: /r/*/pdf
527Disallow: /rss
528Disallow: /setprefs
529Disallow: /*sid=
530Disallow: /*sp=0
531Disallow: /*&start=
532Disallow: /stc
533Disallow: /Stellen/CH/
534Disallow: /Stellen/title
535Disallow: /tellafriend
536Disallow: /tmn/ccs/
537Disallow: /tos/banner.js
538Disallow: /uie/clk
539Disallow: /url
540Disallow: /v1/alerts
541Disallow: /v1/impressions
542Disallow: /vacature/
543Disallow: /vacature-bekijken
544Disallow: /ver-empleo
545Disallow: /ver-emprego
546Disallow: /ver-oferta
547Disallow: /visajobb
548Disallow: /voir-emploi
549Disallow: /Zeige-Job
550Disallow: /hire/o/ats-partners
551Disallow: /hire/ats-partners
552Disallow: /offers
553Disallow: /c/info/
554Disallow: /hire/how-to-hire/
555Disallow: /hire/interview-questions/
556Disallow: /recruiting/interviewfragen/
557Disallow: /recruiting/stellenbeschreibung/
558Disallow: /recrutement/description-du-poste/
559Disallow: /recrutement/questions-entretien-dembauche/
560Disallow: /personeel/functiebeschrijving/
561Disallow: /companies/
562Disallow: /cmp/
563Disallow: /career-advice/
564Disallow: /conselho-de-carreira/
565Disallow: /hire/job-description/
566Disallow: /karriere-guide/
567Disallow: /conseils-carrieres/
568Disallow: /conseils-carriere/
569Disallow: /orientacion-profesional/
570Disallow: /porady-zawodowe/
571Disallow: /guida-alla-carriera/
572Disallow: /職涯貼士/
573Disallow: /carrieregids/
574Disallow: /karriarrad/
575Disallow: /orientacion-laboral/
576Disallow: /career/
577Disallow: /*jobs.html
578Disallow: /jobs
579Disallow: /viewjob
580Disallow: /certifications/
581Disallow: /trabajo
582Disallow: /ofertas
583Disallow: /%D0%B2%D0%B0%D0%BA%D0%B0%D0%BD%D1%81%D0%B8%D0%B8
584Disallow: /emplois
585Disallow: /%E6%B1%82%E4%BA%BA
586Disallow: /m/jobs
587Disallow: /m/viewjob
588Disallow: /vacatures
589Disallow: /offerte-lavoro
590Disallow: /Vi%E1%BB%87c-l%C3%A0m
591Disallow: /Jobs
592Disallow: /%D9%88%D8%B8%D8%A7%D8%A6%D9%81
593Disallow: /%EC%B7%A8%EC%97%85
594Disallow: /Jobb
595Disallow: /empregos
596Disallow: /praca
597Disallow: /%E5%B7%A5%E4%BD%9C
598Disallow: /jobb
599Disallow: /%E0%B8%87%E0%B8%B2%E0%B8%99
600Disallow: /lowongan-kerja
601Disallow: /Stellen
602Disallow: /q-
603Disallow: /l-