NeuralCrawl

Indeed / robots.txt snapshot

← back to indeed.com · fetched 2026-08-18T05:21:46Z (1mo ago) · HTTP 200 · 13090 bytes · sha256 057faad62f9148a0 · raw

final URL: https://www.indeed.com/robots.txt

1User-agent: *
2Allow: /
3Allow: /*&start=0&
4Allow: /*&start=10&
5Allow: /*&start=20&
6Allow: /*&start=30&
7Allow: /*&start=40&
8Allow: /*&start=50&
9Allow: /*&start=60&
10Allow: /*&start=70&
11Allow: /*&start=80&
12Allow: /*&start=90&
13Disallow: /*rt=nc
14Disallow: /*&alid=
15Disallow: /*&calert=
16Disallow: /*&iafilter=
17Disallow: /*&mna=
18Disallow: /*?rss
19Disallow: /*&serpstart=
20Disallow: /*/vaclk
21Disallow: /addlLoc/
22Disallow: /ads/
23Disallow: /advanced_search
24Disallow: /alert
25Disallow: /api/fetch/mc-anon
26Disallow: /api/getrecjobs
27Disallow: /applystart
28Disallow: /cmp/_/
29Disallow: /cmp/_c/claim/*
30Disallow: /cmp/_rpc/
31Disallow: /cmp/addlink
32Disallow: /cmp/addvideo
33Disallow: /cmp/Login*
34Disallow: /cmp/*/analytics
35Disallow: /cmp/*/company-questions
36Disallow: /cmp/*/people
37Disallow: /cmp/*/write-review
38Disallow: /community/
39Disallow: /company/*
40Disallow: /conversion/
41Disallow: /cookiemigrator/
42Disallow: /create-resume/lp/
43Disallow: /cdn-cgi/
44Disallow: /%E4%BB%95%E4%BA%8B?
45Disallow: /%E5%B7%A5%E4%BD%9C/
46Disallow: /%E6%B1%82%E4%BA%BA/JP/
47Disallow: /%E6%B1%82%E4%BA%BA/title/
48Disallow: /%E8%81%8C%E4%BD%8D%E6%98%BE%E7%A4%BA?
49Disallow: /%EC%B7%A8%EC%97%85/KR/
50Disallow: /%EC%B7%A8%EC%97%85/title/
51Disallow: /empleo/
52Disallow: /emploi/
53Disallow: /emplois/FR/
54Disallow: /emplois/title/
55Disallow: /emprego/
56Disallow: /empregos/BR/
57Disallow: /empregos/title
58Disallow: /forum/profile
59Disallow: /g/
60Disallow: /graphql
61Disallow: /imgping
62Disallow: /ita?
63Disallow: /ita/v1/publisher
64Disallow: /ja/clk?
65Disallow: /ja/imp.gif
66Disallow: /job/
67Disallow: /Job/
68Disallow: /jobb/
69Disallow: /jobroll
70Disallow: /jobs/AE/
71Disallow: /jobs/AQ/
72Disallow: /Jobs/AT/
73Disallow: /jobs/AU/
74Disallow: /jobs/BE/
75Disallow: /jobs/BH/
76Disallow: /jobs/CA/
77Disallow: /jobs/CZ/
78Disallow: /jobs/DE/
79Disallow: /Jobs/DE/
80Disallow: /jobs/DK/
81Disallow: /jobs/FI/
82Disallow: /jobs/GB/
83Disallow: /jobs/GR/
84Disallow: /jobs/HK/
85Disallow: /jobs/HU/
86Disallow: /jobs/ID/
87Disallow: /jobs/IE/
88Disallow: /jobs/IL/
89Disallow: /jobs/IN/
90Disallow: /jobs/KW/
91Disallow: /jobs/LU/
92Disallow: /jobs/MY/
93Disallow: /jobs/NO/
94Disallow: /jobs/NZ/
95Disallow: /jobs/OM/
96Disallow: /jobs/PE/
97Disallow: /jobs/PH/
98Disallow: /jobs/PK/
99Disallow: /jobs/PT/
100Disallow: /jobs/QA/
101Disallow: /jobs/RO/
102Disallow: /jobs/RU/
103Disallow: /jobs/SA/
104Disallow: /jobs/SG/
105Disallow: /jobs/title
106Disallow: /Jobs/title
107Disallow: /jobs/TR/
108Disallow: /jobs/TW/
109Disallow: /jobs/US/
110Disallow: /jobs/VE/
111Disallow: /jobs/ZA/
112Disallow: /jobtrends/trends/log/*
113Disallow: /jwidget.js*
114Disallow: /me/*/pdf
115Disallow: /m/basecamp/analytics.js
116Disallow: /m/jobalerts?
117Disallow: /m/moreLoc
118Disallow: /m/newjobs
119Disallow: /m/recommended
120Disallow: /m/rpc/
121Disallow: /m/viewjob?
122Disallow: /my/
123Disallow: /*oc=1
124Disallow: /ofertas/ES/
125Disallow: /ofertas/title/
126Disallow: /offerta-lavoro
127Disallow: /offerte-lavoro/IT/
128Disallow: /offerte-lavoro/title
129Disallow: /pagead/
130Disallow: /poka%C5%BCprac%C4%99?
131Disallow: /praca/
132Disallow: /preferences
133Disallow: /*radius=
134Disallow: /rc/
135Disallow: /rdr/
136Disallow: /recommendedjobs
137Disallow: /record-page-tags
138Disallow: /resumes/account
139Disallow: /resumes/advanced?
140Disallow: /resumes/alert
141Disallow: /resumes*radius=
142Disallow: /resumes*rb=
143Disallow: /resumes*res=
144Disallow: /resumes/rpc/
145Disallow: /resumes*sort=
146Disallow: /resumes*start=
147Disallow: /rpc/
148Disallow: /r/*/pdf
149Disallow: /rss
150Disallow: /setprefs
151Disallow: /*sid=
152Disallow: /*sp=0
153Disallow: /*&start=
154Disallow: /stc
155Disallow: /Stellen/CH/
156Disallow: /Stellen/title
157Disallow: /tellafriend
158Disallow: /tmn/ccs/
159Disallow: /tos/banner.js
160Disallow: /trabajo/
161Disallow: /uie/clk
162Disallow: /url
163Disallow: /vacature/
164Disallow: /vacature-bekijken?
165Disallow: /vacatures/NL/
166Disallow: /vacatures/title/
167Disallow: /ver-empleo?
168Disallow: /ver-emprego?
169Disallow: /ver-oferta?
170Disallow: /viewjob?
171Disallow: /visajobb?
172Disallow: /voir-emploi?
173Disallow: /Zeige-Job?
174Disallow: //applystart
175
176
177User-agent: Googlebot
178User-agent: Google-Extended
179User-agent: ChatGPT-User
180User-agent: Bingbot
181User-agent: Slurp
182User-agent: DuckDuckBot
183User-agent: YahooJP
184User-agent: PetalBot
185User-agent: PerplexityBot
186User-agent: DuckAssistBot
187User-agent: Perplexity-User
188User-agent: YouBot
189User-agent: Claude-User
190User-agent: Gemini-Deep-Research
191User-agent: Claude-SearchBot
192User-agent: OAI-SearchBot
193User-agent: Google-Agent
194User-agent: Google-CloudVertexBot
195User-agent: Meta-WebIndexer
196User-agent: Meta-ExternalFetcher
197User-agent: xAI-Grok
198User-agent: Grok-DeepSearch
199Allow: /
200Allow: /*&start=0&
201Allow: /*&start=10&
202Allow: /*&start=20&
203Allow: /*&start=30&
204Allow: /*&start=40&
205Allow: /*&start=50&
206Allow: /*&start=60&
207Allow: /*&start=70&
208Allow: /*&start=80&
209Allow: /*&start=90&
210Disallow: /*rt=nc
211Disallow: /*&alid=
212Disallow: /*&calert=
213Disallow: /*&iafilter=
214Disallow: /*&mna=
215Disallow: /*?rss
216Disallow: /*&serpstart=
217Disallow: /*/vaclk
218Disallow: /addlLoc/
219Disallow: /ads/
220Disallow: /advanced_search
221Disallow: /alert
222Disallow: /api/fetch/mc-anon
223Disallow: /api/getrecjobs
224Disallow: /applystart
225Disallow: /cmp/_/
226Disallow: /cmp/_c/claim/*
227Disallow: /cmp/_rpc/
228Disallow: /cmp/addlink
229Disallow: /cmp/addvideo
230Disallow: /cmp/Login*
231Disallow: /cmp/*/analytics
232Disallow: /cmp/*/company-questions
233Disallow: /cmp/*/people
234Disallow: /cmp/*/write-review
235Disallow: /community/
236Disallow: /company/*
237Disallow: /conversion/
238Disallow: /cookiemigrator/
239Disallow: /create-resume/lp/
240Disallow: /cdn-cgi/
241Disallow: /%E4%BB%95%E4%BA%8B?
242Disallow: /%E5%B7%A5%E4%BD%9C/
243Disallow: /%E6%B1%82%E4%BA%BA/JP/
244Disallow: /%E6%B1%82%E4%BA%BA/title/
245Disallow: /%E8%81%8C%E4%BD%8D%E6%98%BE%E7%A4%BA?
246Disallow: /%EC%B7%A8%EC%97%85/KR/
247Disallow: /%EC%B7%A8%EC%97%85/title/
248Disallow: /empleo/
249Disallow: /emploi/
250Disallow: /emplois/FR/
251Disallow: /emplois/title/
252Disallow: /emprego/
253Disallow: /empregos/BR/
254Disallow: /empregos/title
255Disallow: /forum/profile
256Disallow: /g/
257Disallow: /generate-question
258Disallow: /graphql
259Disallow: /imgping
260Disallow: /ita?
261Disallow: /ita/v1/publisher
262Disallow: /ja/clk?
263Disallow: /ja/imp.gif
264Disallow: /job/
265Disallow: /Job/
266Disallow: /jobb/
267Disallow: /jobroll
268Disallow: /jobs/AE/
269Disallow: /jobs/AQ/
270Disallow: /Jobs/AT/
271Disallow: /jobs/AU/
272Disallow: /jobs/BE/
273Disallow: /jobs/BH/
274Disallow: /jobs/CA/
275Disallow: /jobs/CZ/
276Disallow: /jobs/DE/
277Disallow: /Jobs/DE/
278Disallow: /jobs/DK/
279Disallow: /jobs/FI/
280Disallow: /jobs/GB/
281Disallow: /jobs/GR/
282Disallow: /jobs/HK/
283Disallow: /jobs/HU/
284Disallow: /jobs/ID/
285Disallow: /jobs/IE/
286Disallow: /jobs/IL/
287Disallow: /jobs/IN/
288Disallow: /jobs/KW/
289Disallow: /jobs/LU/
290Disallow: /jobs/MY/
291Disallow: /jobs/NO/
292Disallow: /jobs/NZ/
293Disallow: /jobs/OM/
294Disallow: /jobs/PE/
295Disallow: /jobs/PH/
296Disallow: /jobs/PK/
297Disallow: /jobs/PT/
298Disallow: /jobs/QA/
299Disallow: /jobs/RO/
300Disallow: /jobs/RU/
301Disallow: /jobs/SA/
302Disallow: /jobs/SG/
303Disallow: /jobs/title
304Disallow: /Jobs/title
305Disallow: /jobs/TR/
306Disallow: /jobs/TW/
307Disallow: /jobs/US/
308Disallow: /jobs/VE/
309Disallow: /jobs/ZA/
310Disallow: /jobtrends/trends/log/*
311Disallow: /jwidget.js*
312Disallow: /me/*/pdf
313Disallow: /m/basecamp/analytics.js
314Disallow: /m/jobalerts?
315Disallow: /m/moreLoc
316Disallow: /m/newjobs
317Disallow: /m/recommended
318Disallow: /m/rpc/
319Disallow: /my/
320Disallow: /*oc=1
321Disallow: /ofertas/ES/
322Disallow: /ofertas/title/
323Disallow: /offerte-lavoro/IT/
324Disallow: /offerte-lavoro/title
325Disallow: /pagead/
326Disallow: /poka%C5%BCprac%C4%99?
327Disallow: /praca/
328Disallow: /preferences
329Disallow: /*radius=
330Disallow: /rc/
331Disallow: /rdr/
332Disallow: /recommendedjobs
333Disallow: /record-page-tags
334Disallow: /resumes/account
335Disallow: /resumes/advanced?
336Disallow: /resumes/alert
337Disallow: /resumes*radius=
338Disallow: /resumes*rb=
339Disallow: /resumes*res=
340Disallow: /resumes/rpc/
341Disallow: /resumes*sort=
342Disallow: /resumes*start=
343Disallow: /rpc/
344Disallow: /r/*/pdf
345Disallow: /rss
346Disallow: /setprefs
347Disallow: /*sid=
348Disallow: /*sp=0
349Disallow: /*&start=
350Disallow: /stc
351Disallow: /Stellen/CH/
352Disallow: /Stellen/title
353Disallow: /tellafriend
354Disallow: /tmn/ccs/
355Disallow: /tos/banner.js
356Disallow: /trabajo/
357Disallow: /uie/clk
358Disallow: /url
359Disallow: /v1/alerts
360Disallow: /v1/impressions
361Disallow: /vacature/
362Disallow: /vacatures/NL/
363Disallow: /vacatures/title/
364Disallow: //applystart
365Disallow: /hire/o/ats-partners
366Disallow: /hire/ats-partners
367Disallow: /offers
368
369
370User-agent: Mediapartners-Google
371Allow: /
372
373
374User-agent: AdsBot-Google
375User-agent: AdsBot-Google-Mobile
376Allow: /create-resume/lp/
377Allow: /*sid=
378Disallow: /rpc/
379
380
381User-agent: adidxbot
382Allow: /*sid=
383
384
385User-agent: Googlebot-Image
386User-agent: msnbot-media
387Disallow: /trendgraph/*
388
389
390User-Agent: OmniExplorer_Bot
391User-agent: ia_archiver
392User-agent: 008
393User-agent: JobdiggerSpider
394User-agent: Cliqzbot
395User-agent: GoogleOther
396User-agent: omgili
397User-agent: omgilibot
398User-agent: Webzio
399User-agent: Webzio-Extended
400User-agent: Scrapy
401User-agent: MyCentralAIScraperBot
402Disallow: /
403
404
405User-agent: facebookexternalhit
406User-agent: WhatsApp
407User-agent: snapchat
408User-agent: TelegramBot
409User-agent: Twitterbot
410User-agent: Slackbot-LinkExpanding
411Allow: /
412
413
414User-agent: GPTBot
415User-agent: CCBot
416User-agent: anthropic-ai
417User-agent: FacebookBot
418User-agent: AmazonBot
419User-agent: Applebot-Extended
420User-agent: Bytespider
421User-agent: Baiduspider
422User-agent: cohere-training-data-crawler
423User-agent: FriendlyCrawler
424User-agent: img2dataset
425User-agent: Meta-ExternalAgent
426User-agent: ClaudeBot
427User-agent: DeepSeekBot
428User-agent: GrokBot
429User-agent: Diffbot
430User-agent: AI2Bot
431User-agent: PanguBot
432User-agent: Timpibot
433Allow: /*&start=0&
434Allow: /*&start=10&
435Allow: /*&start=20&
436Allow: /*&start=30&
437Allow: /*&start=40&
438Allow: /*&start=50&
439Allow: /*&start=60&
440Allow: /*&start=70&
441Allow: /*&start=80&
442Allow: /*&start=90&
443Disallow: /*rt=nc
444Disallow: /*&alid=
445Disallow: /*&calert=
446Disallow: /*&iafilter=
447Disallow: /*&mna=
448Disallow: /*?rss
449Disallow: /*&serpstart=
450Disallow: /*/vaclk
451Disallow: /addlLoc/
452Disallow: /ads/
453Disallow: /advanced_search
454Disallow: /alert
455Disallow: /api/fetch/mc-anon
456Disallow: /api/getrecjobs
457Disallow: /applystart
458Disallow: /community/
459Disallow: /company/*
460Disallow: /conversion/
461Disallow: /cookiemigrator/
462Disallow: /create-resume/lp/
463Disallow: /cdn-cgi/
464Disallow: /%E4%BB%95%E4%BA%8B?
465Disallow: /%E6%B1%82%E4%BA%BA/JP/
466Disallow: /%E6%B1%82%E4%BA%BA/title/
467Disallow: /%E8%81%8C%E4%BD%8D%E6%98%BE%E7%A4%BA?
468Disallow: /%EC%B7%A8%EC%97%85/KR/
469Disallow: /%EC%B7%A8%EC%97%85/title/
470Disallow: /empleo/
471Disallow: /emploi/
472Disallow: /emplois/FR/
473Disallow: /emplois/title/
474Disallow: /emprego/
475Disallow: /forum/profile
476Disallow: /g/
477Disallow: /graphql
478Disallow: /imgping
479Disallow: /ita?
480Disallow: /ita/v1/publisher
481Disallow: /ja/clk?
482Disallow: /ja/imp.gif
483Disallow: /job/
484Disallow: /Job/
485Disallow: /jobroll
486Disallow: /jobtrends/trends/log/*
487Disallow: /jwidget.js*
488Disallow: /me/*/pdf
489Disallow: /m/basecamp/analytics.js
490Disallow: /m/jobalerts?
491Disallow: /m/moreLoc
492Disallow: /m/newjobs
493Disallow: /m/recommended
494Disallow: /m/rpc/
495Disallow: /my/
496Disallow: /*oc=1
497Disallow: /pagead/
498Disallow: /poka%C5%BCprac%C4%99?
499Disallow: /preferences
500Disallow: /*radius=
501Disallow: /rc/
502Disallow: /rdr/
503Disallow: /recommendedjobs
504Disallow: /record-page-tags
505Disallow: /resumes/account
506Disallow: /resumes/advanced?
507Disallow: /resumes/alert
508Disallow: /resumes*radius=
509Disallow: /resumes*rb=
510Disallow: /resumes*res=
511Disallow: /resumes/rpc/
512Disallow: /resumes*sort=
513Disallow: /resumes*start=
514Disallow: /rpc/
515Disallow: /r/*/pdf
516Disallow: /rss
517Disallow: /setprefs
518Disallow: /*sid=
519Disallow: /*sp=0
520Disallow: /*&start=
521Disallow: /stc
522Disallow: /Stellen/CH/
523Disallow: /Stellen/title
524Disallow: /tellafriend
525Disallow: /tmn/ccs/
526Disallow: /tos/banner.js
527Disallow: /uie/clk
528Disallow: /url
529Disallow: /vacature/
530Disallow: /vacature-bekijken?
531Disallow: /ver-empleo?
532Disallow: /ver-emprego?
533Disallow: /ver-oferta?
534Disallow: /visajobb?
535Disallow: /voir-emploi?
536Disallow: /Zeige-Job?
537Disallow: /c/info/
538Disallow: /hire/how-to-hire/
539Disallow: /hire/interview-questions/
540Disallow: /recruiting/interviewfragen/
541Disallow: /recruiting/stellenbeschreibung/
542Disallow: /recrutement/description-du-poste/
543Disallow: /recrutement/questions-entretien-dembauche/
544Disallow: /personeel/functiebeschrijving/
545Disallow: /companies/
546Disallow: /cmp/
547Disallow: /career-advice/
548Disallow: /conselho-de-carreira/
549Disallow: /hire/job-description/
550Disallow: /karriere-guide/
551Disallow: /conseils-carrieres/
552Disallow: /conseils-carriere/
553Disallow: /orientacion-profesional/
554Disallow: /porady-zawodowe/
555Disallow: /guida-alla-carriera/
556Disallow: /職涯貼士/
557Disallow: /carrieregids/
558Disallow: /karriarrad/
559Disallow: /orientacion-laboral/
560Disallow: /career/
561Disallow: /*jobs.html
562Disallow: /jobs
563Disallow: /viewjob
564Disallow: /certifications/
565Disallow: /trabajo
566Disallow: /ofertas
567Disallow: /%D0%B2%D0%B0%D0%BA%D0%B0%D0%BD%D1%81%D0%B8%D0%B8
568Disallow: /emplois
569Disallow: /%E6%B1%82%E4%BA%BA
570Disallow: /m/jobs
571Disallow: /m/viewjob
572Disallow: /vacatures
573Disallow: /offerte-lavoro
574Disallow: /Vi%E1%BB%87c-l%C3%A0m
575Disallow: /Jobs
576Disallow: /%D9%88%D8%B8%D8%A7%D8%A6%D9%81
577Disallow: /%EC%B7%A8%EC%97%85
578Disallow: /Jobb
579Disallow: /empregos
580Disallow: /praca
581Disallow: /%E5%B7%A5%E4%BD%9C
582Disallow: /jobb
583Disallow: /%E0%B8%87%E0%B8%B2%E0%B8%99
584Disallow: /lowongan-kerja
585Disallow: /Stellen
586Disallow: /q-
587Disallow: /l-