NeuralCrawl

GitHub / robots.txt snapshot

← back to github.com · fetched 2026-11-07T08:19:09Z (3h ago) · HTTP 200 · 15093 bytes · sha256 e677c4f6eb05d7b9 · raw

final URL: https://github.com/robots.txt

1# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots
2# We also provide an extensive API: https://docs.github.com
3Sitemap: https://github.com/sitemap_marketing.xml
4
5User-agent: GPTBot
6User-agent: OAI-SearchBot
7User-agent: ChatGPT-User
8User-agent: ClaudeBot
9User-agent: Claude-SearchBot
10User-agent: Claude-User
11User-agent: PerplexityBot
12User-agent: Perplexity-User
13Crawl-delay: 1
14Allow: /*?tab=achievements&achievement=*
15Allow: /marketplace/*?tab=readme$
16Allow: /marketplace/*?tab=transparency$
17Allow: /enterprise/contact?*
18Allow: /enterprise/contact/data-residency?*
19Allow: /features/codespaces?*
20Allow: /features/copilot/plans?*
21Allow: /security/advanced-security/demo?*
22Allow: /security/contact-sales?*
23Allow: /security/plans?*
24Disallow: /account-login
25Disallow: /copilot/
26Disallow: /copilot/c/
27Disallow: /ekansa/Open-Context-Data
28Disallow: /ekansa/opencontext-*
29Disallow: /Explodingstuff/
30Disallow: /gist/
31Disallow: /search/advanced
32Disallow: /search$
33Disallow: /*/*/pulse_committer_data.json$
34Disallow: /*/*/pulse_committer_data.json?*
35Disallow: /*/archive/
36Disallow: /*/blame/
37Disallow: /*/cache/
38Disallow: /*/raw/
39Disallow: /*/tree/
40Disallow: */tarball/
41Disallow: */zipball/
42Disallow: /.git/
43Disallow: */.git/
44Disallow: /*.git$
45Disallow: /*.atom$
46Disallow: /*report-abuse?report=*
47Disallow: /*/*/commits/
48Disallow: /marketplace/*/*?tab=readme$
49Disallow: /marketplace/*/*?tab=transparency$
50Disallow: /marketplace/*?*?tab=readme$
51Disallow: /marketplace/*?*?tab=transparency$
52Disallow: /marketplace/actions?tab=readme$
53Disallow: /marketplace/actions?tab=transparency$
54Disallow: /marketplace/free-trials?tab=readme$
55Disallow: /marketplace/free-trials?tab=transparency$
56Disallow: /marketplace/manage?tab=readme$
57Disallow: /marketplace/manage?tab=transparency$
58Disallow: /marketplace/models?tab=readme$
59Disallow: /marketplace/models?tab=transparency$
60Disallow: /marketplace/new?tab=readme$
61Disallow: /marketplace/new?tab=transparency$
62Disallow: /marketplace/publishers?tab=readme$
63Disallow: /marketplace/publishers?tab=transparency$
64Disallow: /marketplace/recently-added?tab=readme$
65Disallow: /marketplace/recently-added?tab=transparency$
66Disallow: /biztools$
67Disallow: /biztools?*
68Disallow: /biztools/*
69Disallow: /businesses$
70Disallow: /businesses?*
71Disallow: /businesses/*
72Disallow: /enterprises$
73Disallow: /enterprises?*
74Disallow: /enterprises/*
75Disallow: /organizations$
76Disallow: /organizations?*
77Disallow: /organizations/*
78Disallow: /settings$
79Disallow: /settings?*
80Disallow: /settings/*
81Disallow: /stafftools$
82Disallow: /stafftools?*
83Disallow: /stafftools/*
84Disallow: /users/*/billing$
85Disallow: /users/*/billing?*
86Disallow: /users/*/billing/*
87Disallow: /writing$
88Disallow: /writing?*
89Disallow: /writing/*
90Disallow: /*/*/branches$
91Disallow: /*/*/branches?*
92Disallow: /*/*/branches/*
93Disallow: /*/*/comments$
94Disallow: /*/*/comments?*
95Disallow: /*/*/comments/*
96Disallow: /*/*/compare$
97Disallow: /*/*/compare?*
98Disallow: /*/*/compare/*
99Disallow: /*/*/contributors$
100Disallow: /*/*/contributors?*
101Disallow: /*/*/contributors/*
102Disallow: /*/*/contributors_list$
103Disallow: /*/*/contributors_list?*
104Disallow: /*/*/contributors_list/*
105Disallow: /*/*/contributors-list$
106Disallow: /*/*/contributors-list?*
107Disallow: /*/*/contributors-list/*
108Disallow: /*/*/download$
109Disallow: /*/*/download?*
110Disallow: /*/*/download/*
111Disallow: /*/*/forks$
112Disallow: /*/*/forks?*
113Disallow: /*/*/forks/*
114Disallow: /*/*/graphs$
115Disallow: /*/*/graphs?*
116Disallow: /*/*/graphs/*
117Disallow: /*/*/issues/new$
118Disallow: /*/*/issues/new?*
119Disallow: /*/*/issues/new/*
120Disallow: /*/*/issues/search$
121Disallow: /*/*/issues/search?*
122Disallow: /*/*/issues/search/*
123Disallow: /*/*/milestones/new$
124Disallow: /*/*/milestones/new?*
125Disallow: /*/*/milestones/new/*
126Disallow: /*/*/network$
127Disallow: /*/*/network?*
128Disallow: /*/*/network/*
129Disallow: /*/*/projects$
130Disallow: /*/*/projects?*
131Disallow: /*/*/projects/*
132Disallow: /*/*/pulse$
133Disallow: /*/*/pulse?*
134Disallow: /*/*/pulse/*
135Disallow: /*/*/pulse-new$
136Disallow: /*/*/pulse-new?*
137Disallow: /*/*/pulse-new/*
138Disallow: /*/*/pulse_committer_data$
139Disallow: /*/*/pulse_committer_data?*
140Disallow: /*/*/pulse_committer_data/*
141Disallow: /*/*/pulse_diffstat_summary$
142Disallow: /*/*/pulse_diffstat_summary?*
143Disallow: /*/*/pulse_diffstat_summary/*
144Disallow: /*/*/revisions$
145Disallow: /*/*/revisions?*
146Disallow: /*/*/revisions/*
147Disallow: /*/*/stargazers$
148Disallow: /*/*/stargazers?*
149Disallow: /*/*/stargazers/*
150Disallow: /*/*/tags$
151Disallow: /*/*/tags?*
152Disallow: /*/*/tags/*
153Disallow: /*/*/watchers$
154Disallow: /*/*/watchers?*
155Disallow: /*/*/watchers/*
156Disallow: /*?source=*
157Disallow: /*&source=*
158Disallow: /*?ref_cta=*
159Disallow: /*&ref_cta=*
160Disallow: /*?plan=*
161Disallow: /*&plan=*
162Disallow: /*?return_to=*
163Disallow: /*&return_to=*
164Disallow: /*?ref_loc=*
165Disallow: /*&ref_loc=*
166Disallow: /*?setup_organization=*
167Disallow: /*&setup_organization=*
168Disallow: /*?source_repo=*
169Disallow: /*&source_repo=*
170Disallow: /*?ref_page=*
171Disallow: /*&ref_page=*
172Disallow: /*?referrer=*
173Disallow: /*&referrer=*
174Disallow: /*?report=*
175Disallow: /*&report=*
176Disallow: /*?author=*
177Disallow: /*&author=*
178Disallow: /*?since=*
179Disallow: /*&since=*
180Disallow: /*?until=*
181Disallow: /*&until=*
182Disallow: /*?tab=*
183Disallow: /*&tab=*
184Disallow: /*?q=*
185Disallow: /*&q=*
186
187User-agent: Bytespider
188Disallow: /
189
190User-agent: AdsBot-Google
191User-agent: AdsBot-Google-Mobile
192Disallow: /account-login
193Disallow: /copilot/
194Disallow: /copilot/c/
195Disallow: /ekansa/Open-Context-Data
196Disallow: /ekansa/opencontext-*
197Disallow: /Explodingstuff/
198Disallow: /gist/
199Disallow: /search/advanced
200Disallow: /search$
201Disallow: /*/*/pulse_committer_data.json$
202Disallow: /*/*/pulse_committer_data.json?*
203Disallow: /*/archive/
204Disallow: /*/blame/
205Disallow: /*/cache/
206Disallow: /*/raw/
207Disallow: /*/tree/
208Disallow: */tarball/
209Disallow: */zipball/
210Disallow: /.git/
211Disallow: */.git/
212Disallow: /*.git$
213Disallow: /*.atom$
214Disallow: /*report-abuse?report=*
215Disallow: /*/*/commits/
216Disallow: /marketplace/*/*?tab=readme$
217Disallow: /marketplace/*/*?tab=transparency$
218Disallow: /marketplace/*?*?tab=readme$
219Disallow: /marketplace/*?*?tab=transparency$
220Disallow: /marketplace/actions?tab=readme$
221Disallow: /marketplace/actions?tab=transparency$
222Disallow: /marketplace/free-trials?tab=readme$
223Disallow: /marketplace/free-trials?tab=transparency$
224Disallow: /marketplace/manage?tab=readme$
225Disallow: /marketplace/manage?tab=transparency$
226Disallow: /marketplace/models?tab=readme$
227Disallow: /marketplace/models?tab=transparency$
228Disallow: /marketplace/new?tab=readme$
229Disallow: /marketplace/new?tab=transparency$
230Disallow: /marketplace/publishers?tab=readme$
231Disallow: /marketplace/publishers?tab=transparency$
232Disallow: /marketplace/recently-added?tab=readme$
233Disallow: /marketplace/recently-added?tab=transparency$
234Disallow: /biztools$
235Disallow: /biztools?*
236Disallow: /biztools/*
237Disallow: /businesses$
238Disallow: /businesses?*
239Disallow: /businesses/*
240Disallow: /enterprises$
241Disallow: /enterprises?*
242Disallow: /enterprises/*
243Disallow: /organizations$
244Disallow: /organizations?*
245Disallow: /organizations/*
246Disallow: /settings$
247Disallow: /settings?*
248Disallow: /settings/*
249Disallow: /stafftools$
250Disallow: /stafftools?*
251Disallow: /stafftools/*
252Disallow: /users/*/billing$
253Disallow: /users/*/billing?*
254Disallow: /users/*/billing/*
255Disallow: /writing$
256Disallow: /writing?*
257Disallow: /writing/*
258Disallow: /*/*/branches$
259Disallow: /*/*/branches?*
260Disallow: /*/*/branches/*
261Disallow: /*/*/comments$
262Disallow: /*/*/comments?*
263Disallow: /*/*/comments/*
264Disallow: /*/*/compare$
265Disallow: /*/*/compare?*
266Disallow: /*/*/compare/*
267Disallow: /*/*/contributors$
268Disallow: /*/*/contributors?*
269Disallow: /*/*/contributors/*
270Disallow: /*/*/contributors_list$
271Disallow: /*/*/contributors_list?*
272Disallow: /*/*/contributors_list/*
273Disallow: /*/*/contributors-list$
274Disallow: /*/*/contributors-list?*
275Disallow: /*/*/contributors-list/*
276Disallow: /*/*/download$
277Disallow: /*/*/download?*
278Disallow: /*/*/download/*
279Disallow: /*/*/forks$
280Disallow: /*/*/forks?*
281Disallow: /*/*/forks/*
282Disallow: /*/*/graphs$
283Disallow: /*/*/graphs?*
284Disallow: /*/*/graphs/*
285Disallow: /*/*/issues/new$
286Disallow: /*/*/issues/new?*
287Disallow: /*/*/issues/new/*
288Disallow: /*/*/issues/search$
289Disallow: /*/*/issues/search?*
290Disallow: /*/*/issues/search/*
291Disallow: /*/*/milestones/new$
292Disallow: /*/*/milestones/new?*
293Disallow: /*/*/milestones/new/*
294Disallow: /*/*/network$
295Disallow: /*/*/network?*
296Disallow: /*/*/network/*
297Disallow: /*/*/projects$
298Disallow: /*/*/projects?*
299Disallow: /*/*/projects/*
300Disallow: /*/*/pulse$
301Disallow: /*/*/pulse?*
302Disallow: /*/*/pulse/*
303Disallow: /*/*/pulse-new$
304Disallow: /*/*/pulse-new?*
305Disallow: /*/*/pulse-new/*
306Disallow: /*/*/pulse_committer_data$
307Disallow: /*/*/pulse_committer_data?*
308Disallow: /*/*/pulse_committer_data/*
309Disallow: /*/*/pulse_diffstat_summary$
310Disallow: /*/*/pulse_diffstat_summary?*
311Disallow: /*/*/pulse_diffstat_summary/*
312Disallow: /*/*/revisions$
313Disallow: /*/*/revisions?*
314Disallow: /*/*/revisions/*
315Disallow: /*/*/stargazers$
316Disallow: /*/*/stargazers?*
317Disallow: /*/*/stargazers/*
318Disallow: /*/*/tags$
319Disallow: /*/*/tags?*
320Disallow: /*/*/tags/*
321Disallow: /*/*/watchers$
322Disallow: /*/*/watchers?*
323Disallow: /*/*/watchers/*
324Disallow: /*?source=*
325Disallow: /*&source=*
326Disallow: /*?ref_cta=*
327Disallow: /*&ref_cta=*
328Disallow: /*?plan=*
329Disallow: /*&plan=*
330Disallow: /*?return_to=*
331Disallow: /*&return_to=*
332Disallow: /*?ref_loc=*
333Disallow: /*&ref_loc=*
334Disallow: /*?setup_organization=*
335Disallow: /*&setup_organization=*
336Disallow: /*?source_repo=*
337Disallow: /*&source_repo=*
338Disallow: /*?ref_page=*
339Disallow: /*&ref_page=*
340Disallow: /*?referrer=*
341Disallow: /*&referrer=*
342Disallow: /*?report=*
343Disallow: /*&report=*
344Disallow: /*?author=*
345Disallow: /*&author=*
346Disallow: /*?since=*
347Disallow: /*&since=*
348Disallow: /*?until=*
349Disallow: /*&until=*
350Disallow: /*?tab=*
351Disallow: /*&tab=*
352Disallow: /*?q=*
353Disallow: /*&q=*
354
355User-agent: bingbot
356User-agent: adidxbot
357User-agent: BingPreview
358Disallow: /ekansa/Open-Context-Data
359Disallow: /ekansa/opencontext-*
360Disallow: /account-login
361Disallow: */tarball/
362Disallow: */zipball/
363Disallow: /Explodingstuff/
364Disallow: /copilot/
365Disallow: /copilot/c/
366
367User-agent: baidu
368crawl-delay: 1
369
370User-agent: *
371Allow: /*?tab=achievements&achievement=*
372Allow: /marketplace/*?tab=readme$
373Allow: /marketplace/*?tab=transparency$
374Allow: /enterprise/contact?*
375Allow: /enterprise/contact/data-residency?*
376Allow: /features/codespaces?*
377Allow: /features/copilot/plans?*
378Allow: /security/advanced-security/demo?*
379Allow: /security/contact-sales?*
380Allow: /security/plans?*
381Disallow: /account-login
382Disallow: /copilot/
383Disallow: /copilot/c/
384Disallow: /ekansa/Open-Context-Data
385Disallow: /ekansa/opencontext-*
386Disallow: /Explodingstuff/
387Disallow: /gist/
388Disallow: /search/advanced
389Disallow: /search$
390Disallow: /*/*/pulse_committer_data.json$
391Disallow: /*/*/pulse_committer_data.json?*
392Disallow: /*/archive/
393Disallow: /*/blame/
394Disallow: /*/cache/
395Disallow: /*/raw/
396Disallow: /*/tree/
397Disallow: */tarball/
398Disallow: */zipball/
399Disallow: /.git/
400Disallow: */.git/
401Disallow: /*.git$
402Disallow: /*.atom$
403Disallow: /*report-abuse?report=*
404Disallow: /*/*/commits/
405Disallow: /marketplace/*/*?tab=readme$
406Disallow: /marketplace/*/*?tab=transparency$
407Disallow: /marketplace/*?*?tab=readme$
408Disallow: /marketplace/*?*?tab=transparency$
409Disallow: /marketplace/actions?tab=readme$
410Disallow: /marketplace/actions?tab=transparency$
411Disallow: /marketplace/free-trials?tab=readme$
412Disallow: /marketplace/free-trials?tab=transparency$
413Disallow: /marketplace/manage?tab=readme$
414Disallow: /marketplace/manage?tab=transparency$
415Disallow: /marketplace/models?tab=readme$
416Disallow: /marketplace/models?tab=transparency$
417Disallow: /marketplace/new?tab=readme$
418Disallow: /marketplace/new?tab=transparency$
419Disallow: /marketplace/publishers?tab=readme$
420Disallow: /marketplace/publishers?tab=transparency$
421Disallow: /marketplace/recently-added?tab=readme$
422Disallow: /marketplace/recently-added?tab=transparency$
423Disallow: /biztools$
424Disallow: /biztools?*
425Disallow: /biztools/*
426Disallow: /businesses$
427Disallow: /businesses?*
428Disallow: /businesses/*
429Disallow: /enterprises$
430Disallow: /enterprises?*
431Disallow: /enterprises/*
432Disallow: /organizations$
433Disallow: /organizations?*
434Disallow: /organizations/*
435Disallow: /settings$
436Disallow: /settings?*
437Disallow: /settings/*
438Disallow: /stafftools$
439Disallow: /stafftools?*
440Disallow: /stafftools/*
441Disallow: /users/*/billing$
442Disallow: /users/*/billing?*
443Disallow: /users/*/billing/*
444Disallow: /writing$
445Disallow: /writing?*
446Disallow: /writing/*
447Disallow: /*/*/branches$
448Disallow: /*/*/branches?*
449Disallow: /*/*/branches/*
450Disallow: /*/*/comments$
451Disallow: /*/*/comments?*
452Disallow: /*/*/comments/*
453Disallow: /*/*/compare$
454Disallow: /*/*/compare?*
455Disallow: /*/*/compare/*
456Disallow: /*/*/contributors$
457Disallow: /*/*/contributors?*
458Disallow: /*/*/contributors/*
459Disallow: /*/*/contributors_list$
460Disallow: /*/*/contributors_list?*
461Disallow: /*/*/contributors_list/*
462Disallow: /*/*/contributors-list$
463Disallow: /*/*/contributors-list?*
464Disallow: /*/*/contributors-list/*
465Disallow: /*/*/download$
466Disallow: /*/*/download?*
467Disallow: /*/*/download/*
468Disallow: /*/*/forks$
469Disallow: /*/*/forks?*
470Disallow: /*/*/forks/*
471Disallow: /*/*/graphs$
472Disallow: /*/*/graphs?*
473Disallow: /*/*/graphs/*
474Disallow: /*/*/issues/new$
475Disallow: /*/*/issues/new?*
476Disallow: /*/*/issues/new/*
477Disallow: /*/*/issues/search$
478Disallow: /*/*/issues/search?*
479Disallow: /*/*/issues/search/*
480Disallow: /*/*/milestones/new$
481Disallow: /*/*/milestones/new?*
482Disallow: /*/*/milestones/new/*
483Disallow: /*/*/network$
484Disallow: /*/*/network?*
485Disallow: /*/*/network/*
486Disallow: /*/*/projects$
487Disallow: /*/*/projects?*
488Disallow: /*/*/projects/*
489Disallow: /*/*/pulse$
490Disallow: /*/*/pulse?*
491Disallow: /*/*/pulse/*
492Disallow: /*/*/pulse-new$
493Disallow: /*/*/pulse-new?*
494Disallow: /*/*/pulse-new/*
495Disallow: /*/*/pulse_committer_data$
496Disallow: /*/*/pulse_committer_data?*
497Disallow: /*/*/pulse_committer_data/*
498Disallow: /*/*/pulse_diffstat_summary$
499Disallow: /*/*/pulse_diffstat_summary?*
500Disallow: /*/*/pulse_diffstat_summary/*
501Disallow: /*/*/revisions$
502Disallow: /*/*/revisions?*
503Disallow: /*/*/revisions/*
504Disallow: /*/*/stargazers$
505Disallow: /*/*/stargazers?*
506Disallow: /*/*/stargazers/*
507Disallow: /*/*/tags$
508Disallow: /*/*/tags?*
509Disallow: /*/*/tags/*
510Disallow: /*/*/watchers$
511Disallow: /*/*/watchers?*
512Disallow: /*/*/watchers/*
513Disallow: /*?source=*
514Disallow: /*&source=*
515Disallow: /*?ref_cta=*
516Disallow: /*&ref_cta=*
517Disallow: /*?plan=*
518Disallow: /*&plan=*
519Disallow: /*?return_to=*
520Disallow: /*&return_to=*
521Disallow: /*?ref_loc=*
522Disallow: /*&ref_loc=*
523Disallow: /*?setup_organization=*
524Disallow: /*&setup_organization=*
525Disallow: /*?source_repo=*
526Disallow: /*&source_repo=*
527Disallow: /*?ref_page=*
528Disallow: /*&ref_page=*
529Disallow: /*?referrer=*
530Disallow: /*&referrer=*
531Disallow: /*?report=*
532Disallow: /*&report=*
533Disallow: /*?author=*
534Disallow: /*&author=*
535Disallow: /*?since=*
536Disallow: /*&since=*
537Disallow: /*?until=*
538Disallow: /*&until=*
539Disallow: /*?tab=*
540Disallow: /*&tab=*
541Disallow: /*?q=*
542Disallow: /*&q=*