NeuralCrawl

GitHub / robots.txt snapshot

← back to github.com · fetched 2026-10-17T01:09:51Z (21d ago) · HTTP 200 · 14489 bytes · sha256 d2f2d319b7631f56 · raw

final URL: https://github.com/robots.txt

1# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots
2# We also provide an extensive API: https://docs.github.com
3User-agent: GPTBot
4User-agent: OAI-SearchBot
5User-agent: ClaudeBot
6User-agent: anthropic-ai
7User-agent: PerplexityBot
8Crawl-delay: 1
9Allow: /*?tab=achievements&achievement=*
10Allow: /marketplace/*?tab=readme$
11Allow: /marketplace/*?tab=transparency$
12Disallow: /account-login
13Disallow: /copilot/
14Disallow: /copilot/c/
15Disallow: /ekansa/Open-Context-Data
16Disallow: /ekansa/opencontext-*
17Disallow: /Explodingstuff/
18Disallow: /gist/
19Disallow: /search/advanced
20Disallow: /search$
21Disallow: /*/*/pulse_committer_data.json$
22Disallow: /*/*/pulse_committer_data.json?*
23Disallow: /*/archive/
24Disallow: /*/blame/
25Disallow: /*/cache/
26Disallow: /*/raw/
27Disallow: /*/tree/
28Disallow: */tarball/
29Disallow: */zipball/
30Disallow: /.git/
31Disallow: */.git/
32Disallow: /*.git$
33Disallow: /*.atom$
34Disallow: /*report-abuse?report=*
35Disallow: /*/*/commits/
36Disallow: /marketplace/*/*?tab=readme$
37Disallow: /marketplace/*/*?tab=transparency$
38Disallow: /marketplace/*?*?tab=readme$
39Disallow: /marketplace/*?*?tab=transparency$
40Disallow: /marketplace/actions?tab=readme$
41Disallow: /marketplace/actions?tab=transparency$
42Disallow: /marketplace/free-trials?tab=readme$
43Disallow: /marketplace/free-trials?tab=transparency$
44Disallow: /marketplace/manage?tab=readme$
45Disallow: /marketplace/manage?tab=transparency$
46Disallow: /marketplace/models?tab=readme$
47Disallow: /marketplace/models?tab=transparency$
48Disallow: /marketplace/new?tab=readme$
49Disallow: /marketplace/new?tab=transparency$
50Disallow: /marketplace/publishers?tab=readme$
51Disallow: /marketplace/publishers?tab=transparency$
52Disallow: /marketplace/recently-added?tab=readme$
53Disallow: /marketplace/recently-added?tab=transparency$
54Disallow: /biztools$
55Disallow: /biztools?*
56Disallow: /biztools/*
57Disallow: /businesses$
58Disallow: /businesses?*
59Disallow: /businesses/*
60Disallow: /enterprises$
61Disallow: /enterprises?*
62Disallow: /enterprises/*
63Disallow: /organizations$
64Disallow: /organizations?*
65Disallow: /organizations/*
66Disallow: /settings$
67Disallow: /settings?*
68Disallow: /settings/*
69Disallow: /stafftools$
70Disallow: /stafftools?*
71Disallow: /stafftools/*
72Disallow: /users/*/billing$
73Disallow: /users/*/billing?*
74Disallow: /users/*/billing/*
75Disallow: /writing$
76Disallow: /writing?*
77Disallow: /writing/*
78Disallow: /*/*/branches$
79Disallow: /*/*/branches?*
80Disallow: /*/*/branches/*
81Disallow: /*/*/comments$
82Disallow: /*/*/comments?*
83Disallow: /*/*/comments/*
84Disallow: /*/*/compare$
85Disallow: /*/*/compare?*
86Disallow: /*/*/compare/*
87Disallow: /*/*/contributors$
88Disallow: /*/*/contributors?*
89Disallow: /*/*/contributors/*
90Disallow: /*/*/contributors_list$
91Disallow: /*/*/contributors_list?*
92Disallow: /*/*/contributors_list/*
93Disallow: /*/*/contributors-list$
94Disallow: /*/*/contributors-list?*
95Disallow: /*/*/contributors-list/*
96Disallow: /*/*/download$
97Disallow: /*/*/download?*
98Disallow: /*/*/download/*
99Disallow: /*/*/forks$
100Disallow: /*/*/forks?*
101Disallow: /*/*/forks/*
102Disallow: /*/*/graphs$
103Disallow: /*/*/graphs?*
104Disallow: /*/*/graphs/*
105Disallow: /*/*/issues/new$
106Disallow: /*/*/issues/new?*
107Disallow: /*/*/issues/new/*
108Disallow: /*/*/issues/search$
109Disallow: /*/*/issues/search?*
110Disallow: /*/*/issues/search/*
111Disallow: /*/*/milestones/new$
112Disallow: /*/*/milestones/new?*
113Disallow: /*/*/milestones/new/*
114Disallow: /*/*/network$
115Disallow: /*/*/network?*
116Disallow: /*/*/network/*
117Disallow: /*/*/projects$
118Disallow: /*/*/projects?*
119Disallow: /*/*/projects/*
120Disallow: /*/*/pulse$
121Disallow: /*/*/pulse?*
122Disallow: /*/*/pulse/*
123Disallow: /*/*/pulse-new$
124Disallow: /*/*/pulse-new?*
125Disallow: /*/*/pulse-new/*
126Disallow: /*/*/pulse_committer_data$
127Disallow: /*/*/pulse_committer_data?*
128Disallow: /*/*/pulse_committer_data/*
129Disallow: /*/*/pulse_diffstat_summary$
130Disallow: /*/*/pulse_diffstat_summary?*
131Disallow: /*/*/pulse_diffstat_summary/*
132Disallow: /*/*/revisions$
133Disallow: /*/*/revisions?*
134Disallow: /*/*/revisions/*
135Disallow: /*/*/stargazers$
136Disallow: /*/*/stargazers?*
137Disallow: /*/*/stargazers/*
138Disallow: /*/*/tags$
139Disallow: /*/*/tags?*
140Disallow: /*/*/tags/*
141Disallow: /*/*/watchers$
142Disallow: /*/*/watchers?*
143Disallow: /*/*/watchers/*
144Disallow: /*?source=*
145Disallow: /*&source=*
146Disallow: /*?ref_cta=*
147Disallow: /*&ref_cta=*
148Disallow: /*?plan=*
149Disallow: /*&plan=*
150Disallow: /*?return_to=*
151Disallow: /*&return_to=*
152Disallow: /*?ref_loc=*
153Disallow: /*&ref_loc=*
154Disallow: /*?setup_organization=*
155Disallow: /*&setup_organization=*
156Disallow: /*?source_repo=*
157Disallow: /*&source_repo=*
158Disallow: /*?ref_page=*
159Disallow: /*&ref_page=*
160Disallow: /*?referrer=*
161Disallow: /*&referrer=*
162Disallow: /*?report=*
163Disallow: /*&report=*
164Disallow: /*?author=*
165Disallow: /*&author=*
166Disallow: /*?since=*
167Disallow: /*&since=*
168Disallow: /*?until=*
169Disallow: /*&until=*
170Disallow: /*?tab=*
171Disallow: /*&tab=*
172Disallow: /*?q=*
173Disallow: /*&q=*
174
175User-agent: Bytespider
176Disallow: /
177
178User-agent: AdsBot-Google
179User-agent: AdsBot-Google-Mobile
180Disallow: /account-login
181Disallow: /copilot/
182Disallow: /copilot/c/
183Disallow: /ekansa/Open-Context-Data
184Disallow: /ekansa/opencontext-*
185Disallow: /Explodingstuff/
186Disallow: /gist/
187Disallow: /search/advanced
188Disallow: /search$
189Disallow: /*/*/pulse_committer_data.json$
190Disallow: /*/*/pulse_committer_data.json?*
191Disallow: /*/archive/
192Disallow: /*/blame/
193Disallow: /*/cache/
194Disallow: /*/raw/
195Disallow: /*/tree/
196Disallow: */tarball/
197Disallow: */zipball/
198Disallow: /.git/
199Disallow: */.git/
200Disallow: /*.git$
201Disallow: /*.atom$
202Disallow: /*report-abuse?report=*
203Disallow: /*/*/commits/
204Disallow: /marketplace/*/*?tab=readme$
205Disallow: /marketplace/*/*?tab=transparency$
206Disallow: /marketplace/*?*?tab=readme$
207Disallow: /marketplace/*?*?tab=transparency$
208Disallow: /marketplace/actions?tab=readme$
209Disallow: /marketplace/actions?tab=transparency$
210Disallow: /marketplace/free-trials?tab=readme$
211Disallow: /marketplace/free-trials?tab=transparency$
212Disallow: /marketplace/manage?tab=readme$
213Disallow: /marketplace/manage?tab=transparency$
214Disallow: /marketplace/models?tab=readme$
215Disallow: /marketplace/models?tab=transparency$
216Disallow: /marketplace/new?tab=readme$
217Disallow: /marketplace/new?tab=transparency$
218Disallow: /marketplace/publishers?tab=readme$
219Disallow: /marketplace/publishers?tab=transparency$
220Disallow: /marketplace/recently-added?tab=readme$
221Disallow: /marketplace/recently-added?tab=transparency$
222Disallow: /biztools$
223Disallow: /biztools?*
224Disallow: /biztools/*
225Disallow: /businesses$
226Disallow: /businesses?*
227Disallow: /businesses/*
228Disallow: /enterprises$
229Disallow: /enterprises?*
230Disallow: /enterprises/*
231Disallow: /organizations$
232Disallow: /organizations?*
233Disallow: /organizations/*
234Disallow: /settings$
235Disallow: /settings?*
236Disallow: /settings/*
237Disallow: /stafftools$
238Disallow: /stafftools?*
239Disallow: /stafftools/*
240Disallow: /users/*/billing$
241Disallow: /users/*/billing?*
242Disallow: /users/*/billing/*
243Disallow: /writing$
244Disallow: /writing?*
245Disallow: /writing/*
246Disallow: /*/*/branches$
247Disallow: /*/*/branches?*
248Disallow: /*/*/branches/*
249Disallow: /*/*/comments$
250Disallow: /*/*/comments?*
251Disallow: /*/*/comments/*
252Disallow: /*/*/compare$
253Disallow: /*/*/compare?*
254Disallow: /*/*/compare/*
255Disallow: /*/*/contributors$
256Disallow: /*/*/contributors?*
257Disallow: /*/*/contributors/*
258Disallow: /*/*/contributors_list$
259Disallow: /*/*/contributors_list?*
260Disallow: /*/*/contributors_list/*
261Disallow: /*/*/contributors-list$
262Disallow: /*/*/contributors-list?*
263Disallow: /*/*/contributors-list/*
264Disallow: /*/*/download$
265Disallow: /*/*/download?*
266Disallow: /*/*/download/*
267Disallow: /*/*/forks$
268Disallow: /*/*/forks?*
269Disallow: /*/*/forks/*
270Disallow: /*/*/graphs$
271Disallow: /*/*/graphs?*
272Disallow: /*/*/graphs/*
273Disallow: /*/*/issues/new$
274Disallow: /*/*/issues/new?*
275Disallow: /*/*/issues/new/*
276Disallow: /*/*/issues/search$
277Disallow: /*/*/issues/search?*
278Disallow: /*/*/issues/search/*
279Disallow: /*/*/milestones/new$
280Disallow: /*/*/milestones/new?*
281Disallow: /*/*/milestones/new/*
282Disallow: /*/*/network$
283Disallow: /*/*/network?*
284Disallow: /*/*/network/*
285Disallow: /*/*/projects$
286Disallow: /*/*/projects?*
287Disallow: /*/*/projects/*
288Disallow: /*/*/pulse$
289Disallow: /*/*/pulse?*
290Disallow: /*/*/pulse/*
291Disallow: /*/*/pulse-new$
292Disallow: /*/*/pulse-new?*
293Disallow: /*/*/pulse-new/*
294Disallow: /*/*/pulse_committer_data$
295Disallow: /*/*/pulse_committer_data?*
296Disallow: /*/*/pulse_committer_data/*
297Disallow: /*/*/pulse_diffstat_summary$
298Disallow: /*/*/pulse_diffstat_summary?*
299Disallow: /*/*/pulse_diffstat_summary/*
300Disallow: /*/*/revisions$
301Disallow: /*/*/revisions?*
302Disallow: /*/*/revisions/*
303Disallow: /*/*/stargazers$
304Disallow: /*/*/stargazers?*
305Disallow: /*/*/stargazers/*
306Disallow: /*/*/tags$
307Disallow: /*/*/tags?*
308Disallow: /*/*/tags/*
309Disallow: /*/*/watchers$
310Disallow: /*/*/watchers?*
311Disallow: /*/*/watchers/*
312Disallow: /*?source=*
313Disallow: /*&source=*
314Disallow: /*?ref_cta=*
315Disallow: /*&ref_cta=*
316Disallow: /*?plan=*
317Disallow: /*&plan=*
318Disallow: /*?return_to=*
319Disallow: /*&return_to=*
320Disallow: /*?ref_loc=*
321Disallow: /*&ref_loc=*
322Disallow: /*?setup_organization=*
323Disallow: /*&setup_organization=*
324Disallow: /*?source_repo=*
325Disallow: /*&source_repo=*
326Disallow: /*?ref_page=*
327Disallow: /*&ref_page=*
328Disallow: /*?referrer=*
329Disallow: /*&referrer=*
330Disallow: /*?report=*
331Disallow: /*&report=*
332Disallow: /*?author=*
333Disallow: /*&author=*
334Disallow: /*?since=*
335Disallow: /*&since=*
336Disallow: /*?until=*
337Disallow: /*&until=*
338Disallow: /*?tab=*
339Disallow: /*&tab=*
340Disallow: /*?q=*
341Disallow: /*&q=*
342
343User-agent: bingbot
344User-agent: adidxbot
345User-agent: BingPreview
346Disallow: /ekansa/Open-Context-Data
347Disallow: /ekansa/opencontext-*
348Disallow: /account-login
349Disallow: */tarball/
350Disallow: */zipball/
351Disallow: /Explodingstuff/
352Disallow: /copilot/
353Disallow: /copilot/c/
354
355User-agent: baidu
356crawl-delay: 1
357
358User-agent: *
359Allow: /*?tab=achievements&achievement=*
360Allow: /marketplace/*?tab=readme$
361Allow: /marketplace/*?tab=transparency$
362Disallow: /account-login
363Disallow: /copilot/
364Disallow: /copilot/c/
365Disallow: /ekansa/Open-Context-Data
366Disallow: /ekansa/opencontext-*
367Disallow: /Explodingstuff/
368Disallow: /gist/
369Disallow: /search/advanced
370Disallow: /search$
371Disallow: /*/*/pulse_committer_data.json$
372Disallow: /*/*/pulse_committer_data.json?*
373Disallow: /*/archive/
374Disallow: /*/blame/
375Disallow: /*/cache/
376Disallow: /*/raw/
377Disallow: /*/tree/
378Disallow: */tarball/
379Disallow: */zipball/
380Disallow: /.git/
381Disallow: */.git/
382Disallow: /*.git$
383Disallow: /*.atom$
384Disallow: /*report-abuse?report=*
385Disallow: /*/*/commits/
386Disallow: /marketplace/*/*?tab=readme$
387Disallow: /marketplace/*/*?tab=transparency$
388Disallow: /marketplace/*?*?tab=readme$
389Disallow: /marketplace/*?*?tab=transparency$
390Disallow: /marketplace/actions?tab=readme$
391Disallow: /marketplace/actions?tab=transparency$
392Disallow: /marketplace/free-trials?tab=readme$
393Disallow: /marketplace/free-trials?tab=transparency$
394Disallow: /marketplace/manage?tab=readme$
395Disallow: /marketplace/manage?tab=transparency$
396Disallow: /marketplace/models?tab=readme$
397Disallow: /marketplace/models?tab=transparency$
398Disallow: /marketplace/new?tab=readme$
399Disallow: /marketplace/new?tab=transparency$
400Disallow: /marketplace/publishers?tab=readme$
401Disallow: /marketplace/publishers?tab=transparency$
402Disallow: /marketplace/recently-added?tab=readme$
403Disallow: /marketplace/recently-added?tab=transparency$
404Disallow: /biztools$
405Disallow: /biztools?*
406Disallow: /biztools/*
407Disallow: /businesses$
408Disallow: /businesses?*
409Disallow: /businesses/*
410Disallow: /enterprises$
411Disallow: /enterprises?*
412Disallow: /enterprises/*
413Disallow: /organizations$
414Disallow: /organizations?*
415Disallow: /organizations/*
416Disallow: /settings$
417Disallow: /settings?*
418Disallow: /settings/*
419Disallow: /stafftools$
420Disallow: /stafftools?*
421Disallow: /stafftools/*
422Disallow: /users/*/billing$
423Disallow: /users/*/billing?*
424Disallow: /users/*/billing/*
425Disallow: /writing$
426Disallow: /writing?*
427Disallow: /writing/*
428Disallow: /*/*/branches$
429Disallow: /*/*/branches?*
430Disallow: /*/*/branches/*
431Disallow: /*/*/comments$
432Disallow: /*/*/comments?*
433Disallow: /*/*/comments/*
434Disallow: /*/*/compare$
435Disallow: /*/*/compare?*
436Disallow: /*/*/compare/*
437Disallow: /*/*/contributors$
438Disallow: /*/*/contributors?*
439Disallow: /*/*/contributors/*
440Disallow: /*/*/contributors_list$
441Disallow: /*/*/contributors_list?*
442Disallow: /*/*/contributors_list/*
443Disallow: /*/*/contributors-list$
444Disallow: /*/*/contributors-list?*
445Disallow: /*/*/contributors-list/*
446Disallow: /*/*/download$
447Disallow: /*/*/download?*
448Disallow: /*/*/download/*
449Disallow: /*/*/forks$
450Disallow: /*/*/forks?*
451Disallow: /*/*/forks/*
452Disallow: /*/*/graphs$
453Disallow: /*/*/graphs?*
454Disallow: /*/*/graphs/*
455Disallow: /*/*/issues/new$
456Disallow: /*/*/issues/new?*
457Disallow: /*/*/issues/new/*
458Disallow: /*/*/issues/search$
459Disallow: /*/*/issues/search?*
460Disallow: /*/*/issues/search/*
461Disallow: /*/*/milestones/new$
462Disallow: /*/*/milestones/new?*
463Disallow: /*/*/milestones/new/*
464Disallow: /*/*/network$
465Disallow: /*/*/network?*
466Disallow: /*/*/network/*
467Disallow: /*/*/projects$
468Disallow: /*/*/projects?*
469Disallow: /*/*/projects/*
470Disallow: /*/*/pulse$
471Disallow: /*/*/pulse?*
472Disallow: /*/*/pulse/*
473Disallow: /*/*/pulse-new$
474Disallow: /*/*/pulse-new?*
475Disallow: /*/*/pulse-new/*
476Disallow: /*/*/pulse_committer_data$
477Disallow: /*/*/pulse_committer_data?*
478Disallow: /*/*/pulse_committer_data/*
479Disallow: /*/*/pulse_diffstat_summary$
480Disallow: /*/*/pulse_diffstat_summary?*
481Disallow: /*/*/pulse_diffstat_summary/*
482Disallow: /*/*/revisions$
483Disallow: /*/*/revisions?*
484Disallow: /*/*/revisions/*
485Disallow: /*/*/stargazers$
486Disallow: /*/*/stargazers?*
487Disallow: /*/*/stargazers/*
488Disallow: /*/*/tags$
489Disallow: /*/*/tags?*
490Disallow: /*/*/tags/*
491Disallow: /*/*/watchers$
492Disallow: /*/*/watchers?*
493Disallow: /*/*/watchers/*
494Disallow: /*?source=*
495Disallow: /*&source=*
496Disallow: /*?ref_cta=*
497Disallow: /*&ref_cta=*
498Disallow: /*?plan=*
499Disallow: /*&plan=*
500Disallow: /*?return_to=*
501Disallow: /*&return_to=*
502Disallow: /*?ref_loc=*
503Disallow: /*&ref_loc=*
504Disallow: /*?setup_organization=*
505Disallow: /*&setup_organization=*
506Disallow: /*?source_repo=*
507Disallow: /*&source_repo=*
508Disallow: /*?ref_page=*
509Disallow: /*&ref_page=*
510Disallow: /*?referrer=*
511Disallow: /*&referrer=*
512Disallow: /*?report=*
513Disallow: /*&report=*
514Disallow: /*?author=*
515Disallow: /*&author=*
516Disallow: /*?since=*
517Disallow: /*&since=*
518Disallow: /*?until=*
519Disallow: /*&until=*
520Disallow: /*?tab=*
521Disallow: /*&tab=*
522Disallow: /*?q=*
523Disallow: /*&q=*