NeuralCrawl

GitHub / robots.txt snapshot

← back to github.com · fetched 2026-10-02T01:50:39Z (1mo ago) · HTTP 200 · 16739 bytes · sha256 12bc55582b1ddb8e · raw

final URL: https://github.com/robots.txt

1# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots
2# We also provide an extensive API: https://docs.github.com
3User-agent: GPTBot
4User-agent: OAI-SearchBot
5User-agent: ClaudeBot
6User-agent: anthropic-ai
7User-agent: PerplexityBot
8Crawl-delay: 1
9Allow: /$
10Allow: /about
11Allow: /customer-stories
12Allow: /enterprise
13Allow: /features
14Allow: /marketplace
15Allow: /mcp
16Allow: /newsroom
17Allow: /pricing
18Allow: /resources
19Allow: /security
20Allow: /solutions
21Allow: /topics/download*
22Allow: /topics/comments*
23Allow: /marketplace/actions/download*
24Disallow: /*/*/pulse
25Disallow: /*/*/projects
26Disallow: /*/*/forks
27Disallow: /*/*/issues/new
28Disallow: /*/*/milestones/new
29Disallow: /*/*/issues/search
30Disallow: /*/*/commits/
31Disallow: /*/*/branches
32Disallow: /*/*/contributors
33Disallow: /*/*/tags
34Disallow: /*/*/stargazers
35Disallow: /*/*/watchers
36Disallow: /*/*/network
37Disallow: /*/*/graphs
38Disallow: /*/*/compare
39Disallow: /*/tree/
40Disallow: /gist/
41Disallow: /*/download
42Disallow: /*/revisions
43Disallow: /*/commits/*?author
44Disallow: /*/commits/*?path
45Disallow: /*/comments
46Disallow: /*/archive/
47Disallow: /*/blame/
48Disallow: /*/raw/
49Disallow: /*/cache/
50Disallow: /.git/
51Disallow: */.git/
52Disallow: /*.git$
53Disallow: /search/advanced
54Disallow: /search$
55Disallow: /*.atom$
56Disallow: /ekansa/Open-Context-Data
57Disallow: /ekansa/opencontext-*
58Disallow: */tarball/
59Disallow: */zipball/
60Disallow: /*commits?author=*
61Disallow: /*report-abuse?report=*
62Disallow: /account-login
63Disallow: /Explodingstuff/
64Disallow: /copilot/
65Disallow: /copilot/c/
66Disallow: /marketplace/*/*?tab=readme$
67Disallow: /marketplace/*?*?tab=readme$
68Disallow: /marketplace/actions?tab=readme$
69Disallow: /marketplace/free-trials?tab=readme$
70Disallow: /marketplace/manage?tab=readme$
71Disallow: /marketplace/models?tab=readme$
72Disallow: /marketplace/new?tab=readme$
73Disallow: /marketplace/publishers?tab=readme$
74Disallow: /marketplace/recently-added?tab=readme$
75Disallow: /marketplace/*/*?tab=transparency$
76Disallow: /marketplace/*?*?tab=transparency$
77Disallow: /marketplace/actions?tab=transparency$
78Disallow: /marketplace/free-trials?tab=transparency$
79Disallow: /marketplace/manage?tab=transparency$
80Disallow: /marketplace/models?tab=transparency$
81Disallow: /marketplace/new?tab=transparency$
82Disallow: /marketplace/publishers?tab=transparency$
83Disallow: /marketplace/recently-added?tab=transparency$
84Disallow: /*?source=*
85Disallow: /*&source=*
86Disallow: /*?ref_cta=*
87Disallow: /*&ref_cta=*
88Disallow: /*?plan=*
89Disallow: /*&plan=*
90Disallow: /*?return_to=*
91Disallow: /*&return_to=*
92Disallow: /*?ref_loc=*
93Disallow: /*&ref_loc=*
94Disallow: /*?setup_organization=*
95Disallow: /*&setup_organization=*
96Disallow: /*?source_repo=*
97Disallow: /*&source_repo=*
98Disallow: /*?ref_page=*
99Disallow: /*&ref_page=*
100Disallow: /*?referrer=*
101Disallow: /*&referrer=*
102Disallow: /*?report=*
103Disallow: /*&report=*
104Disallow: /*?author=*
105Disallow: /*&author=*
106Disallow: /*?since=*
107Disallow: /*&since=*
108Disallow: /*?until=*
109Disallow: /*&until=*
110Disallow: /*?tab=*
111Disallow: /*&tab=*
112Disallow: /*?q=*
113Disallow: /*&q=*
114Disallow: /topics/download*?source=*
115Disallow: /topics/download*&source=*
116Disallow: /topics/download*?ref_cta=*
117Disallow: /topics/download*&ref_cta=*
118Disallow: /topics/download*?plan=*
119Disallow: /topics/download*&plan=*
120Disallow: /topics/download*?return_to=*
121Disallow: /topics/download*&return_to=*
122Disallow: /topics/download*?ref_loc=*
123Disallow: /topics/download*&ref_loc=*
124Disallow: /topics/download*?setup_organization=*
125Disallow: /topics/download*&setup_organization=*
126Disallow: /topics/download*?source_repo=*
127Disallow: /topics/download*&source_repo=*
128Disallow: /topics/download*?ref_page=*
129Disallow: /topics/download*&ref_page=*
130Disallow: /topics/download*?referrer=*
131Disallow: /topics/download*&referrer=*
132Disallow: /topics/download*?report=*
133Disallow: /topics/download*&report=*
134Disallow: /topics/download*?author=*
135Disallow: /topics/download*&author=*
136Disallow: /topics/download*?since=*
137Disallow: /topics/download*&since=*
138Disallow: /topics/download*?until=*
139Disallow: /topics/download*&until=*
140Disallow: /topics/download*?tab=*
141Disallow: /topics/download*&tab=*
142Disallow: /topics/download*?q=*
143Disallow: /topics/download*&q=*
144Disallow: /topics/comments*?source=*
145Disallow: /topics/comments*&source=*
146Disallow: /topics/comments*?ref_cta=*
147Disallow: /topics/comments*&ref_cta=*
148Disallow: /topics/comments*?plan=*
149Disallow: /topics/comments*&plan=*
150Disallow: /topics/comments*?return_to=*
151Disallow: /topics/comments*&return_to=*
152Disallow: /topics/comments*?ref_loc=*
153Disallow: /topics/comments*&ref_loc=*
154Disallow: /topics/comments*?setup_organization=*
155Disallow: /topics/comments*&setup_organization=*
156Disallow: /topics/comments*?source_repo=*
157Disallow: /topics/comments*&source_repo=*
158Disallow: /topics/comments*?ref_page=*
159Disallow: /topics/comments*&ref_page=*
160Disallow: /topics/comments*?referrer=*
161Disallow: /topics/comments*&referrer=*
162Disallow: /topics/comments*?report=*
163Disallow: /topics/comments*&report=*
164Disallow: /topics/comments*?author=*
165Disallow: /topics/comments*&author=*
166Disallow: /topics/comments*?since=*
167Disallow: /topics/comments*&since=*
168Disallow: /topics/comments*?until=*
169Disallow: /topics/comments*&until=*
170Disallow: /topics/comments*?tab=*
171Disallow: /topics/comments*&tab=*
172Disallow: /topics/comments*?q=*
173Disallow: /topics/comments*&q=*
174Disallow: /marketplace/actions/download*?source=*
175Disallow: /marketplace/actions/download*&source=*
176Disallow: /marketplace/actions/download*?ref_cta=*
177Disallow: /marketplace/actions/download*&ref_cta=*
178Disallow: /marketplace/actions/download*?plan=*
179Disallow: /marketplace/actions/download*&plan=*
180Disallow: /marketplace/actions/download*?return_to=*
181Disallow: /marketplace/actions/download*&return_to=*
182Disallow: /marketplace/actions/download*?ref_loc=*
183Disallow: /marketplace/actions/download*&ref_loc=*
184Disallow: /marketplace/actions/download*?setup_organization=*
185Disallow: /marketplace/actions/download*&setup_organization=*
186Disallow: /marketplace/actions/download*?source_repo=*
187Disallow: /marketplace/actions/download*&source_repo=*
188Disallow: /marketplace/actions/download*?ref_page=*
189Disallow: /marketplace/actions/download*&ref_page=*
190Disallow: /marketplace/actions/download*?referrer=*
191Disallow: /marketplace/actions/download*&referrer=*
192Disallow: /marketplace/actions/download*?report=*
193Disallow: /marketplace/actions/download*&report=*
194Disallow: /marketplace/actions/download*?author=*
195Disallow: /marketplace/actions/download*&author=*
196Disallow: /marketplace/actions/download*?since=*
197Disallow: /marketplace/actions/download*&since=*
198Disallow: /marketplace/actions/download*?until=*
199Disallow: /marketplace/actions/download*&until=*
200Disallow: /marketplace/actions/download*?tab=*
201Disallow: /marketplace/actions/download*&tab=*
202Disallow: /marketplace/actions/download*?q=*
203Disallow: /marketplace/actions/download*&q=*
204
205User-agent: Bytespider
206Disallow: /
207
208User-agent: AdsBot-Google
209User-agent: AdsBot-Google-Mobile
210Disallow: /*/*/pulse
211Disallow: /*/*/projects
212Disallow: /*/*/forks
213Disallow: /*/*/issues/new
214Disallow: /*/*/milestones/new
215Disallow: /*/*/issues/search
216Disallow: /*/*/commits/
217Disallow: /*/*/branches
218Disallow: /*/*/contributors
219Disallow: /*/*/tags
220Disallow: /*/*/stargazers
221Disallow: /*/*/watchers
222Disallow: /*/*/network
223Disallow: /*/*/graphs
224Disallow: /*/*/compare
225Disallow: /*/tree/
226Disallow: /gist/
227Disallow: /*/download
228Disallow: /*/revisions
229Disallow: /*/commits/*?author
230Disallow: /*/commits/*?path
231Disallow: /*/comments
232Disallow: /*/archive/
233Disallow: /*/blame/
234Disallow: /*/raw/
235Disallow: /*/cache/
236Disallow: /.git/
237Disallow: */.git/
238Disallow: /*.git$
239Disallow: /search/advanced
240Disallow: /search$
241Disallow: /*.atom$
242Disallow: /ekansa/Open-Context-Data
243Disallow: /ekansa/opencontext-*
244Disallow: */tarball/
245Disallow: */zipball/
246Disallow: /*commits?author=*
247Disallow: /*report-abuse?report=*
248Disallow: /account-login
249Disallow: /Explodingstuff/
250Disallow: /copilot/
251Disallow: /copilot/c/
252Disallow: /marketplace/*/*?tab=readme$
253Disallow: /marketplace/*?*?tab=readme$
254Disallow: /marketplace/actions?tab=readme$
255Disallow: /marketplace/free-trials?tab=readme$
256Disallow: /marketplace/manage?tab=readme$
257Disallow: /marketplace/models?tab=readme$
258Disallow: /marketplace/new?tab=readme$
259Disallow: /marketplace/publishers?tab=readme$
260Disallow: /marketplace/recently-added?tab=readme$
261Disallow: /marketplace/*/*?tab=transparency$
262Disallow: /marketplace/*?*?tab=transparency$
263Disallow: /marketplace/actions?tab=transparency$
264Disallow: /marketplace/free-trials?tab=transparency$
265Disallow: /marketplace/manage?tab=transparency$
266Disallow: /marketplace/models?tab=transparency$
267Disallow: /marketplace/new?tab=transparency$
268Disallow: /marketplace/publishers?tab=transparency$
269Disallow: /marketplace/recently-added?tab=transparency$
270Disallow: /*?source=*
271Disallow: /*&source=*
272Disallow: /*?ref_cta=*
273Disallow: /*&ref_cta=*
274Disallow: /*?plan=*
275Disallow: /*&plan=*
276Disallow: /*?return_to=*
277Disallow: /*&return_to=*
278Disallow: /*?ref_loc=*
279Disallow: /*&ref_loc=*
280Disallow: /*?setup_organization=*
281Disallow: /*&setup_organization=*
282Disallow: /*?source_repo=*
283Disallow: /*&source_repo=*
284Disallow: /*?ref_page=*
285Disallow: /*&ref_page=*
286Disallow: /*?referrer=*
287Disallow: /*&referrer=*
288Disallow: /*?report=*
289Disallow: /*&report=*
290Disallow: /*?author=*
291Disallow: /*&author=*
292Disallow: /*?since=*
293Disallow: /*&since=*
294Disallow: /*?until=*
295Disallow: /*&until=*
296Disallow: /*?tab=*
297Disallow: /*&tab=*
298Disallow: /*?q=*
299Disallow: /*&q=*
300
301User-agent: bingbot
302Disallow: /ekansa/Open-Context-Data
303Disallow: /ekansa/opencontext-*
304Disallow: /account-login
305Disallow: */tarball/
306Disallow: */zipball/
307Disallow: /Explodingstuff/
308Disallow: /copilot/
309Disallow: /copilot/c/
310
311User-agent: adidxbot
312Disallow: /ekansa/Open-Context-Data
313Disallow: /ekansa/opencontext-*
314Disallow: /account-login
315Disallow: */tarball/
316Disallow: */zipball/
317Disallow: /Explodingstuff/
318Disallow: /copilot/
319Disallow: /copilot/c/
320
321User-agent: BingPreview
322Disallow: /ekansa/Open-Context-Data
323Disallow: /ekansa/opencontext-*
324Disallow: /account-login
325Disallow: */tarball/
326Disallow: */zipball/
327Disallow: /Explodingstuff/
328Disallow: /copilot/
329Disallow: /copilot/c/
330
331User-agent: baidu
332crawl-delay: 1
333
334
335User-agent: *
336Allow: /*?tab=achievements&achievement=*
337Allow: /marketplace/*?tab=readme$
338Allow: /marketplace/*?tab=transparency$
339Allow: /topics/download*
340Allow: /topics/comments*
341Allow: /marketplace/actions/download*
342Disallow: /*/*/pulse
343Disallow: /*/*/projects
344Disallow: /*/*/forks
345Disallow: /*/*/issues/new
346Disallow: /*/*/milestones/new
347Disallow: /*/*/issues/search
348Disallow: /*/*/commits/
349Disallow: /*/*/branches
350Disallow: /*/*/contributors
351Disallow: /*/*/tags
352Disallow: /*/*/stargazers
353Disallow: /*/*/watchers
354Disallow: /*/*/network
355Disallow: /*/*/graphs
356Disallow: /*/*/compare
357Disallow: /*/tree/
358Disallow: /gist/
359Disallow: /*/download
360Disallow: /*/revisions
361Disallow: /*/commits/*?author
362Disallow: /*/commits/*?path
363Disallow: /*/comments
364Disallow: /*/archive/
365Disallow: /*/blame/
366Disallow: /*/raw/
367Disallow: /*/cache/
368Disallow: /.git/
369Disallow: */.git/
370Disallow: /*.git$
371Disallow: /search/advanced
372Disallow: /search$
373Disallow: /*.atom$
374Disallow: /ekansa/Open-Context-Data
375Disallow: /ekansa/opencontext-*
376Disallow: */tarball/
377Disallow: */zipball/
378Disallow: /*commits?author=*
379Disallow: /*report-abuse?report=*
380Disallow: /account-login
381Disallow: /Explodingstuff/
382Disallow: /copilot/
383Disallow: /copilot/c/
384Disallow: /marketplace/*/*?tab=readme$
385Disallow: /marketplace/*?*?tab=readme$
386Disallow: /marketplace/actions?tab=readme$
387Disallow: /marketplace/free-trials?tab=readme$
388Disallow: /marketplace/manage?tab=readme$
389Disallow: /marketplace/models?tab=readme$
390Disallow: /marketplace/new?tab=readme$
391Disallow: /marketplace/publishers?tab=readme$
392Disallow: /marketplace/recently-added?tab=readme$
393Disallow: /marketplace/*/*?tab=transparency$
394Disallow: /marketplace/*?*?tab=transparency$
395Disallow: /marketplace/actions?tab=transparency$
396Disallow: /marketplace/free-trials?tab=transparency$
397Disallow: /marketplace/manage?tab=transparency$
398Disallow: /marketplace/models?tab=transparency$
399Disallow: /marketplace/new?tab=transparency$
400Disallow: /marketplace/publishers?tab=transparency$
401Disallow: /marketplace/recently-added?tab=transparency$
402Disallow: /*?source=*
403Disallow: /*&source=*
404Disallow: /*?ref_cta=*
405Disallow: /*&ref_cta=*
406Disallow: /*?plan=*
407Disallow: /*&plan=*
408Disallow: /*?return_to=*
409Disallow: /*&return_to=*
410Disallow: /*?ref_loc=*
411Disallow: /*&ref_loc=*
412Disallow: /*?setup_organization=*
413Disallow: /*&setup_organization=*
414Disallow: /*?source_repo=*
415Disallow: /*&source_repo=*
416Disallow: /*?ref_page=*
417Disallow: /*&ref_page=*
418Disallow: /*?referrer=*
419Disallow: /*&referrer=*
420Disallow: /*?report=*
421Disallow: /*&report=*
422Disallow: /*?author=*
423Disallow: /*&author=*
424Disallow: /*?since=*
425Disallow: /*&since=*
426Disallow: /*?until=*
427Disallow: /*&until=*
428Disallow: /*?tab=*
429Disallow: /*&tab=*
430Disallow: /*?q=*
431Disallow: /*&q=*
432Disallow: /topics/download*?source=*
433Disallow: /topics/download*&source=*
434Disallow: /topics/download*?ref_cta=*
435Disallow: /topics/download*&ref_cta=*
436Disallow: /topics/download*?plan=*
437Disallow: /topics/download*&plan=*
438Disallow: /topics/download*?return_to=*
439Disallow: /topics/download*&return_to=*
440Disallow: /topics/download*?ref_loc=*
441Disallow: /topics/download*&ref_loc=*
442Disallow: /topics/download*?setup_organization=*
443Disallow: /topics/download*&setup_organization=*
444Disallow: /topics/download*?source_repo=*
445Disallow: /topics/download*&source_repo=*
446Disallow: /topics/download*?ref_page=*
447Disallow: /topics/download*&ref_page=*
448Disallow: /topics/download*?referrer=*
449Disallow: /topics/download*&referrer=*
450Disallow: /topics/download*?report=*
451Disallow: /topics/download*&report=*
452Disallow: /topics/download*?author=*
453Disallow: /topics/download*&author=*
454Disallow: /topics/download*?since=*
455Disallow: /topics/download*&since=*
456Disallow: /topics/download*?until=*
457Disallow: /topics/download*&until=*
458Disallow: /topics/download*?tab=*
459Disallow: /topics/download*&tab=*
460Disallow: /topics/download*?q=*
461Disallow: /topics/download*&q=*
462Disallow: /topics/comments*?source=*
463Disallow: /topics/comments*&source=*
464Disallow: /topics/comments*?ref_cta=*
465Disallow: /topics/comments*&ref_cta=*
466Disallow: /topics/comments*?plan=*
467Disallow: /topics/comments*&plan=*
468Disallow: /topics/comments*?return_to=*
469Disallow: /topics/comments*&return_to=*
470Disallow: /topics/comments*?ref_loc=*
471Disallow: /topics/comments*&ref_loc=*
472Disallow: /topics/comments*?setup_organization=*
473Disallow: /topics/comments*&setup_organization=*
474Disallow: /topics/comments*?source_repo=*
475Disallow: /topics/comments*&source_repo=*
476Disallow: /topics/comments*?ref_page=*
477Disallow: /topics/comments*&ref_page=*
478Disallow: /topics/comments*?referrer=*
479Disallow: /topics/comments*&referrer=*
480Disallow: /topics/comments*?report=*
481Disallow: /topics/comments*&report=*
482Disallow: /topics/comments*?author=*
483Disallow: /topics/comments*&author=*
484Disallow: /topics/comments*?since=*
485Disallow: /topics/comments*&since=*
486Disallow: /topics/comments*?until=*
487Disallow: /topics/comments*&until=*
488Disallow: /topics/comments*?tab=*
489Disallow: /topics/comments*&tab=*
490Disallow: /topics/comments*?q=*
491Disallow: /topics/comments*&q=*
492Disallow: /marketplace/actions/download*?source=*
493Disallow: /marketplace/actions/download*&source=*
494Disallow: /marketplace/actions/download*?ref_cta=*
495Disallow: /marketplace/actions/download*&ref_cta=*
496Disallow: /marketplace/actions/download*?plan=*
497Disallow: /marketplace/actions/download*&plan=*
498Disallow: /marketplace/actions/download*?return_to=*
499Disallow: /marketplace/actions/download*&return_to=*
500Disallow: /marketplace/actions/download*?ref_loc=*
501Disallow: /marketplace/actions/download*&ref_loc=*
502Disallow: /marketplace/actions/download*?setup_organization=*
503Disallow: /marketplace/actions/download*&setup_organization=*
504Disallow: /marketplace/actions/download*?source_repo=*
505Disallow: /marketplace/actions/download*&source_repo=*
506Disallow: /marketplace/actions/download*?ref_page=*
507Disallow: /marketplace/actions/download*&ref_page=*
508Disallow: /marketplace/actions/download*?referrer=*
509Disallow: /marketplace/actions/download*&referrer=*
510Disallow: /marketplace/actions/download*?report=*
511Disallow: /marketplace/actions/download*&report=*
512Disallow: /marketplace/actions/download*?author=*
513Disallow: /marketplace/actions/download*&author=*
514Disallow: /marketplace/actions/download*?since=*
515Disallow: /marketplace/actions/download*&since=*
516Disallow: /marketplace/actions/download*?until=*
517Disallow: /marketplace/actions/download*&until=*
518Disallow: /marketplace/actions/download*?tab=*
519Disallow: /marketplace/actions/download*&tab=*
520Disallow: /marketplace/actions/download*?q=*
521Disallow: /marketplace/actions/download*&q=*