NeuralCrawl

GitHub / robots.txt snapshot

← back to github.com · fetched 2026-09-30T19:18:16Z (1mo ago) · HTTP 200 · 14181 bytes · sha256 9b7ae20fa219e3e4 · raw

final URL: https://github.com/robots.txt

1# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots
2# We also provide an extensive API: https://docs.github.com
3User-agent: GPTBot
4User-agent: OAI-SearchBot
5User-agent: ClaudeBot
6User-agent: anthropic-ai
7User-agent: PerplexityBot
8Crawl-delay: 1
9Allow: /$
10Allow: /about
11Allow: /customer-stories
12Allow: /enterprise
13Allow: /features
14Allow: /marketplace
15Allow: /mcp
16Allow: /newsroom
17Allow: /pricing
18Allow: /resources
19Allow: /security
20Allow: /solutions
21Allow: /topics/download*
22Allow: /topics/comments*
23Allow: /marketplace/actions/download*
24Disallow: /*/*/pulse
25Disallow: /*/*/projects
26Disallow: /*/*/forks
27Disallow: /*/*/issues/new
28Disallow: /*/*/milestones/new
29Disallow: /*/*/issues/search
30Disallow: /*/*/commits/
31Disallow: /*/*/branches
32Disallow: /*/*/contributors
33Disallow: /*/*/tags
34Disallow: /*/*/stargazers
35Disallow: /*/*/watchers
36Disallow: /*/*/network
37Disallow: /*/*/graphs
38Disallow: /*/*/compare
39Disallow: /*/tree/
40Disallow: /gist/
41Disallow: /*/download
42Disallow: /*/revisions
43Disallow: /*/commits/*?author
44Disallow: /*/commits/*?path
45Disallow: /*/comments
46Disallow: /*/archive/
47Disallow: /*/blame/
48Disallow: /*/raw/
49Disallow: /*/cache/
50Disallow: /.git/
51Disallow: */.git/
52Disallow: /*.git$
53Disallow: /search/advanced
54Disallow: /search$
55Disallow: /*.atom$
56Disallow: /ekansa/Open-Context-Data
57Disallow: /ekansa/opencontext-*
58Disallow: */tarball/
59Disallow: */zipball/
60Disallow: /*commits?author=*
61Disallow: /*report-abuse?report=*
62Disallow: /account-login
63Disallow: /Explodingstuff/
64Disallow: /copilot/
65Disallow: /copilot/c/
66Disallow: /*?source=*
67Disallow: /*&source=*
68Disallow: /*?ref_cta=*
69Disallow: /*&ref_cta=*
70Disallow: /*?plan=*
71Disallow: /*&plan=*
72Disallow: /*?return_to=*
73Disallow: /*&return_to=*
74Disallow: /*?ref_loc=*
75Disallow: /*&ref_loc=*
76Disallow: /*?setup_organization=*
77Disallow: /*&setup_organization=*
78Disallow: /*?source_repo=*
79Disallow: /*&source_repo=*
80Disallow: /*?ref_page=*
81Disallow: /*&ref_page=*
82Disallow: /*?referrer=*
83Disallow: /*&referrer=*
84Disallow: /*?report=*
85Disallow: /*&report=*
86Disallow: /*?author=*
87Disallow: /*&author=*
88Disallow: /*?since=*
89Disallow: /*&since=*
90Disallow: /*?until=*
91Disallow: /*&until=*
92Disallow: /*?tab=*
93Disallow: /*&tab=*
94Disallow: /*?q=*
95Disallow: /*&q=*
96Disallow: /topics/download*?source=*
97Disallow: /topics/download*&source=*
98Disallow: /topics/download*?ref_cta=*
99Disallow: /topics/download*&ref_cta=*
100Disallow: /topics/download*?plan=*
101Disallow: /topics/download*&plan=*
102Disallow: /topics/download*?return_to=*
103Disallow: /topics/download*&return_to=*
104Disallow: /topics/download*?ref_loc=*
105Disallow: /topics/download*&ref_loc=*
106Disallow: /topics/download*?setup_organization=*
107Disallow: /topics/download*&setup_organization=*
108Disallow: /topics/download*?source_repo=*
109Disallow: /topics/download*&source_repo=*
110Disallow: /topics/download*?ref_page=*
111Disallow: /topics/download*&ref_page=*
112Disallow: /topics/download*?referrer=*
113Disallow: /topics/download*&referrer=*
114Disallow: /topics/download*?report=*
115Disallow: /topics/download*&report=*
116Disallow: /topics/download*?author=*
117Disallow: /topics/download*&author=*
118Disallow: /topics/download*?since=*
119Disallow: /topics/download*&since=*
120Disallow: /topics/download*?until=*
121Disallow: /topics/download*&until=*
122Disallow: /topics/download*?tab=*
123Disallow: /topics/download*&tab=*
124Disallow: /topics/download*?q=*
125Disallow: /topics/download*&q=*
126Disallow: /topics/comments*?source=*
127Disallow: /topics/comments*&source=*
128Disallow: /topics/comments*?ref_cta=*
129Disallow: /topics/comments*&ref_cta=*
130Disallow: /topics/comments*?plan=*
131Disallow: /topics/comments*&plan=*
132Disallow: /topics/comments*?return_to=*
133Disallow: /topics/comments*&return_to=*
134Disallow: /topics/comments*?ref_loc=*
135Disallow: /topics/comments*&ref_loc=*
136Disallow: /topics/comments*?setup_organization=*
137Disallow: /topics/comments*&setup_organization=*
138Disallow: /topics/comments*?source_repo=*
139Disallow: /topics/comments*&source_repo=*
140Disallow: /topics/comments*?ref_page=*
141Disallow: /topics/comments*&ref_page=*
142Disallow: /topics/comments*?referrer=*
143Disallow: /topics/comments*&referrer=*
144Disallow: /topics/comments*?report=*
145Disallow: /topics/comments*&report=*
146Disallow: /topics/comments*?author=*
147Disallow: /topics/comments*&author=*
148Disallow: /topics/comments*?since=*
149Disallow: /topics/comments*&since=*
150Disallow: /topics/comments*?until=*
151Disallow: /topics/comments*&until=*
152Disallow: /topics/comments*?tab=*
153Disallow: /topics/comments*&tab=*
154Disallow: /topics/comments*?q=*
155Disallow: /topics/comments*&q=*
156Disallow: /marketplace/actions/download*?source=*
157Disallow: /marketplace/actions/download*&source=*
158Disallow: /marketplace/actions/download*?ref_cta=*
159Disallow: /marketplace/actions/download*&ref_cta=*
160Disallow: /marketplace/actions/download*?plan=*
161Disallow: /marketplace/actions/download*&plan=*
162Disallow: /marketplace/actions/download*?return_to=*
163Disallow: /marketplace/actions/download*&return_to=*
164Disallow: /marketplace/actions/download*?ref_loc=*
165Disallow: /marketplace/actions/download*&ref_loc=*
166Disallow: /marketplace/actions/download*?setup_organization=*
167Disallow: /marketplace/actions/download*&setup_organization=*
168Disallow: /marketplace/actions/download*?source_repo=*
169Disallow: /marketplace/actions/download*&source_repo=*
170Disallow: /marketplace/actions/download*?ref_page=*
171Disallow: /marketplace/actions/download*&ref_page=*
172Disallow: /marketplace/actions/download*?referrer=*
173Disallow: /marketplace/actions/download*&referrer=*
174Disallow: /marketplace/actions/download*?report=*
175Disallow: /marketplace/actions/download*&report=*
176Disallow: /marketplace/actions/download*?author=*
177Disallow: /marketplace/actions/download*&author=*
178Disallow: /marketplace/actions/download*?since=*
179Disallow: /marketplace/actions/download*&since=*
180Disallow: /marketplace/actions/download*?until=*
181Disallow: /marketplace/actions/download*&until=*
182Disallow: /marketplace/actions/download*?tab=*
183Disallow: /marketplace/actions/download*&tab=*
184Disallow: /marketplace/actions/download*?q=*
185Disallow: /marketplace/actions/download*&q=*
186
187User-agent: Bytespider
188Disallow: /
189
190User-agent: AdsBot-Google
191User-agent: AdsBot-Google-Mobile
192Disallow: /*/*/pulse
193Disallow: /*/*/projects
194Disallow: /*/*/forks
195Disallow: /*/*/issues/new
196Disallow: /*/*/milestones/new
197Disallow: /*/*/issues/search
198Disallow: /*/*/commits/
199Disallow: /*/*/branches
200Disallow: /*/*/contributors
201Disallow: /*/*/tags
202Disallow: /*/*/stargazers
203Disallow: /*/*/watchers
204Disallow: /*/*/network
205Disallow: /*/*/graphs
206Disallow: /*/*/compare
207Disallow: /*/tree/
208Disallow: /gist/
209Disallow: /*/download
210Disallow: /*/revisions
211Disallow: /*/commits/*?author
212Disallow: /*/commits/*?path
213Disallow: /*/comments
214Disallow: /*/archive/
215Disallow: /*/blame/
216Disallow: /*/raw/
217Disallow: /*/cache/
218Disallow: /.git/
219Disallow: */.git/
220Disallow: /*.git$
221Disallow: /search/advanced
222Disallow: /search$
223Disallow: /*.atom$
224Disallow: /ekansa/Open-Context-Data
225Disallow: /ekansa/opencontext-*
226Disallow: */tarball/
227Disallow: */zipball/
228Disallow: /*commits?author=*
229Disallow: /*report-abuse?report=*
230Disallow: /account-login
231Disallow: /Explodingstuff/
232Disallow: /copilot/
233Disallow: /copilot/c/
234Disallow: /*?source=*
235Disallow: /*&source=*
236Disallow: /*?ref_cta=*
237Disallow: /*&ref_cta=*
238Disallow: /*?plan=*
239Disallow: /*&plan=*
240Disallow: /*?return_to=*
241Disallow: /*&return_to=*
242Disallow: /*?ref_loc=*
243Disallow: /*&ref_loc=*
244Disallow: /*?setup_organization=*
245Disallow: /*&setup_organization=*
246Disallow: /*?source_repo=*
247Disallow: /*&source_repo=*
248Disallow: /*?ref_page=*
249Disallow: /*&ref_page=*
250Disallow: /*?referrer=*
251Disallow: /*&referrer=*
252Disallow: /*?report=*
253Disallow: /*&report=*
254Disallow: /*?author=*
255Disallow: /*&author=*
256Disallow: /*?since=*
257Disallow: /*&since=*
258Disallow: /*?until=*
259Disallow: /*&until=*
260Disallow: /*?tab=*
261Disallow: /*&tab=*
262Disallow: /*?q=*
263Disallow: /*&q=*
264
265User-agent: bingbot
266Disallow: /ekansa/Open-Context-Data
267Disallow: /ekansa/opencontext-*
268Disallow: /account-login
269Disallow: */tarball/
270Disallow: */zipball/
271Disallow: /Explodingstuff/
272Disallow: /copilot/
273Disallow: /copilot/c/
274
275User-agent: adidxbot
276Disallow: /ekansa/Open-Context-Data
277Disallow: /ekansa/opencontext-*
278Disallow: /account-login
279Disallow: */tarball/
280Disallow: */zipball/
281Disallow: /Explodingstuff/
282Disallow: /copilot/
283Disallow: /copilot/c/
284
285User-agent: BingPreview
286Disallow: /ekansa/Open-Context-Data
287Disallow: /ekansa/opencontext-*
288Disallow: /account-login
289Disallow: */tarball/
290Disallow: */zipball/
291Disallow: /Explodingstuff/
292Disallow: /copilot/
293Disallow: /copilot/c/
294
295User-agent: baidu
296crawl-delay: 1
297
298
299User-agent: *
300Allow: /*?tab=achievements&achievement=*
301Allow: /topics/download*
302Allow: /topics/comments*
303Allow: /marketplace/actions/download*
304Disallow: /*/*/pulse
305Disallow: /*/*/projects
306Disallow: /*/*/forks
307Disallow: /*/*/issues/new
308Disallow: /*/*/milestones/new
309Disallow: /*/*/issues/search
310Disallow: /*/*/commits/
311Disallow: /*/*/branches
312Disallow: /*/*/contributors
313Disallow: /*/*/tags
314Disallow: /*/*/stargazers
315Disallow: /*/*/watchers
316Disallow: /*/*/network
317Disallow: /*/*/graphs
318Disallow: /*/*/compare
319Disallow: /*/tree/
320Disallow: /gist/
321Disallow: /*/download
322Disallow: /*/revisions
323Disallow: /*/commits/*?author
324Disallow: /*/commits/*?path
325Disallow: /*/comments
326Disallow: /*/archive/
327Disallow: /*/blame/
328Disallow: /*/raw/
329Disallow: /*/cache/
330Disallow: /.git/
331Disallow: */.git/
332Disallow: /*.git$
333Disallow: /search/advanced
334Disallow: /search$
335Disallow: /*.atom$
336Disallow: /ekansa/Open-Context-Data
337Disallow: /ekansa/opencontext-*
338Disallow: */tarball/
339Disallow: */zipball/
340Disallow: /*commits?author=*
341Disallow: /*report-abuse?report=*
342Disallow: /account-login
343Disallow: /Explodingstuff/
344Disallow: /copilot/
345Disallow: /copilot/c/
346Disallow: /*?source=*
347Disallow: /*&source=*
348Disallow: /*?ref_cta=*
349Disallow: /*&ref_cta=*
350Disallow: /*?plan=*
351Disallow: /*&plan=*
352Disallow: /*?return_to=*
353Disallow: /*&return_to=*
354Disallow: /*?ref_loc=*
355Disallow: /*&ref_loc=*
356Disallow: /*?setup_organization=*
357Disallow: /*&setup_organization=*
358Disallow: /*?source_repo=*
359Disallow: /*&source_repo=*
360Disallow: /*?ref_page=*
361Disallow: /*&ref_page=*
362Disallow: /*?referrer=*
363Disallow: /*&referrer=*
364Disallow: /*?report=*
365Disallow: /*&report=*
366Disallow: /*?author=*
367Disallow: /*&author=*
368Disallow: /*?since=*
369Disallow: /*&since=*
370Disallow: /*?until=*
371Disallow: /*&until=*
372Disallow: /*?tab=*
373Disallow: /*&tab=*
374Disallow: /*?q=*
375Disallow: /*&q=*
376Disallow: /topics/download*?source=*
377Disallow: /topics/download*&source=*
378Disallow: /topics/download*?ref_cta=*
379Disallow: /topics/download*&ref_cta=*
380Disallow: /topics/download*?plan=*
381Disallow: /topics/download*&plan=*
382Disallow: /topics/download*?return_to=*
383Disallow: /topics/download*&return_to=*
384Disallow: /topics/download*?ref_loc=*
385Disallow: /topics/download*&ref_loc=*
386Disallow: /topics/download*?setup_organization=*
387Disallow: /topics/download*&setup_organization=*
388Disallow: /topics/download*?source_repo=*
389Disallow: /topics/download*&source_repo=*
390Disallow: /topics/download*?ref_page=*
391Disallow: /topics/download*&ref_page=*
392Disallow: /topics/download*?referrer=*
393Disallow: /topics/download*&referrer=*
394Disallow: /topics/download*?report=*
395Disallow: /topics/download*&report=*
396Disallow: /topics/download*?author=*
397Disallow: /topics/download*&author=*
398Disallow: /topics/download*?since=*
399Disallow: /topics/download*&since=*
400Disallow: /topics/download*?until=*
401Disallow: /topics/download*&until=*
402Disallow: /topics/download*?tab=*
403Disallow: /topics/download*&tab=*
404Disallow: /topics/download*?q=*
405Disallow: /topics/download*&q=*
406Disallow: /topics/comments*?source=*
407Disallow: /topics/comments*&source=*
408Disallow: /topics/comments*?ref_cta=*
409Disallow: /topics/comments*&ref_cta=*
410Disallow: /topics/comments*?plan=*
411Disallow: /topics/comments*&plan=*
412Disallow: /topics/comments*?return_to=*
413Disallow: /topics/comments*&return_to=*
414Disallow: /topics/comments*?ref_loc=*
415Disallow: /topics/comments*&ref_loc=*
416Disallow: /topics/comments*?setup_organization=*
417Disallow: /topics/comments*&setup_organization=*
418Disallow: /topics/comments*?source_repo=*
419Disallow: /topics/comments*&source_repo=*
420Disallow: /topics/comments*?ref_page=*
421Disallow: /topics/comments*&ref_page=*
422Disallow: /topics/comments*?referrer=*
423Disallow: /topics/comments*&referrer=*
424Disallow: /topics/comments*?report=*
425Disallow: /topics/comments*&report=*
426Disallow: /topics/comments*?author=*
427Disallow: /topics/comments*&author=*
428Disallow: /topics/comments*?since=*
429Disallow: /topics/comments*&since=*
430Disallow: /topics/comments*?until=*
431Disallow: /topics/comments*&until=*
432Disallow: /topics/comments*?tab=*
433Disallow: /topics/comments*&tab=*
434Disallow: /topics/comments*?q=*
435Disallow: /topics/comments*&q=*
436Disallow: /marketplace/actions/download*?source=*
437Disallow: /marketplace/actions/download*&source=*
438Disallow: /marketplace/actions/download*?ref_cta=*
439Disallow: /marketplace/actions/download*&ref_cta=*
440Disallow: /marketplace/actions/download*?plan=*
441Disallow: /marketplace/actions/download*&plan=*
442Disallow: /marketplace/actions/download*?return_to=*
443Disallow: /marketplace/actions/download*&return_to=*
444Disallow: /marketplace/actions/download*?ref_loc=*
445Disallow: /marketplace/actions/download*&ref_loc=*
446Disallow: /marketplace/actions/download*?setup_organization=*
447Disallow: /marketplace/actions/download*&setup_organization=*
448Disallow: /marketplace/actions/download*?source_repo=*
449Disallow: /marketplace/actions/download*&source_repo=*
450Disallow: /marketplace/actions/download*?ref_page=*
451Disallow: /marketplace/actions/download*&ref_page=*
452Disallow: /marketplace/actions/download*?referrer=*
453Disallow: /marketplace/actions/download*&referrer=*
454Disallow: /marketplace/actions/download*?report=*
455Disallow: /marketplace/actions/download*&report=*
456Disallow: /marketplace/actions/download*?author=*
457Disallow: /marketplace/actions/download*&author=*
458Disallow: /marketplace/actions/download*?since=*
459Disallow: /marketplace/actions/download*&since=*
460Disallow: /marketplace/actions/download*?until=*
461Disallow: /marketplace/actions/download*&until=*
462Disallow: /marketplace/actions/download*?tab=*
463Disallow: /marketplace/actions/download*&tab=*
464Disallow: /marketplace/actions/download*?q=*
465Disallow: /marketplace/actions/download*&q=*