NeuralCrawl

GitHub / robots.txt snapshot

← back to github.com · fetched 2026-09-18T15:27:37Z (20h ago) · HTTP 200 · 6397 bytes · sha256 384a9642ae240a8f · raw

final URL: https://github.com/robots.txt

1# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots
2# We also provide an extensive API: https://docs.github.com
3User-agent: GPTBot
4User-agent: OAI-SearchBot
5User-agent: ClaudeBot
6User-agent: anthropic-ai
7User-agent: PerplexityBot
8Crawl-delay: 1
9Allow: /$
10Allow: /about
11Allow: /customer-stories
12Allow: /enterprise
13Allow: /features
14Allow: /marketplace
15Allow: /mcp
16Allow: /newsroom
17Allow: /pricing
18Allow: /resources
19Allow: /security
20Allow: /solutions
21Disallow: /*/*/pulse
22Disallow: /*/*/projects
23Disallow: /*/*/forks
24Disallow: /*/*/issues/new
25Disallow: /*/*/milestones/new
26Disallow: /*/*/issues/search
27Disallow: /*/*/commits/
28Disallow: /*/*/branches
29Disallow: /*/*/contributors
30Disallow: /*/*/tags
31Disallow: /*/*/stargazers
32Disallow: /*/*/watchers
33Disallow: /*/*/network
34Disallow: /*/*/graphs
35Disallow: /*/*/compare
36Disallow: /*/tree/
37Disallow: /gist/
38Disallow: /*/download
39Disallow: /*/revisions
40Disallow: /*/commits/*?author
41Disallow: /*/commits/*?path
42Disallow: /*/comments
43Disallow: /*/archive/
44Disallow: /*/blame/
45Disallow: /*/raw/
46Disallow: /*/cache/
47Disallow: /.git/
48Disallow: */.git/
49Disallow: /*.git$
50Disallow: /search/advanced
51Disallow: /search$
52Disallow: /*.atom$
53Disallow: /ekansa/Open-Context-Data
54Disallow: /ekansa/opencontext-*
55Disallow: */tarball/
56Disallow: */zipball/
57Disallow: /*commits?author=*
58Disallow: /*report-abuse?report=*
59Disallow: /account-login
60Disallow: /Explodingstuff/
61Disallow: /copilot/
62Disallow: /copilot/c/
63Disallow: /*?source=*
64Disallow: /*&source=*
65Disallow: /*?ref_cta=*
66Disallow: /*&ref_cta=*
67Disallow: /*?plan=*
68Disallow: /*&plan=*
69Disallow: /*?return_to=*
70Disallow: /*&return_to=*
71Disallow: /*?ref_loc=*
72Disallow: /*&ref_loc=*
73Disallow: /*?setup_organization=*
74Disallow: /*&setup_organization=*
75Disallow: /*?source_repo=*
76Disallow: /*&source_repo=*
77Disallow: /*?ref_page=*
78Disallow: /*&ref_page=*
79Disallow: /*?referrer=*
80Disallow: /*&referrer=*
81Disallow: /*?report=*
82Disallow: /*&report=*
83Disallow: /*?author=*
84Disallow: /*&author=*
85Disallow: /*?since=*
86Disallow: /*&since=*
87Disallow: /*?until=*
88Disallow: /*&until=*
89Disallow: /*?tab=*
90Disallow: /*&tab=*
91Disallow: /*?q=*
92Disallow: /*&q=*
93
94User-agent: Bytespider
95Disallow: /
96
97User-agent: AdsBot-Google
98User-agent: AdsBot-Google-Mobile
99Disallow: /*/*/pulse
100Disallow: /*/*/projects
101Disallow: /*/*/forks
102Disallow: /*/*/issues/new
103Disallow: /*/*/milestones/new
104Disallow: /*/*/issues/search
105Disallow: /*/*/commits/
106Disallow: /*/*/branches
107Disallow: /*/*/contributors
108Disallow: /*/*/tags
109Disallow: /*/*/stargazers
110Disallow: /*/*/watchers
111Disallow: /*/*/network
112Disallow: /*/*/graphs
113Disallow: /*/*/compare
114Disallow: /*/tree/
115Disallow: /gist/
116Disallow: /*/download
117Disallow: /*/revisions
118Disallow: /*/commits/*?author
119Disallow: /*/commits/*?path
120Disallow: /*/comments
121Disallow: /*/archive/
122Disallow: /*/blame/
123Disallow: /*/raw/
124Disallow: /*/cache/
125Disallow: /.git/
126Disallow: */.git/
127Disallow: /*.git$
128Disallow: /search/advanced
129Disallow: /search$
130Disallow: /*.atom$
131Disallow: /ekansa/Open-Context-Data
132Disallow: /ekansa/opencontext-*
133Disallow: */tarball/
134Disallow: */zipball/
135Disallow: /*commits?author=*
136Disallow: /*report-abuse?report=*
137Disallow: /account-login
138Disallow: /Explodingstuff/
139Disallow: /copilot/
140Disallow: /copilot/c/
141Disallow: /*?source=*
142Disallow: /*&source=*
143Disallow: /*?ref_cta=*
144Disallow: /*&ref_cta=*
145Disallow: /*?plan=*
146Disallow: /*&plan=*
147Disallow: /*?return_to=*
148Disallow: /*&return_to=*
149Disallow: /*?ref_loc=*
150Disallow: /*&ref_loc=*
151Disallow: /*?setup_organization=*
152Disallow: /*&setup_organization=*
153Disallow: /*?source_repo=*
154Disallow: /*&source_repo=*
155Disallow: /*?ref_page=*
156Disallow: /*&ref_page=*
157Disallow: /*?referrer=*
158Disallow: /*&referrer=*
159Disallow: /*?report=*
160Disallow: /*&report=*
161Disallow: /*?author=*
162Disallow: /*&author=*
163Disallow: /*?since=*
164Disallow: /*&since=*
165Disallow: /*?until=*
166Disallow: /*&until=*
167Disallow: /*?tab=*
168Disallow: /*&tab=*
169Disallow: /*?q=*
170Disallow: /*&q=*
171
172User-agent: bingbot
173Disallow: /ekansa/Open-Context-Data
174Disallow: /ekansa/opencontext-*
175Disallow: /account-login
176Disallow: */tarball/
177Disallow: */zipball/
178Disallow: /Explodingstuff/
179Disallow: /copilot/
180Disallow: /copilot/c/
181
182User-agent: adidxbot
183Disallow: /ekansa/Open-Context-Data
184Disallow: /ekansa/opencontext-*
185Disallow: /account-login
186Disallow: */tarball/
187Disallow: */zipball/
188Disallow: /Explodingstuff/
189Disallow: /copilot/
190Disallow: /copilot/c/
191
192User-agent: BingPreview
193Disallow: /ekansa/Open-Context-Data
194Disallow: /ekansa/opencontext-*
195Disallow: /account-login
196Disallow: */tarball/
197Disallow: */zipball/
198Disallow: /Explodingstuff/
199Disallow: /copilot/
200Disallow: /copilot/c/
201
202User-agent: baidu
203crawl-delay: 1
204
205
206User-agent: *
207Allow: /*?tab=achievements&achievement=*
208Disallow: /*/*/pulse
209Disallow: /*/*/projects
210Disallow: /*/*/forks
211Disallow: /*/*/issues/new
212Disallow: /*/*/milestones/new
213Disallow: /*/*/issues/search
214Disallow: /*/*/commits/
215Disallow: /*/*/branches
216Disallow: /*/*/contributors
217Disallow: /*/*/tags
218Disallow: /*/*/stargazers
219Disallow: /*/*/watchers
220Disallow: /*/*/network
221Disallow: /*/*/graphs
222Disallow: /*/*/compare
223Disallow: /*/tree/
224Disallow: /gist/
225Disallow: /*/download
226Disallow: /*/revisions
227Disallow: /*/commits/*?author
228Disallow: /*/commits/*?path
229Disallow: /*/comments
230Disallow: /*/archive/
231Disallow: /*/blame/
232Disallow: /*/raw/
233Disallow: /*/cache/
234Disallow: /.git/
235Disallow: */.git/
236Disallow: /*.git$
237Disallow: /search/advanced
238Disallow: /search$
239Disallow: /*.atom$
240Disallow: /ekansa/Open-Context-Data
241Disallow: /ekansa/opencontext-*
242Disallow: */tarball/
243Disallow: */zipball/
244Disallow: /*commits?author=*
245Disallow: /*report-abuse?report=*
246Disallow: /account-login
247Disallow: /Explodingstuff/
248Disallow: /copilot/
249Disallow: /copilot/c/
250Disallow: /*?source=*
251Disallow: /*&source=*
252Disallow: /*?ref_cta=*
253Disallow: /*&ref_cta=*
254Disallow: /*?plan=*
255Disallow: /*&plan=*
256Disallow: /*?return_to=*
257Disallow: /*&return_to=*
258Disallow: /*?ref_loc=*
259Disallow: /*&ref_loc=*
260Disallow: /*?setup_organization=*
261Disallow: /*&setup_organization=*
262Disallow: /*?source_repo=*
263Disallow: /*&source_repo=*
264Disallow: /*?ref_page=*
265Disallow: /*&ref_page=*
266Disallow: /*?referrer=*
267Disallow: /*&referrer=*
268Disallow: /*?report=*
269Disallow: /*&report=*
270Disallow: /*?author=*
271Disallow: /*&author=*
272Disallow: /*?since=*
273Disallow: /*&since=*
274Disallow: /*?until=*
275Disallow: /*&until=*
276Disallow: /*?tab=*
277Disallow: /*&tab=*
278Disallow: /*?q=*
279Disallow: /*&q=*