NeuralCrawl

GitHub / robots.txt snapshot

← back to github.com · fetched 2026-09-12T01:21:55Z (7d ago) · HTTP 200 · 5317 bytes · sha256 2bae6090fc5d11ab · raw

final URL: https://github.com/robots.txt

1# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots
2# We also provide an extensive API: https://docs.github.com
3User-agent: GPTBot
4User-agent: OAI-SearchBot
5User-agent: ClaudeBot
6User-agent: anthropic-ai
7User-agent: PerplexityBot
8Crawl-delay: 1
9Allow: /$
10Allow: /about
11Allow: /customer-stories
12Allow: /enterprise
13Allow: /features
14Allow: /marketplace
15Allow: /mcp
16Allow: /newsroom
17Allow: /pricing
18Allow: /resources
19Allow: /security
20Allow: /solutions
21Disallow: /*/*/pulse
22Disallow: /*/*/projects
23Disallow: /*/*/forks
24Disallow: /*/*/issues/new
25Disallow: /*/*/milestones/new
26Disallow: /*/*/issues/search
27Disallow: /*/*/commits/
28Disallow: /*/*/branches
29Disallow: /*/*/contributors
30Disallow: /*/*/tags
31Disallow: /*/*/stargazers
32Disallow: /*/*/watchers
33Disallow: /*/*/network
34Disallow: /*/*/graphs
35Disallow: /*/*/compare
36Disallow: /*/tree/
37Disallow: /gist/
38Disallow: /*/download
39Disallow: /*/revisions
40Disallow: /*/commits/*?author
41Disallow: /*/commits/*?path
42Disallow: /*/comments
43Disallow: /*/archive/
44Disallow: /*/blame/
45Disallow: /*/raw/
46Disallow: /*/cache/
47Disallow: /.git/
48Disallow: */.git/
49Disallow: /*.git$
50Disallow: /search/advanced
51Disallow: /search$
52Disallow: /*q=
53Disallow: /*.atom$
54Disallow: /ekansa/Open-Context-Data
55Disallow: /ekansa/opencontext-*
56Disallow: */tarball/
57Disallow: */zipball/
58Disallow: /*source=*
59Disallow: /*ref_cta=*
60Disallow: /*plan=*
61Disallow: /*return_to=*
62Disallow: /*ref_loc=*
63Disallow: /*setup_organization=*
64Disallow: /*source_repo=*
65Disallow: /*ref_page=*
66Disallow: /*referrer=*
67Disallow: /*report=*
68Disallow: /*author=*
69Disallow: /*since=*
70Disallow: /*until=*
71Disallow: /*commits?author=*
72Disallow: /*report-abuse?report=*
73Disallow: /*tab=*
74Disallow: /account-login
75Disallow: /Explodingstuff/
76Disallow: /copilot/
77Disallow: /copilot/c/
78
79User-agent: Bytespider
80Disallow: /
81
82User-agent: AdsBot-Google
83User-agent: AdsBot-Google-Mobile
84Disallow: /*/*/pulse
85Disallow: /*/*/projects
86Disallow: /*/*/forks
87Disallow: /*/*/issues/new
88Disallow: /*/*/milestones/new
89Disallow: /*/*/issues/search
90Disallow: /*/*/commits/
91Disallow: /*/*/branches
92Disallow: /*/*/contributors
93Disallow: /*/*/tags
94Disallow: /*/*/stargazers
95Disallow: /*/*/watchers
96Disallow: /*/*/network
97Disallow: /*/*/graphs
98Disallow: /*/*/compare
99Disallow: /*/tree/
100Disallow: /gist/
101Disallow: /*/download
102Disallow: /*/revisions
103Disallow: /*/commits/*?author
104Disallow: /*/commits/*?path
105Disallow: /*/comments
106Disallow: /*/archive/
107Disallow: /*/blame/
108Disallow: /*/raw/
109Disallow: /*/cache/
110Disallow: /.git/
111Disallow: */.git/
112Disallow: /*.git$
113Disallow: /search/advanced
114Disallow: /search$
115Disallow: /*q=
116Disallow: /*.atom$
117Disallow: /ekansa/Open-Context-Data
118Disallow: /ekansa/opencontext-*
119Disallow: */tarball/
120Disallow: */zipball/
121Disallow: /*source=*
122Disallow: /*ref_cta=*
123Disallow: /*plan=*
124Disallow: /*return_to=*
125Disallow: /*ref_loc=*
126Disallow: /*setup_organization=*
127Disallow: /*source_repo=*
128Disallow: /*ref_page=*
129Disallow: /*referrer=*
130Disallow: /*report=*
131Disallow: /*author=*
132Disallow: /*since=*
133Disallow: /*until=*
134Disallow: /*commits?author=*
135Disallow: /*report-abuse?report=*
136Disallow: /*tab=*
137Disallow: /account-login
138Disallow: /Explodingstuff/
139Disallow: /copilot/
140Disallow: /copilot/c/
141
142User-agent: bingbot
143Disallow: /ekansa/Open-Context-Data
144Disallow: /ekansa/opencontext-*
145Disallow: /account-login
146Disallow: */tarball/
147Disallow: */zipball/
148Disallow: /Explodingstuff/
149Disallow: /copilot/
150Disallow: /copilot/c/
151
152User-agent: adidxbot
153Disallow: /ekansa/Open-Context-Data
154Disallow: /ekansa/opencontext-*
155Disallow: /account-login
156Disallow: */tarball/
157Disallow: */zipball/
158Disallow: /Explodingstuff/
159Disallow: /copilot/
160Disallow: /copilot/c/
161
162User-agent: BingPreview
163Disallow: /ekansa/Open-Context-Data
164Disallow: /ekansa/opencontext-*
165Disallow: /account-login
166Disallow: */tarball/
167Disallow: */zipball/
168Disallow: /Explodingstuff/
169Disallow: /copilot/
170Disallow: /copilot/c/
171
172User-agent: baidu
173crawl-delay: 1
174
175
176User-agent: *
177Allow: /*?tab=achievements&achievement=*
178Disallow: /*/*/pulse
179Disallow: /*/*/projects
180Disallow: /*/*/forks
181Disallow: /*/*/issues/new
182Disallow: /*/*/milestones/new
183Disallow: /*/*/issues/search
184Disallow: /*/*/commits/
185Disallow: /*/*/branches
186Disallow: /*/*/contributors
187Disallow: /*/*/tags
188Disallow: /*/*/stargazers
189Disallow: /*/*/watchers
190Disallow: /*/*/network
191Disallow: /*/*/graphs
192Disallow: /*/*/compare
193Disallow: /*/tree/
194Disallow: /gist/
195Disallow: /*/download
196Disallow: /*/revisions
197Disallow: /*/commits/*?author
198Disallow: /*/commits/*?path
199Disallow: /*/comments
200Disallow: /*/archive/
201Disallow: /*/blame/
202Disallow: /*/raw/
203Disallow: /*/cache/
204Disallow: /.git/
205Disallow: */.git/
206Disallow: /*.git$
207Disallow: /search/advanced
208Disallow: /search$
209Disallow: /*q=
210Disallow: /*.atom$
211Disallow: /ekansa/Open-Context-Data
212Disallow: /ekansa/opencontext-*
213Disallow: */tarball/
214Disallow: */zipball/
215Disallow: /*source=*
216Disallow: /*ref_cta=*
217Disallow: /*plan=*
218Disallow: /*return_to=*
219Disallow: /*ref_loc=*
220Disallow: /*setup_organization=*
221Disallow: /*source_repo=*
222Disallow: /*ref_page=*
223Disallow: /*referrer=*
224Disallow: /*report=*
225Disallow: /*author=*
226Disallow: /*since=*
227Disallow: /*until=*
228Disallow: /*commits?author=*
229Disallow: /*report-abuse?report=*
230Disallow: /*tab=*
231Disallow: /account-login
232Disallow: /Explodingstuff/
233Disallow: /copilot/
234Disallow: /copilot/c/