NeuralCrawl

Cricbuzz / robots.txt snapshot

← back to cricbuzz.com · fetched 2026-06-26T13:23:15Z (3h ago) · HTTP 200 · 9658 bytes · sha256 fbd98b09466f852e · raw

final URL: https://www.cricbuzz.com/robots.txt

1# Cricbuzz - The Interactive Cricket Portal
2# Nothing very exciting here for you I'm afraid.
3# Despictable and evil robots to keep out :)
4User-agent: *
5Allow: /ads.txt
6Disallow: /
7User-agent: Googlebot
8Allow: /ads.txt
9Allow: /app-ads.txt
10Disallow: /api/
11Disallow: /register/
12Disallow: /match/
13Disallow: /match2/
14Disallow: /match3/
15Disallow: /match4/
16Disallow: /cgi-bin/
17Disallow: /protected/
18Disallow: /livescore/
19Disallow: /component/
20Disallow: /livecricketscore/
21Disallow: /syndication/
22Disallow: /includes/
23Disallow: /mobile/subscribe
24Disallow: /*.json
25Disallow: /*.txt
26Disallow: /*.html
27Disallow: /*.php
28Disallow: /archive/archive/
29Disallow: /search?*
30Disallow: /search/?*
31User-agent: Googlebot-Image
32Allow: /ads.txt
33Disallow: /api/
34Disallow: /register/
35Disallow: /match/
36Disallow: /match2/
37Disallow: /match3/
38Disallow: /match4/
39Disallow: /cgi-bin/
40Disallow: /protected/
41Disallow: /livescore/
42Disallow: /component/
43Disallow: /livecricketscore/
44Disallow: /syndication/
45Disallow: /includes/
46Disallow: /mobile/subscribe
47Disallow: /*.json
48Disallow: /*.txt
49Disallow: /*.html
50Disallow: /*.php
51Disallow: /archive/archive/
52Disallow: /search?*
53Disallow: /search/?*
54User-agent: Google-Mobile
55Allow: /ads.txt
56Disallow: /api/
57Disallow: /register/
58Disallow: /match/
59Disallow: /match2/
60Disallow: /match3/
61Disallow: /match4/
62Disallow: /cgi-bin/
63Disallow: /protected/
64Disallow: /livescore/
65Disallow: /component/
66Disallow: /livecricketscore/
67Disallow: /syndication/
68Disallow: /includes/
69Disallow: /mobile/subscribe
70Disallow: /*.json
71Disallow: /*.txt
72Disallow: /*.html
73Disallow: /*.php
74Disallow: /archive/archive/
75Disallow: /search?*
76Disallow: /search/?*
77User-agent: Mediapartners-Google
78Allow: /ads.txt
79Allow: /app-ads.txt
80Disallow: /api/
81Disallow: /register/
82Disallow: /match/
83Disallow: /match2/
84Disallow: /match3/
85Disallow: /match4/
86Disallow: /cgi-bin/
87Disallow: /protected/
88Disallow: /livescore/
89Disallow: /component/
90Disallow: /livecricketscore/
91Disallow: /syndication/
92Disallow: /includes/
93Disallow: /mobile/subscribe
94Disallow: /*.json
95Disallow: /*.txt
96Disallow: /*.html
97Disallow: /*.php
98Disallow: /archive/archive/
99Disallow: /search?*
100Disallow: /search/?*
101User-agent: Yahoo
102Allow: /ads.txt
103Disallow: /api/
104Disallow: /register/
105Disallow: /match/
106Disallow: /match2/
107Disallow: /match3/
108Disallow: /match4/
109Disallow: /cgi-bin/
110Disallow: /protected/
111Disallow: /livescore/
112Disallow: /component/
113Disallow: /livecricketscore/
114Disallow: /syndication/
115Disallow: /includes/
116Disallow: /mobile/subscribe
117Disallow: /*.json
118Disallow: /*.txt
119Disallow: /*.html
120Disallow: /*.php
121Disallow: /archive/archive/
122Disallow: /search?*
123Disallow: /search/?*
124User-agent: Slurp
125Allow: /ads.txt
126Disallow: /api/
127Disallow: /register/
128Disallow: /match/
129Disallow: /match2/
130Disallow: /match3/
131Disallow: /match4/
132Disallow: /cgi-bin/
133Disallow: /protected/
134Disallow: /livescore/
135Disallow: /component/
136Disallow: /livecricketscore/
137Disallow: /syndication/
138Disallow: /includes/
139Disallow: /mobile/subscribe
140Disallow: /*.json
141Disallow: /*.txt
142Disallow: /*.html
143Disallow: /*.php
144Disallow: /archive/archive/
145Disallow: /search?*
146Disallow: /search/?*
147User-agent: Bingbot
148Allow: /ads.txt
149Disallow: /api/
150Disallow: /register/
151Disallow: /match/
152Disallow: /match2/
153Disallow: /match3/
154Disallow: /match4/
155Disallow: /cgi-bin/
156Disallow: /protected/
157Disallow: /livescore/
158Disallow: /component/
159Disallow: /livecricketscore/
160Disallow: /syndication/
161Disallow: /includes/
162Disallow: /mobile/subscribe
163Disallow: /*.json
164Disallow: /*.txt
165Disallow: /*.html
166Disallow: /*.php
167Disallow: /archive/archive/
168Disallow: /search?*
169Disallow: /search/?*
170User-agent: MSNBot
171Allow: /ads.txt
172Disallow: /api/
173Disallow: /register/
174Disallow: /match/
175Disallow: /match2/
176Disallow: /match3/
177Disallow: /match4/
178Disallow: /cgi-bin/
179Disallow: /protected/
180Disallow: /livescore/
181Disallow: /component/
182Disallow: /livecricketscore/
183Disallow: /syndication/
184Disallow: /includes/
185Disallow: /mobile/subscribe
186Disallow: /*.json
187Disallow: /*.txt
188Disallow: /*.html
189Disallow: /*.php
190Disallow: /archive/archive/
191Disallow: /search?*
192Disallow: /search/?*
193User-agent: MSNBot-Media
194Allow: /ads.txt
195Disallow: /api/
196Disallow: /register/
197Disallow: /match/
198Disallow: /match2/
199Disallow: /match3/
200Disallow: /match4/
201Disallow: /cgi-bin/
202Disallow: /protected/
203Disallow: /livescore/
204Disallow: /component/
205Disallow: /livecricketscore/
206Disallow: /syndication/
207Disallow: /includes/
208Disallow: /mobile/subscribe
209Disallow: /*.json
210Disallow: /*.txt
211Disallow: /*.html
212Disallow: /*.php
213Disallow: /archive/archive/
214Disallow: /search?*
215Disallow: /search/?*
216User-agent: AdIdxBot
217Allow: /ads.txt
218Disallow: /api/
219Disallow: /register/
220Disallow: /match/
221Disallow: /match2/
222Disallow: /match3/
223Disallow: /match4/
224Disallow: /cgi-bin/
225Disallow: /protected/
226Disallow: /livescore/
227Disallow: /component/
228Disallow: /livecricketscore/
229Disallow: /syndication/
230Disallow: /includes/
231Disallow: /mobile/subscribe
232Disallow: /*.json
233Disallow: /*.txt
234Disallow: /*.html
235Disallow: /*.php
236Disallow: /archive/archive/
237Disallow: /search?*
238Disallow: /search/?*
239User-agent: BingPreview
240Allow: /ads.txt
241Disallow: /api/
242Disallow: /register/
243Disallow: /match/
244Disallow: /match2/
245Disallow: /match3/
246Disallow: /match4/
247Disallow: /cgi-bin/
248Disallow: /protected/
249Disallow: /livescore/
250Disallow: /component/
251Disallow: /livecricketscore/
252Disallow: /syndication/
253Disallow: /includes/
254Disallow: /mobile/subscribe
255Disallow: /*.json
256Disallow: /*.txt
257Disallow: /*.html
258Disallow: /*.php
259Disallow: /archive/archive/
260Disallow: /search?*
261Disallow: /search/?*
262User-agent: Yandex
263Allow: /ads.txt
264Disallow: /api/
265Disallow: /register/
266Disallow: /match/
267Disallow: /match2/
268Disallow: /match3/
269Disallow: /match4/
270Disallow: /cgi-bin/
271Disallow: /protected/
272Disallow: /livescore/
273Disallow: /component/
274Disallow: /livecricketscore/
275Disallow: /syndication/
276Disallow: /includes/
277Disallow: /mobile/subscribe
278Disallow: /*.json
279Disallow: /*.txt
280Disallow: /*.html
281Disallow: /*.php
282Disallow: /archive/archive/
283Disallow: /search?*
284Disallow: /search/?*
285User-agent: Baidu
286Allow: /ads.txt
287Disallow: /api/
288Disallow: /register/
289Disallow: /match/
290Disallow: /match2/
291Disallow: /match3/
292Disallow: /match4/
293Disallow: /cgi-bin/
294Disallow: /protected/
295Disallow: /livescore/
296Disallow: /component/
297Disallow: /livecricketscore/
298Disallow: /syndication/
299Disallow: /includes/
300Disallow: /mobile/subscribe
301Disallow: /*.json
302Disallow: /*.txt
303Disallow: /*.html
304Disallow: /*.php
305Disallow: /archive/archive/
306Disallow: /search?*
307Disallow: /search/?*
308User-agent: Twitterbot
309Allow: /ads.txt
310Disallow: /api/
311Disallow: /register/
312Disallow: /match/
313Disallow: /match2/
314Disallow: /match3/
315Disallow: /match4/
316Disallow: /cgi-bin/
317Disallow: /protected/
318Disallow: /livescore/
319Disallow: /component/
320Disallow: /livecricketscore/
321Disallow: /syndication/
322Disallow: /includes/
323Disallow: /mobile/subscribe
324Disallow: /*.json
325Disallow: /*.txt
326Disallow: /*.html
327Disallow: /*.php
328Disallow: /archive/archive/
329Disallow: /search?*
330Disallow: /search/?*
331User-agent: Yeti
332Allow: /ads.txt
333Disallow: /api/
334Disallow: /register/
335Disallow: /match/
336Disallow: /match2/
337Disallow: /match3/
338Disallow: /match4/
339Disallow: /cgi-bin/
340Disallow: /protected/
341Disallow: /livescore/
342Disallow: /component/
343Disallow: /livecricketscore/
344Disallow: /syndication/
345Disallow: /includes/
346Disallow: /mobile/subscribe
347Disallow: /*.json
348Disallow: /*.txt
349Disallow: /*.html
350Disallow: /*.php
351Disallow: /archive/archive/
352Disallow: /search?*
353Disallow: /search/?*
354User-agent: Excite
355Allow: /ads.txt
356Disallow: /api/
357Disallow: /register/
358Disallow: /match/
359Disallow: /match2/
360Disallow: /match3/
361Disallow: /match4/
362Disallow: /cgi-bin/
363Disallow: /protected/
364Disallow: /livescore/
365Disallow: /component/
366Disallow: /livecricketscore/
367Disallow: /syndication/
368Disallow: /includes/
369Disallow: /mobile/subscribe
370Disallow: /*.json
371Disallow: /*.txt
372Disallow: /*.html
373Disallow: /*.php
374Disallow: /archive/archive/
375Disallow: /search?*
376Disallow: /search/?*
377User-agent: Duckduckgo
378Allow: /ads.txt
379Disallow: /api/
380Disallow: /register/
381Disallow: /match/
382Disallow: /match2/
383Disallow: /match3/
384Disallow: /match4/
385Disallow: /cgi-bin/
386Disallow: /protected/
387Disallow: /livescore/
388Disallow: /component/
389Disallow: /livecricketscore/
390Disallow: /syndication/
391Disallow: /includes/
392Disallow: /mobile/subscribe
393Disallow: /*.json
394Disallow: /*.txt
395Disallow: /*.html
396Disallow: /*.php
397Disallow: /archive/archive/
398Disallow: /search?*
399Disallow: /search/?*
400User-agent: Lycos
401Allow: /ads.txt
402Disallow: /api/
403Disallow: /register/
404Disallow: /match/
405Disallow: /match2/
406Disallow: /match3/
407Disallow: /match4/
408Disallow: /cgi-bin/
409Disallow: /protected/
410Disallow: /livescore/
411Disallow: /component/
412Disallow: /livecricketscore/
413Disallow: /syndication/
414Disallow: /includes/
415Disallow: /mobile/subscribe
416Disallow: /*.json
417Disallow: /*.txt
418Disallow: /*.html
419Disallow: /*.php
420Disallow: /archive/archive/
421Disallow: /search?*
422Disallow: /search/?*
423User-agent: Applebot
424Allow: /ads.txt
425Disallow: /api/
426Disallow: /register/
427Disallow: /match/
428Disallow: /match2/
429Disallow: /match3/
430Disallow: /match4/
431Disallow: /cgi-bin/
432Disallow: /protected/
433Disallow: /livescore/
434Disallow: /component/
435Disallow: /livecricketscore/
436Disallow: /syndication/
437Disallow: /includes/
438Disallow: /mobile/subscribe
439Disallow: /*.json
440Disallow: /*.txt
441Disallow: /*.html
442Disallow: /*.php
443Disallow: /archive/archive/
444Disallow: /search?*
445Disallow: /search/?*
446User-agent: AmazonAdBot
447Allow: /ads.txt
448Allow: /app-ads.txt
449Allow: /
450Disallow: /api/
451Disallow: /search?*
452Disallow: /search/?*
453Disallow: /*.json
454Sitemap: https://www.cricbuzz.com/sitemap/cricket-videos.xml
455Sitemap: https://www.cricbuzz.com/sitemap/live-matches.xml
456Sitemap: https://www.cricbuzz.com/sitemap/cricket-schedule-series.xml
457Sitemap: https://www.cricbuzz.com/sitemap.xml