NeuralCrawl

The Times of India / robots.txt snapshot

← back to timesofindia.indiatimes.com · fetched 2026-07-20T13:08:59Z (16d ago) · HTTP 200 · 11012 bytes · sha256 41efdaf52803de82 · raw

final URL: https://timesofindia.indiatimes.com/robots.txt

1User-agent: AmazonAdBot
2Allow: /
3Allow: /*
4
5User-agent: *
6Allow: /
7Disallow: /PDATOI/
8Disallow: /topic/quote/
9Disallow: /*cms.dll*
10Disallow: /*rssarticleshow*
11Disallow: /topic/photo/
12Disallow: /default.cms
13Disallow: /default1.cms
14Disallow: /artshowcmt/
15Disallow: */opinions/
16Disallow: /breakingnews/toihome.html
17Disallow: /topic/article/
18Disallow: /logtopickeywords*
19Disallow: /new_cmtofart*
20Disallow: /pmcomment/
21Disallow: /feeds/
22Disallow: */configspace/*
23Disallow: /configspace/maintenance.html
24Disallow: /topiclist/
25Disallow: */pmredirectshow/*
26Disallow: */stoi/*
27Disallow: */moviearticlelistalphabet1/*
28Disallow: */cinema-entertainment/*
29Disallow: */life-style-landing/*
30Disallow: *navlvl=
31Disallow: /lspoll/
32Disallow: *query=
33Disallow: /beta_hotlydbt/
34Disallow: *defalpha=
35Disallow: *,page
36Disallow: /new_cmtofart2_nit_v1/
37Disallow: *prtpage
38Disallow: */examshow/*
39Disallow: */speednewsshow/*
40Disallow: */speednewsbytopic/*
41Disallow: */itslideshow/*
42Disallow: */vsarticleshow/*
43Disallow: */articleshow_amitr/*
44Disallow: */pmtoinavigation/*
45Disallow: */mvopinions/*
46Disallow: */opinionslogged/*
47Disallow: */rodate.js
48Disallow: */load.js
49Disallow: */cookie.js
50Disallow: */jquery.js
51Disallow: */navigation.js
52Disallow: */pgtrack.js
53Disallow: */lazy.js
54Disallow: */breakingnews.js
55Disallow: */rodate.js
56Disallow: */mail/*
57Disallow: */profileshow/*
58Disallow: */tpoint_articleshow/*
59Disallow: */trivia/*
60Disallow: */articleshowajax/*
61Disallow: /articlecomment/
62Disallow: /artshowcmt/
63Disallow: /7176/TOI_Travel/
64Disallow: /search
65Disallow: /fbpostaction.cms?article=
66Disallow: /7176/TOI_Travel_Test/TOI_Travel_TOP_TST
67Disallow: /postmail.cms
68Disallow: /mostviewed
69Disallow: /related
70Disallow: /comments
71Disallow: *Jai-Mrug*
72Disallow: */affiliate_amazon.cms*
73Disallow: *-mostviwed*
74Disallow: */techgadgets/*
75Disallow: */covid-19-delhi/*
76Disallow: */covid-19-bangalore/*
77Disallow: */covid-19-hyderabad/*
78Disallow: */covid-19-kolkata/*
79Disallow: */covid-19-mumbai/*
80Disallow: */covid-19-across-world/*
81Disallow: */crshow/*
82Disallow: */briefs/articleid-*
83Disallow: /intarnel/*
84Disallow: */dynalist/*
85Disallow: */amp_msp_amazon_gadget.cms*
86Disallow: */affiliates_content_iframe.cms*
87Disallow: */affiliates_content.cms*
88Disallow: */affiliates_content_iframe*
89Disallow: */affiliates_library*
90Disallow: */affiliates_content*
91Disallow: */ifrmwdt/*
92Disallow: *articleshowbriefs*
93Disallow: *quickstirshow*
94Disallow: *mobile=no*
95Disallow: *etarticleshow*
96Disallow: *?mobile=no*
97Disallow: *homeajax_nav.cms?*
98Disallow: *articleshowbriefs?*
99Disallow: *affiliates_content_iframe_v2*
100Disallow: *?frmapp=yes*
101Disallow: *embedvideo.cms*
102Disallow: *videodash.cms*
103Disallow: *etopneinappamp.cms*
104Disallow: *as_comments*
105Disallow: *RBP_HOME_URL*
106Disallow: *?comments*
107Disallow: *?showcomment*
108Disallow: *?cmfid*
109Disallow: *?pg=*
110Disallow: *?ps=*
111Disallow: *articleshow_new*
112Disallow: *articleshowprint*
113Disallow: /microsite_v2
114Disallow: */undefined$
115Disallow: */articleshowhsbc/
116Disallow: */storylist/
117Disallow: *asrelated.cms*
118Disallow: *amp_next_page_ga.cms*
119Disallow: */travel/search?*
120Disallow: */articlelist/*
121Disallow: /toi_webvital_events_min_js.cms*
122Disallow: /popup_manager_priority.cms*
123Disallow: /locateservice_js/minify-1,version-14.cms*
124Disallow: /slick_min_js_v18/*
125Disallow: /enttinyscrolljs/*
126Disallow: /beta_gohintjs_v7.cms*
127Disallow: /locateservice_js/*
128Disallow: /trackjs_newnav.cms*
129Disallow: /grxpushnotification_js/version-23,minify-1.cms*
130Disallow: /aqiscript/*
131Disallow: /navincludejs.cms*
132Disallow: /articlelist.cms.js*
133Disallow: /navincludejs.cms*
134Disallow: /pn/version-39,minify-1.cms*
135Disallow: /etimes_login/*
136Disallow: /etnewsletter_js/*
137Disallow: /glblnavjs_v19.cms*
138Disallow: /gaview_script/*
139Disallow: /etpn/version-26,minify-1.cms*
140Disallow: /new_bookmarkjs_v2.cms*
141Disallow: /new_slidjs_v1.cms*
142Disallow: /grxevent_config/*
143Disallow: /et_gdpr_callback_js/*
144Disallow: /et_onetap_signin_js/*
145Disallow: /getseeallhref.cms*
146Disallow: /getdataofseason.cms*
147Disallow: /affiliate_wrapper_amp.cms*
148Disallow: *feedtype-sjson*
149Disallow: *feedtype-json*
150Disallow: *feedtype=sjson*
151Disallow: *feedtype=json*
152Disallow: /ncCentreFeed*
153Disallow: /allauthrlist_v2.cms*
154Disallow: /newsletter_rhs_js.cms?minify=1&version=30*
155Disallow: /ettoiplus_verifymobile_screen_js/version-90,minify-1,isArchive-1.cms*
156Disallow: /requirejs/version-9,minify-1.cms*
157Disallow: /jquery_1_11_0_min/version-13,minify-1.cms*
158Disallow: /et_gdpr_js/minify-1,v-14.cms*
159Disallow: /lgads_v2/minify-1,channel-nbt.cms*
160Disallow: /etads_v2/minify-1.cms*
161Disallow: *js/v2/web-sdk-main.js*
162Disallow: *v2/track*
163Disallow: /toiads_prebid/minify-1.cms*
164Disallow: /toiads_react_lazy/minify-1.cms*
165Disallow: /google_ads/ads/ve-1.cms*
166Disallow: /requirejs/version-9,minify-1.cms*
167Disallow: /toiads_new/minify-1.cms*
168Disallow: /newsletter_rhs_js.cms*
169Disallow: /toiads_prebid_checker/adset/ve-1.cms*
170Disallow: /ext_*
171Disallow: /*_rsc=
172Disallow: /meta.json
173Disallow: */lottery/*
174
175User-agent: Googlebot-News
176Allow: /
177Disallow: /PDATOI/
178Disallow: /topic/quote/
179Disallow: /*cms.dll*
180Disallow: /*rssarticleshow*
181Disallow: /topic/photo/
182Disallow: /default.cms
183Disallow: /default1.cms
184Disallow: /artshowcmt/
185Disallow: */opinions/
186Disallow: /breakingnews/toihome.html
187Disallow: /topic/article/
188Disallow: /logtopickeywords*
189Disallow: /new_cmtofart*
190Disallow: /pmcomment/
191Disallow: /feeds/
192Disallow: */configspace/*
193Disallow: /configspace/maintenance.html
194Disallow: /topiclist/
195Disallow: */pmredirectshow/*
196Disallow: */stoi/*
197Disallow: */moviearticlelistalphabet1/*
198Disallow: */cinema-entertainment/*
199Disallow: */life-style-landing/*
200Disallow: *navlvl=
201Disallow: /lspoll/
202Disallow: *query=
203Disallow: /beta_hotlydbt/
204Disallow: *defalpha=
205Disallow: *,page
206Disallow: /new_cmtofart2_nit_v1/
207Disallow: *prtpage
208Disallow: */examshow/*
209Disallow: */speednewsshow/*
210Disallow: */speednewsbytopic/*
211Disallow: */itslideshow/*
212Disallow: */vsarticleshow/*
213Disallow: */articleshow_amitr/*
214Disallow: */pmtoinavigation/*
215Disallow: */mvopinions/*
216Disallow: */opinionslogged/*
217Disallow: */rodate.js
218Disallow: */load.js
219Disallow: */cookie.js
220Disallow: */jquery.js
221Disallow: */navigation.js
222Disallow: */pgtrack.js
223Disallow: */lazy.js
224Disallow: */breakingnews.js
225Disallow: */rodate.js
226Disallow: */mail/*
227Disallow: */profileshow/*
228Disallow: */tpoint_articleshow/*
229Disallow: */trivia/*
230Disallow: */articleshowajax/*
231Disallow: /articlecomment/
232Disallow: /artshowcmt/
233Disallow: /7176/TOI_Travel/
234Disallow: /search
235Disallow: /fbpostaction.cms?article=
236Disallow: /7176/TOI_Travel_Test/TOI_Travel_TOP_TST
237Disallow: /postmail.cms
238Disallow: /mostviewed
239Disallow: /related
240Disallow: /comments
241Disallow: *Jai-Mrug*
242Disallow: */affiliate_amazon.cms*
243Disallow: *-mostviwed*
244Disallow: */techgadgets/*
245Disallow: */covid-19-delhi/*
246Disallow: */covid-19-bangalore/*
247Disallow: */covid-19-hyderabad/*
248Disallow: */covid-19-kolkata/*
249Disallow: */covid-19-mumbai/*
250Disallow: */covid-19-across-world/*
251Disallow: */crshow/*
252Disallow: */briefs/articleid-*
253Disallow: /intarnel/*
254Disallow: */dynalist/*
255Disallow: */amp_msp_amazon_gadget.cms*
256Disallow: */affiliates_content_iframe.cms*
257Disallow: */affiliates_content.cms*
258Disallow: */affiliates_content_iframe*
259Disallow: */affiliates_library*
260Disallow: */affiliates_content*
261Disallow: */ifrmwdt/*
262Disallow: *articleshowbriefs*
263Disallow: *quickstirshow*
264Disallow: *mobile=no*
265Disallow: *etarticleshow*
266Disallow: *?mobile=no*
267Disallow: *homeajax_nav.cms?*
268Disallow: *articleshowbriefs?*
269Disallow: *affiliates_content_iframe_v2*
270Disallow: *?frmapp=yes*
271Disallow: *embedvideo.cms*
272Disallow: *videodash.cms*
273Disallow: *etopneinappamp.cms*
274Disallow: *as_comments*
275Disallow: *RBP_HOME_URL*
276Disallow: *?comments*
277Disallow: *?showcomment*
278Disallow: *?cmfid*
279Disallow: *?pg=*
280Disallow: *?ps=*
281Disallow: *articleshow_new*
282Disallow: *articleshowprint*
283Disallow: /microsite_v2
284Disallow: */undefined$
285Disallow: */articleshowhsbc/
286Disallow: */storylist/
287Disallow: *asrelated.cms*
288Disallow: *amp_next_page_ga.cms*
289Disallow: */travel/search?*
290Disallow: */articlelist/*
291Disallow: /toi_webvital_events_min_js.cms*
292Disallow: /popup_manager_priority.cms*
293Disallow: /locateservice_js/minify-1,version-14.cms*
294Disallow: /slick_min_js_v18/*
295Disallow: /enttinyscrolljs/*
296Disallow: /beta_gohintjs_v7.cms*
297Disallow: /locateservice_js/*
298Disallow: /trackjs_newnav.cms*
299Disallow: /grxpushnotification_js/version-23,minify-1.cms*
300Disallow: /aqiscript/*
301Disallow: /navincludejs.cms*
302Disallow: /articlelist.cms.js*
303Disallow: /navincludejs.cms*
304Disallow: /pn/version-39,minify-1.cms*
305Disallow: /etimes_login/*
306Disallow: /etnewsletter_js/*
307Disallow: /glblnavjs_v19.cms*
308Disallow: /gaview_script/*
309Disallow: /etpn/version-26,minify-1.cms*
310Disallow: /new_bookmarkjs_v2.cms*
311Disallow: /new_slidjs_v1.cms*
312Disallow: /grxevent_config/*
313Disallow: /et_gdpr_callback_js/*
314Disallow: /et_onetap_signin_js/*
315Disallow: /getseeallhref.cms*
316Disallow: /getdataofseason.cms*
317Disallow: /affiliate_wrapper_amp.cms*
318Disallow: *feedtype-sjson*
319Disallow: *feedtype-json*
320Disallow: *feedtype=sjson*
321Disallow: *feedtype=json*
322Disallow: /ncCentreFeed*
323Disallow: /allauthrlist_v2.cms*
324Disallow: /newsletter_rhs_js.cms?minify=1&version=30*
325Disallow: /ettoiplus_verifymobile_screen_js/version-90,minify-1,isArchive-1.cms*
326Disallow: /requirejs/version-9,minify-1.cms*
327Disallow: /jquery_1_11_0_min/version-13,minify-1.cms*
328Disallow: /et_gdpr_js/minify-1,v-14.cms*
329Disallow: /lgads_v2/minify-1,channel-nbt.cms*
330Disallow: /etads_v2/minify-1.cms*
331Disallow: *js/v2/web-sdk-main.js*
332Disallow: *v2/track*
333Disallow: /toiads_prebid/minify-1.cms*
334Disallow: /toiads_react_lazy/minify-1.cms*
335Disallow: /google_ads/ads/ve-1.cms*
336Disallow: /requirejs/version-9,minify-1.cms*
337Disallow: /toiads_new/minify-1.cms*
338Disallow: /newsletter_rhs_js.cms*
339Disallow: /toiads_prebid_checker/adset/ve-1.cms*
340Disallow: /ext_*
341Disallow: /*_rsc=
342Disallow: /meta.json
343Disallow: */lottery/*
344
345User-agent: AdsBot-Google
346Allow: /trivia/
347
348User-agent: Mediapartners-Google
349Allow: /trivia/
350
351User-agent: AmazonAdBot
352Allow: /trivia/
353
354User-agent: meta-externalagent
355Disallow: /
356
357Sitemap: https://timesofindia.indiatimes.com/sitemap/today
358Sitemap: https://timesofindia.indiatimes.com/sitemap/yesterday
359Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/news/sitemap-index.xml
360Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/category/city/sitemap-index.xml
361Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/category/entertainment/sitemap-index.xml
362Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/category/india/sitemap-index.xml
363Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/category/sports/sitemap-index.xml
364Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/category/business/sitemap-index.xml
365Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/category/world/sitemap-index.xml
366Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/slideshow/sitemap-index.xml
367Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/category/life-style/sitemap-index.xml