NeuralCrawl

The Times of India / robots.txt snapshot

← back to timesofindia.indiatimes.com · fetched 2026-06-20T01:10:30Z (18h ago) · HTTP 200 · 10968 bytes · sha256 ca22def5661d0d0a · raw

final URL: https://timesofindia.indiatimes.com/robots.txt

1User-agent: AmazonAdBot
2Allow: /
3Allow: /*
4
5User-agent: *
6Allow: /
7Disallow: /PDATOI/
8Disallow: /topic/quote/
9Disallow: /*cms.dll*
10Disallow: /*rssarticleshow*
11Disallow: /topic/photo/
12Disallow: /default.cms
13Disallow: /default1.cms
14Disallow: /artshowcmt/
15Disallow: */opinions/
16Disallow: /breakingnews/toihome.html
17Disallow: /topic/article/
18Disallow: /logtopickeywords*
19Disallow: /new_cmtofart*
20Disallow: /pmcomment/
21Disallow: /feeds/
22Disallow: */configspace/*
23Disallow: /configspace/maintenance.html
24Disallow: /topiclist/
25Disallow: */pmredirectshow/*
26Disallow: */stoi/*
27Disallow: */moviearticlelistalphabet1/*
28Disallow: */cinema-entertainment/*
29Disallow: */life-style-landing/*
30Disallow: *navlvl=
31Disallow: /lspoll/
32Disallow: *query=
33Disallow: /beta_hotlydbt/
34Disallow: *defalpha=
35Disallow: *,page
36Disallow: /new_cmtofart2_nit_v1/
37Disallow: *prtpage
38Disallow: */examshow/*
39Disallow: */speednewsshow/*
40Disallow: */speednewsbytopic/*
41Disallow: */itslideshow/*
42Disallow: */vsarticleshow/*
43Disallow: */articleshow_amitr/*
44Disallow: */pmtoinavigation/*
45Disallow: */mvopinions/*
46Disallow: */opinionslogged/*
47Disallow: */rodate.js
48Disallow: */load.js
49Disallow: */cookie.js
50Disallow: */jquery.js
51Disallow: */navigation.js
52Disallow: */pgtrack.js
53Disallow: */lazy.js
54Disallow: */breakingnews.js
55Disallow: */rodate.js
56Disallow: */mail/*
57Disallow: */profileshow/*
58Disallow: */tpoint_articleshow/*
59Disallow: */trivia/*
60Disallow: */articleshowajax/*
61Disallow: /articlecomment/
62Disallow: /artshowcmt/
63Disallow: /7176/TOI_Travel/
64Disallow: /search
65Disallow: /fbpostaction.cms?article=
66Disallow: /7176/TOI_Travel_Test/TOI_Travel_TOP_TST
67Disallow: /postmail.cms
68Disallow: /mostviewed
69Disallow: /related
70Disallow: /comments
71Disallow: *Jai-Mrug*
72Disallow: */affiliate_amazon.cms*
73Disallow: *-mostviwed*
74Disallow: */techgadgets/*
75Disallow: */covid-19-delhi/*
76Disallow: */covid-19-bangalore/*
77Disallow: */covid-19-hyderabad/*
78Disallow: */covid-19-kolkata/*
79Disallow: */covid-19-mumbai/*
80Disallow: */covid-19-across-world/*
81Disallow: */crshow/*
82Disallow: */briefs/articleid-*
83Disallow: /intarnel/*
84Disallow: */dynalist/*
85Disallow: */amp_msp_amazon_gadget.cms*
86Disallow: */affiliates_content_iframe.cms*
87Disallow: */affiliates_content.cms*
88Disallow: */affiliates_content_iframe*
89Disallow: */affiliates_library*
90Disallow: */affiliates_content*
91Disallow: */ifrmwdt/*
92Disallow: *articleshowbriefs*
93Disallow: *quickstirshow*
94Disallow: *mobile=no*
95Disallow: *etarticleshow*
96Disallow: *?mobile=no*
97Disallow: *homeajax_nav.cms?*
98Disallow: *articleshowbriefs?*
99Disallow: *affiliates_content_iframe_v2*
100Disallow: *?frmapp=yes*
101Disallow: *embedvideo.cms*
102Disallow: *videodash.cms*
103Disallow: *etopneinappamp.cms*
104Disallow: *as_comments*
105Disallow: *RBP_HOME_URL*
106Disallow: *?comments*
107Disallow: *?showcomment*
108Disallow: *?cmfid*
109Disallow: *?pg=*
110Disallow: *?ps=*
111Disallow: *articleshow_new*
112Disallow: *articleshowprint*
113Disallow: /microsite_v2
114Disallow: */undefined$
115Disallow: */articleshowhsbc/
116Disallow: */storylist/
117Disallow: *asrelated.cms*
118Disallow: *amp_next_page_ga.cms*
119Disallow: */travel/search?*
120Disallow: */articlelist/*
121Disallow: /toi_webvital_events_min_js.cms*
122Disallow: /popup_manager_priority.cms*
123Disallow: /locateservice_js/minify-1,version-14.cms*
124Disallow: /slick_min_js_v18/*
125Disallow: /enttinyscrolljs/*
126Disallow: /beta_gohintjs_v7.cms*
127Disallow: /locateservice_js/*
128Disallow: /trackjs_newnav.cms*
129Disallow: /grxpushnotification_js/version-23,minify-1.cms*
130Disallow: /aqiscript/*
131Disallow: /navincludejs.cms*
132Disallow: /articlelist.cms.js*
133Disallow: /navincludejs.cms*
134Disallow: /pn/version-39,minify-1.cms*
135Disallow: /etimes_login/*
136Disallow: /etnewsletter_js/*
137Disallow: /glblnavjs_v19.cms*
138Disallow: /gaview_script/*
139Disallow: /etpn/version-26,minify-1.cms*
140Disallow: /new_bookmarkjs_v2.cms*
141Disallow: /new_slidjs_v1.cms*
142Disallow: /grxevent_config/*
143Disallow: /et_gdpr_callback_js/*
144Disallow: /et_onetap_signin_js/*
145Disallow: /getseeallhref.cms*
146Disallow: /getdataofseason.cms*
147Disallow: /affiliate_wrapper_amp.cms*
148Disallow: *feedtype-sjson*
149Disallow: *feedtype-json*
150Disallow: *feedtype=sjson*
151Disallow: *feedtype=json*
152Disallow: /ncCentreFeed*
153Disallow: /allauthrlist_v2.cms*
154Disallow: /newsletter_rhs_js.cms?minify=1&version=30*
155Disallow: /ettoiplus_verifymobile_screen_js/version-90,minify-1,isArchive-1.cms*
156Disallow: /requirejs/version-9,minify-1.cms*
157Disallow: /jquery_1_11_0_min/version-13,minify-1.cms*
158Disallow: /et_gdpr_js/minify-1,v-14.cms*
159Disallow: /lgads_v2/minify-1,channel-nbt.cms*
160Disallow: /etads_v2/minify-1.cms*
161Disallow: *js/v2/web-sdk-main.js*
162Disallow: *v2/track*
163Disallow: /toiads_prebid/minify-1.cms*
164Disallow: /toiads_react_lazy/minify-1.cms*
165Disallow: /google_ads/ads/ve-1.cms*
166Disallow: /requirejs/version-9,minify-1.cms*
167Disallow: /toiads_new/minify-1.cms*
168Disallow: /newsletter_rhs_js.cms*
169Disallow: /toiads_prebid_checker/adset/ve-1.cms*
170Disallow: /ext_*
171Disallow: /*_rsc=
172Disallow: /meta.json
173
174User-agent: Googlebot-News
175Allow: /
176Disallow: /PDATOI/
177Disallow: /topic/quote/
178Disallow: /*cms.dll*
179Disallow: /*rssarticleshow*
180Disallow: /topic/photo/
181Disallow: /default.cms
182Disallow: /default1.cms
183Disallow: /artshowcmt/
184Disallow: */opinions/
185Disallow: /breakingnews/toihome.html
186Disallow: /topic/article/
187Disallow: /logtopickeywords*
188Disallow: /new_cmtofart*
189Disallow: /pmcomment/
190Disallow: /feeds/
191Disallow: */configspace/*
192Disallow: /configspace/maintenance.html
193Disallow: /topiclist/
194Disallow: */pmredirectshow/*
195Disallow: */stoi/*
196Disallow: */moviearticlelistalphabet1/*
197Disallow: */cinema-entertainment/*
198Disallow: */life-style-landing/*
199Disallow: *navlvl=
200Disallow: /lspoll/
201Disallow: *query=
202Disallow: /beta_hotlydbt/
203Disallow: *defalpha=
204Disallow: *,page
205Disallow: /new_cmtofart2_nit_v1/
206Disallow: *prtpage
207Disallow: */examshow/*
208Disallow: */speednewsshow/*
209Disallow: */speednewsbytopic/*
210Disallow: */itslideshow/*
211Disallow: */vsarticleshow/*
212Disallow: */articleshow_amitr/*
213Disallow: */pmtoinavigation/*
214Disallow: */mvopinions/*
215Disallow: */opinionslogged/*
216Disallow: */rodate.js
217Disallow: */load.js
218Disallow: */cookie.js
219Disallow: */jquery.js
220Disallow: */navigation.js
221Disallow: */pgtrack.js
222Disallow: */lazy.js
223Disallow: */breakingnews.js
224Disallow: */rodate.js
225Disallow: */mail/*
226Disallow: */profileshow/*
227Disallow: */tpoint_articleshow/*
228Disallow: */trivia/*
229Disallow: */articleshowajax/*
230Disallow: /articlecomment/
231Disallow: /artshowcmt/
232Disallow: /7176/TOI_Travel/
233Disallow: /search
234Disallow: /fbpostaction.cms?article=
235Disallow: /7176/TOI_Travel_Test/TOI_Travel_TOP_TST
236Disallow: /postmail.cms
237Disallow: /mostviewed
238Disallow: /related
239Disallow: /comments
240Disallow: *Jai-Mrug*
241Disallow: */affiliate_amazon.cms*
242Disallow: *-mostviwed*
243Disallow: */techgadgets/*
244Disallow: */covid-19-delhi/*
245Disallow: */covid-19-bangalore/*
246Disallow: */covid-19-hyderabad/*
247Disallow: */covid-19-kolkata/*
248Disallow: */covid-19-mumbai/*
249Disallow: */covid-19-across-world/*
250Disallow: */crshow/*
251Disallow: */briefs/articleid-*
252Disallow: /intarnel/*
253Disallow: */dynalist/*
254Disallow: */amp_msp_amazon_gadget.cms*
255Disallow: */affiliates_content_iframe.cms*
256Disallow: */affiliates_content.cms*
257Disallow: */affiliates_content_iframe*
258Disallow: */affiliates_library*
259Disallow: */affiliates_content*
260Disallow: */ifrmwdt/*
261Disallow: *articleshowbriefs*
262Disallow: *quickstirshow*
263Disallow: *mobile=no*
264Disallow: *etarticleshow*
265Disallow: *?mobile=no*
266Disallow: *homeajax_nav.cms?*
267Disallow: *articleshowbriefs?*
268Disallow: *affiliates_content_iframe_v2*
269Disallow: *?frmapp=yes*
270Disallow: *embedvideo.cms*
271Disallow: *videodash.cms*
272Disallow: *etopneinappamp.cms*
273Disallow: *as_comments*
274Disallow: *RBP_HOME_URL*
275Disallow: *?comments*
276Disallow: *?showcomment*
277Disallow: *?cmfid*
278Disallow: *?pg=*
279Disallow: *?ps=*
280Disallow: *articleshow_new*
281Disallow: *articleshowprint*
282Disallow: /microsite_v2
283Disallow: */undefined$
284Disallow: */articleshowhsbc/
285Disallow: */storylist/
286Disallow: *asrelated.cms*
287Disallow: *amp_next_page_ga.cms*
288Disallow: */travel/search?*
289Disallow: */articlelist/*
290Disallow: /toi_webvital_events_min_js.cms*
291Disallow: /popup_manager_priority.cms*
292Disallow: /locateservice_js/minify-1,version-14.cms*
293Disallow: /slick_min_js_v18/*
294Disallow: /enttinyscrolljs/*
295Disallow: /beta_gohintjs_v7.cms*
296Disallow: /locateservice_js/*
297Disallow: /trackjs_newnav.cms*
298Disallow: /grxpushnotification_js/version-23,minify-1.cms*
299Disallow: /aqiscript/*
300Disallow: /navincludejs.cms*
301Disallow: /articlelist.cms.js*
302Disallow: /navincludejs.cms*
303Disallow: /pn/version-39,minify-1.cms*
304Disallow: /etimes_login/*
305Disallow: /etnewsletter_js/*
306Disallow: /glblnavjs_v19.cms*
307Disallow: /gaview_script/*
308Disallow: /etpn/version-26,minify-1.cms*
309Disallow: /new_bookmarkjs_v2.cms*
310Disallow: /new_slidjs_v1.cms*
311Disallow: /grxevent_config/*
312Disallow: /et_gdpr_callback_js/*
313Disallow: /et_onetap_signin_js/*
314Disallow: /getseeallhref.cms*
315Disallow: /getdataofseason.cms*
316Disallow: /affiliate_wrapper_amp.cms*
317Disallow: *feedtype-sjson*
318Disallow: *feedtype-json*
319Disallow: *feedtype=sjson*
320Disallow: *feedtype=json*
321Disallow: /ncCentreFeed*
322Disallow: /allauthrlist_v2.cms*
323Disallow: /newsletter_rhs_js.cms?minify=1&version=30*
324Disallow: /ettoiplus_verifymobile_screen_js/version-90,minify-1,isArchive-1.cms*
325Disallow: /requirejs/version-9,minify-1.cms*
326Disallow: /jquery_1_11_0_min/version-13,minify-1.cms*
327Disallow: /et_gdpr_js/minify-1,v-14.cms*
328Disallow: /lgads_v2/minify-1,channel-nbt.cms*
329Disallow: /etads_v2/minify-1.cms*
330Disallow: *js/v2/web-sdk-main.js*
331Disallow: *v2/track*
332Disallow: /toiads_prebid/minify-1.cms*
333Disallow: /toiads_react_lazy/minify-1.cms*
334Disallow: /google_ads/ads/ve-1.cms*
335Disallow: /requirejs/version-9,minify-1.cms*
336Disallow: /toiads_new/minify-1.cms*
337Disallow: /newsletter_rhs_js.cms*
338Disallow: /toiads_prebid_checker/adset/ve-1.cms*
339Disallow: /ext_*
340Disallow: /*_rsc=
341Disallow: /meta.json
342
343User-agent: AdsBot-Google
344Allow: /trivia/
345
346User-agent: Mediapartners-Google
347Allow: /trivia/
348
349User-agent: AmazonAdBot
350Allow: /trivia/
351
352User-agent: meta-externalagent
353Disallow: /
354
355Sitemap: https://timesofindia.indiatimes.com/sitemap/today
356Sitemap: https://timesofindia.indiatimes.com/sitemap/yesterday
357Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/news/sitemap-index.xml
358Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/category/city/sitemap-index.xml
359Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/category/entertainment/sitemap-index.xml
360Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/category/india/sitemap-index.xml
361Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/category/sports/sitemap-index.xml
362Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/category/business/sitemap-index.xml
363Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/category/world/sitemap-index.xml
364Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/slideshow/sitemap-index.xml
365Sitemap: https://timesofindia.indiatimes.com/staticsitemap/toi/category/life-style/sitemap-index.xml