NeuralCrawl

Toronto Star / robots.txt snapshot

← back to thestar.com · fetched 2026-06-20T14:27:25Z (4h ago) · HTTP 200 · 7807 bytes · sha256 d429d584cfe04036 · raw

final URL: https://thestar.com/robots.txt

1User-agent: AI2Bot
2Disallow: /
3
4User-agent: Ai2Bot-Dolma
5Disallow: /
6
7User-agent: Applebot-Extended
8Disallow: /
9
10User-agent: Bytespider
11Disallow: /
12
13User-agent: CCBot
14Disallow: /
15
16User-agent: ChatGPT-User
17Disallow: /
18
19User-agent: Claude-Web
20Disallow: /
21
22User-agent: ClaudeBot
23Disallow: /
24
25User-agent: Diffbot
26Disallow: /
27
28User-agent: FacebookBot
29Disallow: /
30
31User-agent: FriendlyCrawler
32Disallow: /
33
34User-agent: GPTBot
35Disallow: /
36
37User-agent: Google-Extended
38Disallow: /
39
40User-agent: ICC-Crawler
41Disallow: /
42
43User-agent: ImagesiftBot
44Disallow: /
45
46User-agent: Meta-ExternalAgent
47Disallow: /
48
49User-agent: OAI-SearchBot
50Disallow: /
51
52User-agent: PerplexityBot
53Disallow: /
54
55User-agent: PetalBot
56Disallow: /
57
58User-agent: Scrapy
59Disallow: /
60
61User-agent: Timpibot
62Disallow: /
63
64User-agent: VelenPublicWebCrawler
65Disallow: /
66
67User-agent: Webzio-Extended
68Disallow: /
69
70User-agent: YouBot
71Disallow: /
72
73User-agent: anthropic-ai
74Disallow: /
75
76User-agent: cohere-ai
77Disallow: /
78
79User-agent: iaskspider/2.0
80Disallow: /
81
82User-agent: img2dataset
83Disallow: /
84
85User-agent: omgili
86Disallow: /
87
88User-agent: omgilibot
89Disallow: /
90
91User-agent: cohere-training-data-crawler
92Disallow: /
93
94User-agent: Kangaroo Bot
95Disallow: /
96
97User-agent: PanguBot
98Disallow: /
99
100User-agent: Amazonbot
101Disallow: /
102
103User-agent: DuckAssistBot
104Disallow: /
105
106User-agent: Meta-ExternalFetcher
107Disallow: /
108
109User-agent: Perplexity-User
110Disallow: /
111
112User-agent: Claude-SearchBot
113Disallow: /
114
115User-agent: Claude-User
116Disallow: /
117
118User-agent: Cotoyogi
119Disallow: /
120
121User-agent: Datenbank
122Disallow: /
123
124User-agent: Devin
125Disallow: /
126
127User-agent: Factset_spyderbot
128Disallow: /
129
130User-agent: Gemini-Deep-Research
131Disallow: /
132
133User-agent: MistralAI-User
134Disallow: /
135
136User-agent: netEstate
137Disallow: /
138
139User-agent: Imprint
140Disallow: /
141
142User-agent: SemrushBot-OCOB
143Disallow: /
144
145User-agent: AddSearchBot
146Disallow: /
147
148User-agent: bigsur.ai
149Disallow: /
150
151User-agent: CloudVertexBot
152Disallow: /
153
154User-agent: GoogleOther
155Disallow: /
156
157User-agent: LinerBot
158Disallow: /
159
160User-agent: meta-externalagent
161Disallow: /
162
163User-agent: QualifiedBot
164Disallow: /
165
166User-agent: Ai2Bot-DeepResearchEval
167Disallow: /
168
169User-agent: ChatGLM-Spider
170Disallow: /
171
172User-agent: DeepSeekBot
173Disallow: /
174
175User-agent: Google-NotebookLM
176Disallow: /
177
178User-agent: iAskBot
179Disallow: /
180
181User-agent: KunatoCrawler
182Disallow: /
183
184User-agent: meta-externalfetcher
185Disallow: /
186
187User-agent: meta-webindexer
188Disallow: /
189
190User-agent: webzio-extended
191Disallow: /
192
193User-agent: WRTNBot
194Disallow: /
195
196User-agent: ZanistaBot
197Disallow: /
198
199User-agent: AI2Bot-DeepResearchEval
200Disallow: /
201
202User-agent: Anomura
203Disallow: /
204
205User-agent: atlassian-bot
206Disallow: /
207
208User-agent: Bravebot
209Disallow: /
210
211User-agent: Channel3Bot
212Disallow: /
213
214User-agent: Cloudflare-AutoRAG
215Disallow: /
216
217User-agent: Crawl4AI
218Disallow: /
219
220User-agent: Google-CloudVertexBot
221Disallow: /
222
223User-agent: imageSpider
224Disallow: /
225
226User-agent: KlaviyoAIBot
227Disallow: /
228
229User-agent: laion-huggingface-processor
230Disallow: /
231
232User-agent: LCC
233Disallow: /
234
235User-agent: LinkupBot
236Disallow: /
237
238User-agent: PhindBot
239Disallow: /
240
241User-agent: Poggio-Citations
242Disallow: /
243
244User-agent: SBIntuitionsBot
245Disallow: /
246
247User-agent: TavilyBot
248Disallow: /
249
250User-agent: Amzn-SearchBot
251Disallow: /
252
253User-agent: Amzn-User
254Disallow: /
255
256User-agent: ApifyBot
257Disallow: /
258
259User-agent: ApifyWebsiteContentCrawler
260Disallow: /
261
262User-agent: Aranet-SearchBot
263Disallow: /
264
265User-agent: AzureAI-SearchBot
266Disallow: /
267
268User-agent: ExaBot
269Disallow: /
270
271User-agent: FirecrawlAgent
272Disallow: /
273
274User-agent: kagi-fetcher
275Disallow: /
276
277User-agent: TerraCotta
278Disallow: /
279
280User-agent: *
281Disallow: /tncms/tracking/
282Disallow: /_services/
283Disallow: /buycanadianbriefing/
284Disallow: /newsletters/books/
285Disallow: /newsletters/business/
286Disallow: /newsletters/evening-headlines/
287Disallow: /newsletters/long-reads/
288Disallow: /newsletters/news/
289Disallow: /newsletters/opinion/
290Disallow: /newsletters/real-estate/
291Disallow: /newsletters/sports/
292Disallow: /newsletters/storyalerts/
293Disallow: /newsletters/success/
294Disallow: /newsletters/unsubscribe/
295Disallow: /newspapers/header/
296Disallow: /registration/
297Disallow: /registration/success/
298Disallow: /search/
299Disallow: /site/advertise/
300Disallow: /static/
301Disallow: /subscribe/3forfree/
302Disallow: /subscribe/amplified/
303Disallow: /subscribe/canoo/
304Disallow: /subscribe/promo/
305Disallow: /subscribe/registrant-onboarding-25/
306Disallow: /subscribe/registrant-onboarding-40/
307Disallow: /subscribe/registrant-onboarding-50/
308Disallow: /subscribe/special/
309Disallow: /subscribe/tryfree/
310Disallow: /subscribe/wb-auto/
311Disallow: /t/
312Disallow: /tapp/
313Disallow: /test/
314Disallow: /test_aplus/
315Disallow: /tncms/tax/
316Disallow: /tncms/media/
317Disallow: /tncms/gtm/
318Disallow: /tncms/admin/
319Disallow: /tncms/counter/
320Disallow: /tncms/csrf/
321Disallow: /tncms/messaging/
322Disallow: /tncms/auth/
323Disallow: /tncms/block/
324Disallow: /tncms/webservice/
325Disallow: /tncms/search/
326Disallow: /tncms/track/
327Disallow: /tncms/weather/
328Disallow: /tncms/dmp/
329Disallow: /tncms/openid2/
330Disallow: /tncms/openweb/*
331Disallow: /content/tncms/assets/v3/form/
332Disallow: /content/tncms/form/
333Disallow: /tncms/user/
334Disallow: /users/admin/
335Disallow: /users/*/?
336Disallow: /marketplace/*action=srch
337Disallow: /tncms/calendar/
338Disallow: /tncms/disqus/
339Disallow: /content/tncms/
340Disallow: /tncms/asset/editorial/http*
341Disallow: /*%7b%7burl%7d%7d
342Disallow: /*%7B%7Burl%7D%7D
343Disallow: /*{{url}}
344Disallow: /content/tncms/live/
345Disallow: /assets/fonts/
346Disallow: /content/dam/
347Disallow: /*li_medium=
348Disallow: /*li_source=
349
350Sitemap: https://www.thestar.com/sitemap.xml
351Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml
352Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=1979
353Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=1980
354Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=1984
355Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=1985
356Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=1992
357Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=1993
358Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=1995
359Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=1998
360Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=1999
361Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2002
362Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2003
363Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2005
364Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2007
365Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2008
366Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2009
367Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2010
368Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2011
369Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2012
370Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2013
371Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2014
372Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2015
373Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2016
374Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2017
375Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2018
376Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2019
377Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2020
378Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2021
379Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2022
380Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2023
381Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2024
382Sitemap: https://www.thestar.com/tncms/sitemap/editorial.xml?year=2025