Toronto Star / robots.txt
← back to thestar.com · change detected 29d ago (2026-10-08T22:07:03Z) · modified +36 −18
What changed
- Blocked user-agent ‘KeenableBot’ entirely
- Blocked user-agent ‘recraft-image-dataset’ entirely
- Blocked user-agent ‘ScryBot’ entirely
- Blocked user-agent ‘SofyaBot’ entirely
- Blocked user-agent ‘TalarionFrontierCrawler’ entirely
- Blocked user-agent ‘YiBot’ entirely
Diff old (2026-10-08T16:07:06Z) → new (2026-10-08T22:07:03Z)
| @@ -682,6 +682,24 @@ | ||
| 682 | 682 | User-agent: Keenable-User |
| 683 | 683 | Disallow: / |
| 684 | 684 | |
| 685 | + User-agent: KeenableBot | |
| 686 | + Disallow: / | |
| 687 | + | |
| 688 | + User-agent: recraft-image-dataset | |
| 689 | + Disallow: / | |
| 690 | + | |
| 691 | + User-agent: ScryBot | |
| 692 | + Disallow: / | |
| 693 | + | |
| 694 | + User-agent: SofyaBot | |
| 695 | + Disallow: / | |
| 696 | + | |
| 697 | + User-agent: TalarionFrontierCrawler | |
| 698 | + Disallow: / | |
| 699 | + | |
| 700 | + User-agent: YiBot | |
| 701 | + Disallow: / | |
| 702 | + | |
| 685 | 703 | User-agent: * |
| 686 | 704 | Disallow: /tncms/tracking/ |
| 687 | 705 | Disallow: /_services/ |
| @@ -717,31 +735,31 @@ | ||
| 717 | 735 | Disallow: /tapp/ |
| 718 | 736 | Disallow: /test/ |
| 719 | 737 | Disallow: /test_aplus/ |
| 738 | + Disallow: /tncms/openid2/ | |
| 739 | + Disallow: /tncms/track/ | |
| 740 | + Disallow: /tncms/counter/ | |
| 720 | 741 | Disallow: /tncms/dmp/ |
| 742 | + Disallow: /tncms/weather/ | |
| 743 | + Disallow: /tncms/media/ | |
| 744 | + Disallow: /tncms/webservice/ | |
| 745 | + Disallow: /tncms/auth/ | |
| 721 | 746 | Disallow: /tncms/block/ |
| 722 | - Disallow: /tncms/openid2/ | |
| 723 | - Disallow: /tncms/webservice/ | |
| 747 | + Disallow: /tncms/csrf/ | |
| 748 | + Disallow: /tncms/messaging/ | |
| 749 | + Disallow: /tncms/admin/ | |
| 750 | + Disallow: /tncms/gtm/ | |
| 751 | + Disallow: /tncms/tax/ | |
| 752 | + Disallow: /tncms/openweb/* | |
| 724 | 753 | Disallow: /tncms/search/ |
| 725 | - Disallow: /tncms/tax/ | |
| 726 | - Disallow: /tncms/counter/ | |
| 727 | - Disallow: /tncms/admin/ | |
| 728 | - Disallow: /tncms/weather/ | |
| 729 | - Disallow: /tncms/openweb/* | |
| 730 | - Disallow: /tncms/track/ | |
| 731 | - Disallow: /tncms/messaging/ | |
| 732 | - Disallow: /tncms/gtm/ | |
| 733 | - Disallow: /tncms/media/ | |
| 734 | - Disallow: /tncms/csrf/ | |
| 735 | - Disallow: /tncms/auth/ | |
| 736 | - Disallow: /content/tncms/ | |
| 754 | + Disallow: /tncms/disqus/ | |
| 755 | + Disallow: /content/tncms/assets/v3/form/ | |
| 756 | + Disallow: /content/tncms/form/ | |
| 757 | + Disallow: /tncms/calendar/ | |
| 737 | 758 | Disallow: /tncms/user/ |
| 738 | 759 | Disallow: /users/admin/ |
| 739 | 760 | Disallow: /users/*/? |
| 740 | - Disallow: /tncms/disqus/ | |
| 741 | - Disallow: /content/tncms/assets/v3/form/ | |
| 742 | - Disallow: /content/tncms/form/ | |
| 761 | + Disallow: /content/tncms/ | |
| 743 | 762 | Disallow: /marketplace/*action=srch |
| 744 | - Disallow: /tncms/calendar/ | |
| 745 | 763 | Disallow: /tncms/asset/editorial/http* |
| 746 | 764 | Disallow: /*%7b%7burl%7d%7d |
| 747 | 765 | Disallow: /*%7B%7Burl%7D%7D |