NeuralCrawl

Trend Micro / robots.txt snapshot

← back to trendmicro.com · fetched 2026-06-20T14:27:25Z (7h ago) · HTTP 200 · 2329 bytes · sha256 028994f021efd41d · raw

final URL: https://www.trendmicro.com/robots.txt

1# Rules for Adsbot-Google (specific bot)
2User-agent: Adsbot-Google
3Allow: /core/docs/datasheets/*.pdf$
4Allow: *.js$
5Allow: *.css$
6Disallow: /*.pdf$
7Disallow: /*/config/
8Disallow: /*/common/
9Disallow: /*/error-messages/
10Disallow: /*/business/trials_parsing_pages/
11Disallow: /*/business/parsing-pages/
12Disallow: /download/*
13Disallow: /form/
14Disallow: /ftp/
15Disallow: /go/
16Disallow: /housecall/
17Disallow: /housecall/us/
18Disallow: /housecall7/
19Disallow: /ja_jp/forHome/xsp/
20Disallow: /machform/
21Disallow: /tools/*
22Disallow: /*?bvstate=*
23
24# Rules for onsite search crawler
25User-agent: cludo
26Allow: *.pdf$
27Disallow: /*/list/*
28Disallow: /*/tag/*
29Disallow: /*/config/
30Disallow: /*/common/
31Disallow: /*/error-messages/
32Disallow: /*/business/trials_parsing_pages/
33Disallow: /*/business/parsing-pages/
34
35# Rules for LLM/AI crawlers
36User-agent: OAI-SearchBot
37User-agent: ChatGPT-User
38User-agent: ChatGPT-User/2.0
39User-agent: ClaudeBot
40User-agent: claude-web
41User-agent: Claude-SearchBot
42User-agent: Claude-User
43User-agent: PerplexityBot
44User-agent: Perplexity-User
45User-agent: Google-Extended
46User-agent: Googlebot
47User-agent: Bingbot
48User-agent: Amazonbot
49User-agent: Applebot
50User-agent: FacebookBot
51User-agent: meta-externalagent
52User-agent: LinkedInBot
53User-agent: Bytespider
54User-agent: DuckAssistBot
55User-agent: cohere-ai
56User-agent: CCBot
57Allow: *.pdf$
58Allow: *.js$
59Allow: *.css$
60Disallow: /*/config/
61Disallow: /*/common/
62Disallow: /*/error-messages/
63Disallow: /*/business/trials_parsing_pages/
64Disallow: /*/business/parsing-pages/
65Disallow: /download/*
66Disallow: /form/
67Disallow: /ftp/
68Disallow: /go/
69Disallow: /housecall/
70Disallow: /housecall/us/
71Disallow: /housecall7/
72Disallow: /ja_jp/forHome/xsp/
73Disallow: /machform/
74Disallow: /tools/*
75Disallow: /*?bvstate=*
76
77# Rules for all other bots
78User-agent: *
79Allow: /core/docs/datasheets/*.pdf$
80Disallow: /*.pdf$
81Disallow: /*/config/
82Disallow: /*/common/
83Disallow: /*/error-messages/
84Disallow: /*/business/trials_parsing_pages/
85Disallow: /*/business/parsing-pages/
86Disallow: /download/*
87Disallow: /form/
88Disallow: /ftp/
89Disallow: /go/
90Disallow: /housecall/
91Disallow: /housecall/us/
92Disallow: /housecall7/
93Disallow: /ja_jp/forHome/xsp/
94Disallow: /machform/
95Disallow: /tools/*
96Disallow: /*?bvstate=*
97Disallow: /cloudoneconformity-staging/*
98
99Sitemap: https://www.trendmicro.com/sitemap.xml