NeuralCrawl

wikiHow / robots.txt snapshot

← back to wikihow.com · fetched 2026-11-05T07:45:14Z (2d ago) · HTTP 200 · 4596 bytes · sha256 c7346a590e62e811 · raw

final URL: https://www.wikihow.com/robots.txt

1# robots.txt for wikiHow
2#
3# Sitemap: https://www.wikihow.com/sitemap_index.xml
4# Or: /sitemap.xml on some other domains.
5#
6# NOTICE: All crawlers and bots, regardless of whether or not they are
7# specified below, are strictly prohibited from using our content for the
8# purposes of training or retrieval for AI models or similar machine learning
9# systems, except where explicit prior permission has been granted by wikiHow
10# through a contractual licensing agreement. Any use of our content for such
11# purposes without our explicit contractual permission is a violation of our
12# terms of service and intellectual property rights.
13
14User-agent: __usage_ai-training
15Disallow: /
16
17User-agent: Ai2Bot
18Disallow: /
19
20User-agent: AI2Bot
21Disallow: /
22
23User-agent: Ai2Bot-Dolma
24Disallow: /
25
26User-agent: Amazonbot
27Disallow: /
28
29User-agent: anthropic-ai
30Disallow: /
31
32User-agent: archive.org
33Disallow: /api.php
34Disallow: /index.php
35Disallow: /Special:
36
37User-agent: Applebot-Extended
38Disallow: /
39
40User-agent: Bytespider
41Disallow: /
42
43User-agent: CCBot
44Disallow: /
45
46User-agent: ChatGPT-User
47Disallow: /
48
49User-agent: ClaudeBot
50Disallow: /
51
52User-agent: Claude-SearchBot
53Disallow: /
54
55User-agent: Claude-User
56Disallow: /
57
58User-agent: Claude-Web
59Disallow: /
60
61User-agent: cohere-ai
62Disallow: /
63
64User-agent: Diffbot
65Disallow: /
66
67User-agent: DOC
68Disallow: /
69
70User-agent: Download Ninja
71Disallow: /
72
73User-agent: DuckAssistBot
74Disallow: /
75
76User-agent: FacebookBot
77Disallow: /
78
79User-agent: Facebookexternalhit
80Disallow: /
81
82User-agent: Fetch
83Disallow: /
84
85User-agent: Firecrawl
86Disallow: /
87
88User-agent: FirecrawlAgent
89Disallow: /
90
91User-agent: FriendlyCrawler
92Disallow: /
93
94User-agent: GPTBot
95Disallow: /
96
97User-agent: GrokApp
98Disallow: /
99
100User-agent: HMSE_Robot
101Disallow: /
102
103User-agent: HTTrack
104Disallow: /
105
106User-agent: ia_archiver
107Disallow: /
108
109User-agent: ICC-Crawler
110Disallow: /
111
112User-agent: ImagesiftBot
113Disallow: /
114
115User-agent: img2dataset
116Disallow: /
117
118User-agent: k2spider
119Disallow: /
120
121User-agent: larbin
122Disallow: /
123
124User-agent: libwww
125Disallow: /
126
127User-agent: linko
128Disallow: /
129
130User-agent: Meta-ExternalAgent
131Disallow: /
132
133User-agent: Meta-ExternalFetcher
134Disallow: /
135
136User-agent: Meta-WebIndexer
137Disallow: /
138
139User-agent: Microsoft.URL.Control
140Disallow: /
141
142User-agent: MistralAI-User
143Disallow: /
144
145User-agent: MSIECrawler
146Disallow: /
147
148# Requests many pages per second
149# http://www.nameprotect.com/botinfo.html
150User-agent: NPBot
151Disallow: /
152
153User-agent: OAI-SearchBot
154Disallow: /
155
156User-agent: Offline Explorer
157Disallow: /
158
159User-agent: omgili
160Disallow: /
161
162User-agent: OmigiliBot
163Disallow: /
164
165User-agent: PerplexityBot
166Disallow: /
167
168User-agent: PetalBot
169Disallow: /
170
171User-agent: Scrapy
172Disallow: /
173
174User-agent: Seekr
175Disallow: /
176
177User-agent: ShapBot
178Disallow: /
179
180# Some bots are known to be trouble, particularly those designed to copy
181# entire sites. Please obey robots.txt.
182User-agent: sitecheck.internetseer.com
183Disallow: /
184
185User-agent: SiteSnagger
186Disallow: /
187
188User-agent: Teleport
189Disallow: /
190
191User-agent: TeleportPro
192Disallow: /
193
194User-agent: TikTokSpider
195Disallow: /
196
197User-agent: Timpibot
198Disallow: /
199
200User-agent: UbiCrawler
201Disallow: /
202
203User-agent: VelenPublicWebCrawler
204Disallow: /
205
206User-agent: WebCopier
207Disallow: /
208
209User-agent: WebReaper
210Disallow: /
211
212User-agent: WebStripper
213Disallow: /
214
215User-agent: Webzio-Extended
216Disallow: /
217
218User-agent: WebZIP
219Disallow: /
220
221# wget in recursive mode uses too many resources for us.
222# Please read the man page and use it properly; there is a
223# --wait option you can use to set the delay between hits,
224# for instance. Please wait 3 seconds between each request.
225User-agent: wget
226Disallow: /
227
228User-agent: Xenu
229Disallow: /
230
231User-agent: YisouSpider
232Disallow: /
233
234User-agent: YouBot
235Disallow: /
236
237User-agent: Zao
238Disallow: /
239
240User-agent: Zealbot
241Disallow: /
242
243User-agent: ZyBORG
244Disallow: /
245
246User-agent: AdsBot-Google
247Allow: /
248
249User-agent: Mediapartners-Google
250Allow: /
251
252User-agent: Googlebot
253Allow: /Special:NewPages
254Allow: /Special:Sitemap
255Allow: /Special:CategoryListing
256Allow: /
257
258User-agent: *
259Allow: /Special:Block
260Allow: /Special:BlockList
261Allow: /Special:Categorylisting
262Allow: /Special:CategoryListing
263Allow: /Special:Charity
264Allow: /Special:EmailUser
265Allow: /Special:LSearch
266Allow: /Special:NewPages
267Allow: /Special:QABox
268Allow: /Special:SearchAd
269Allow: /Special:Sitemap
270Allow: /Special:ThankAuthors
271Allow: /Special:UserLogin
272Allow: /index.php?*action=credits
273Allow: /index.php?*MathShowImage
274Allow: /index.php?*printable
275Disallow: /index.php
276Disallow: /*feed=rss
277Disallow: /*action=delete
278Disallow: /*action=history
279Disallow: /Special:
280Disallow: /*platform=
281Disallow: /*variant=