NeuralCrawl

wikiHow / robots.txt snapshot

← back to wikihow.com · fetched 2026-10-16T06:49:36Z (22d ago) · HTTP 200 · 4514 bytes · sha256 77b692277243824d · raw

final URL: https://www.wikihow.com/robots.txt

1# robots.txt for wikiHow
2#
3# Sitemap: https://www.wikihow.com/sitemap_index.xml
4# Or: /sitemap.xml on some other domains.
5#
6# NOTICE: All crawlers and bots, regardless of whether or not they are
7# specified below, are strictly prohibited from using our content for the
8# purposes of training or retrieval for AI models or similar machine learning
9# systems, except where explicit prior permission has been granted by wikiHow
10# through a contractual licensing agreement. Any use of our content for such
11# purposes without our explicit contractual permission is a violation of our
12# terms of service and intellectual property rights.
13
14User-agent: Ai2Bot
15Disallow: /
16
17User-agent: AI2Bot
18Disallow: /
19
20User-agent: Ai2Bot-Dolma
21Disallow: /
22
23User-agent: Amazonbot
24Disallow: /
25
26User-agent: anthropic-ai
27Disallow: /
28
29User-agent: archive.org
30Disallow: /api.php
31Disallow: /index.php
32Disallow: /Special:
33
34User-agent: Applebot-Extended
35Disallow: /
36
37User-agent: Bytespider
38Disallow: /
39
40User-agent: CCBot
41Disallow: /
42
43User-agent: ChatGPT-User
44Disallow: /
45
46User-agent: ClaudeBot
47Disallow: /
48
49User-agent: Claude-SearchBot
50Disallow: /
51
52User-agent: Claude-User
53Disallow: /
54
55User-agent: Claude-Web
56Disallow: /
57
58User-agent: cohere-ai
59Disallow: /
60
61User-agent: Diffbot
62Disallow: /
63
64User-agent: DOC
65Disallow: /
66
67User-agent: Download Ninja
68Disallow: /
69
70User-agent: DuckAssistBot
71Disallow: /
72
73User-agent: FacebookBot
74Disallow: /
75
76User-agent: Facebookexternalhit
77Disallow: /
78
79User-agent: Fetch
80Disallow: /
81
82User-agent: Firecrawl
83Disallow: /
84
85User-agent: FirecrawlAgent
86Disallow: /
87
88User-agent: FriendlyCrawler
89Disallow: /
90
91User-agent: GPTBot
92Disallow: /
93
94User-agent: GrokApp
95Disallow: /
96
97User-agent: HMSE_Robot
98Disallow: /
99
100User-agent: HTTrack
101Disallow: /
102
103User-agent: ia_archiver
104Disallow: /
105
106User-agent: ICC-Crawler
107Disallow: /
108
109User-agent: ImagesiftBot
110Disallow: /
111
112User-agent: img2dataset
113Disallow: /
114
115User-agent: k2spider
116Disallow: /
117
118User-agent: larbin
119Disallow: /
120
121User-agent: libwww
122Disallow: /
123
124User-agent: linko
125Disallow: /
126
127User-agent: Meta-ExternalAgent
128Disallow: /
129
130User-agent: Meta-ExternalFetcher
131Disallow: /
132
133User-agent: Meta-WebIndexer
134Disallow: /
135
136User-agent: Microsoft.URL.Control
137Disallow: /
138
139User-agent: MistralAI-User
140Disallow: /
141
142User-agent: MSIECrawler
143Disallow: /
144
145# Requests many pages per second
146# http://www.nameprotect.com/botinfo.html
147User-agent: NPBot
148Disallow: /
149
150User-agent: OAI-SearchBot
151Disallow: /
152
153User-agent: Offline Explorer
154Disallow: /
155
156User-agent: omgili
157Disallow: /
158
159User-agent: OmigiliBot
160Disallow: /
161
162User-agent: PerplexityBot
163Disallow: /
164
165User-agent: PetalBot
166Disallow: /
167
168User-agent: Scrapy
169Disallow: /
170
171User-agent: Seekr
172Disallow: /
173
174User-agent: ShapBot
175Disallow: /
176
177# Some bots are known to be trouble, particularly those designed to copy
178# entire sites. Please obey robots.txt.
179User-agent: sitecheck.internetseer.com
180Disallow: /
181
182User-agent: SiteSnagger
183Disallow: /
184
185User-agent: Teleport
186Disallow: /
187
188User-agent: TeleportPro
189Disallow: /
190
191User-agent: TikTokSpider
192Disallow: /
193
194User-agent: Timpibot
195Disallow: /
196
197User-agent: UbiCrawler
198Disallow: /
199
200User-agent: VelenPublicWebCrawler
201Disallow: /
202
203User-agent: WebCopier
204Disallow: /
205
206User-agent: WebReaper
207Disallow: /
208
209User-agent: WebStripper
210Disallow: /
211
212User-agent: Webzio-Extended
213Disallow: /
214
215User-agent: WebZIP
216Disallow: /
217
218# wget in recursive mode uses too many resources for us.
219# Please read the man page and use it properly; there is a
220# --wait option you can use to set the delay between hits,
221# for instance. Please wait 3 seconds between each request.
222User-agent: wget
223Disallow: /
224
225User-agent: Xenu
226Disallow: /
227
228User-agent: YouBot
229Disallow: /
230
231User-agent: Zao
232Disallow: /
233
234User-agent: Zealbot
235Disallow: /
236
237User-agent: ZyBORG
238Disallow: /
239
240User-agent: AdsBot-Google
241Allow: /
242
243User-agent: Mediapartners-Google
244Allow: /
245
246User-agent: Googlebot
247Allow: /Special:NewPages
248Allow: /Special:Sitemap
249Allow: /Special:CategoryListing
250Allow: /
251
252User-agent: *
253Allow: /Special:Block
254Allow: /Special:BlockList
255Allow: /Special:Categorylisting
256Allow: /Special:CategoryListing
257Allow: /Special:Charity
258Allow: /Special:EmailUser
259Allow: /Special:LSearch
260Allow: /Special:NewPages
261Allow: /Special:QABox
262Allow: /Special:SearchAd
263Allow: /Special:Sitemap
264Allow: /Special:ThankAuthors
265Allow: /Special:UserLogin
266Allow: /index.php?*action=credits
267Allow: /index.php?*MathShowImage
268Allow: /index.php?*printable
269Disallow: /index.php
270Disallow: /*feed=rss
271Disallow: /*action=delete
272Disallow: /*action=history
273Disallow: /Special:
274Disallow: /*platform=
275Disallow: /*variant=