NeuralCrawl

Daily Mail / robots.txt snapshot

← back to dailymail.co.uk · fetched 2026-07-03T13:16:54Z (1mo ago) · HTTP 200 · 6030 bytes · sha256 155dba83dad5c02a · raw

final URL: https://www.dailymail.com/robots.txt

1# Robots.txt for https://www.dailymail.com/ updated 03/07/2026
2#
3#
4# Daily Mail content is made available for your personal and non-commercial
5# use subject to our Terms and Conditions of use:
6# https://www.dailymail.com/terms
7# Use of any device, tool, or process designed to data mine or scrape the content
8# using automated means is prohibited without prior written permission from
9# DMG MEDIA LIMITED. Prohibited uses include but are not limited to:
10# (1) text and data mining activities under Art. 4 of the EU Directive on Copyright in
11# the Digital Single Market;
12# (2) the development of any software, machine learning, artificial intelligence (AI),
13# and/or large language models (LLMs);
14# (3) creating or providing archived or cached data sets containing our content to others; and/or
15# (4) any commercial purposes.
16# For any licensing queries please contact [email protected].
17#
18#
19# All Robots
20User-agent: *
21#
22# Begin Standard Rules
23Disallow: /*readcommentshtml*
24Disallow: /*login?redirectPath=
25Disallow: /*logout?redirectPath=
26Disallow: /*nextThread.html
27Disallow: /*previousThread.html
28Disallow: /*questionId*
29Disallow: /*selectedImage*
30Disallow: /*threadIndex=*
31Disallow: /*topGallery*
32Disallow: /ce/item.cms*
33Disallow: /guide/*
34Disallow: /registration/*
35Disallow: /best-buys/clickout/*
36Disallow: /best-buys/m/clickout/*
37Disallow: /weather.html?latitude=*
38Disallow: /weather.html?old=*
39Disallow: /home/search.html*
40Disallow: /best-buys/mercury/out*
41Disallow: /api/infinite-list.html
42Disallow: /api/infinite-list.html
43Disallow: /mobileapps/*
44Disallow: /news/the-harry-trial/*
45
46#
47# Disallow Money for Google News
48User-agent: Googlebot-News
49Disallow: /tmoney/*
50#
51# Allow Adsense
52User-agent: Mediapartners-Google
53Disallow:
54#
55#
56User-agent: CrystalSemanticsBot
57Disallow: /
58#
59User-agent: archive-it.org
60User-agent: archive.org_bot
61User-agent: ArchiveBot
62User-agent: Arquivo-web-crawler
63User-agent: Authory
64User-agent: babel
65User-agent: bl.uk_ldfc_bot
66User-agent: bl.uk_ldfc_renderbot
67User-agent: bne.es_bot
68User-agent: bnf.fr_bot
69User-agent: ecoresearch
70User-agent: ecoresearchCrawler
71User-agent: Europarchive.org
72User-agent: kb.dk_bot
73User-agent: Kulturarw3
74User-agent: LivelapBot
75User-agent: mediacloud
76User-agent: mirrorweb
77User-agent: MSIECrawler
78User-agent: nationalarchives.gov.uk
79User-agent: netarkivet.dk
80User-agent: netEstate NE Crawler
81User-agent: Nicecrawler
82User-agent: nl-israel_iaharvester2026
83User-agent: nlnbot
84User-agent: nlncrawler
85User-agent: PageFreezer
86User-agent: SafeDNS
87User-agent: SmarshBot
88User-agent: special_archiver
89User-agent: tkl.iis.u-tokyo.ac.jp
90User-agent: TurnitinBot
91User-agent: webarchiv.cz
92User-agent: WordPress.com mShots
93User-agent: XY-Archive-Compliance-Archiver
94User-agent: XY-Archive-Compliance-Crawler
95Disallow: /
96#
97User-agent: AhrefsBot
98User-agent: AI2Bot
99User-agent: Ai2Bot-Dolma
100User-agent: Amazonbot
101User-agent: amazon-kendra-
102User-agent: anthropic-ai
103User-agent: Applebot-Extended
104User-agent: bedrockbot
105User-agent: Bytespider
106User-agent: CloudflareBrowserRenderingCrawler
107User-agent: CCBot
108User-agent: ChatGLM-Spider
109User-agent: ChatGPT-User
110User-agent: ClaudeBot
111User-agent: Claude-SearchBot
112User-agent: Claude-User
113User-agent: Claude-Web
114User-agent: cohere-ai
115User-agent: Cotoyogi
116User-agent: DeepSeekBot
117User-agent: Diffbot
118User-agent: DuckAssistBot
119User-agent: EchoboxBot
120User-agent: FacebookBot
121User-agent: FriendlyCrawler
122User-agent: Gemini-Deep-Research
123User-agent: Google-CloudVertexBot
124User-agent: Google-Extended
125User-agent: GoogleOther-Image
126User-agent: GoogleOther-Video
127User-agent: GPTBot
128User-agent: Grok
129User-agent: iaskspider/2.0
130User-agent: ICC-Crawler
131User-agent: ImagesiftBot
132User-agent: img2dataset
133User-agent: ISSCyberRiskCrawler
134User-agent: Kangaroo Bot
135User-agent: KunatoCrawler
136User-agent: LinerBot
137User-agent: Meltwater
138User-agent: Meta-ExternalAgent
139User-agent: Meta-ExternalFetcher
140User-agent: MistralAI-User
141User-agent: OAI-Operator
142User-agent: OAI-SearchBot
143User-agent: omgili
144User-agent: omgilibot
145User-agent: PanguBot
146User-agent: PerplexityBot
147User-agent: Perplexity-User
148User-agent: PetalBot
149User-agent: QualifiedBot
150User-agent: Scrapy
151User-agent: Seekr
152User-agent: Sidetrade indexer bot
153User-agent: TaraGroup Intelligent Bot
154User-agent: TikTokSpider
155User-agent: Timpibot
156User-agent: VelenPublicWebCrawler
157User-agent: WARDBot
158User-agent: Webzio-Extended
159User-agent: wpbot
160User-agent: WRTNBot
161User-agent: YouBot
162Allow: /buyline/
163Allow: /*_.html
164Disallow: /
165#
166#Allow Google sandbox privacy testing
167User-agent: GoogleOther
168Allow: /.well-known/privacy-sandbox-attestations.json
169#
170# Sitemap Files
171Sitemap: https://www.dailymail.com/google-news-sitemap.xml
172Sitemap: https://www.dailymail.com/sitemap-channels.xml
173Sitemap: https://www.dailymail.com/sitemap-columnist.xml
174Sitemap: https://www.dailymail.com/sitemap-topics.xml
175Sitemap: https://www.dailymail.com/sitemap-videos-year~2021.xml
176Sitemap: https://www.dailymail.com/sitemap-facts-year~2021.xml
177Sitemap: https://www.dailymail.com/sitemap-articles-year~2021.xml
178Sitemap: https://www.dailymail.com/sitemap-videos-year~2022.xml
179Sitemap: https://www.dailymail.com/sitemap-articles-year~2022.xml
180Sitemap: https://www.dailymail.com/sitemap-facts-year~2022.xml
181Sitemap: https://www.dailymail.com/sitemap-videos-year~2023.xml
182Sitemap: https://www.dailymail.com/sitemap-articles-year~2023.xml
183Sitemap: https://www.dailymail.com/sitemap-facts-year~2023.xml
184Sitemap: https://www.dailymail.com/sitemap-videos-year~2024.xml
185Sitemap: https://www.dailymail.com/sitemap-articles-year~2024.xml
186Sitemap: https://www.dailymail.com/sitemap-facts-year~2024.xml
187Sitemap: https://www.dailymail.com/sitemap-videos-year~2025.xml
188Sitemap: https://www.dailymail.com/sitemap-articles-year~2025.xml
189Sitemap: https://www.dailymail.com/sitemap-facts-year~2025.xml
190Sitemap: https://www.dailymail.com/sitemap-videos-year~2026.xml
191Sitemap: https://www.dailymail.com/sitemap-articles-year~2026.xml
192Sitemap: https://www.dailymail.com/sitemap-facts-year~2026.xml
193Sitemap: https://www.dailymail.com/sitemap-authors.xml