NeuralCrawl

Daily Mail / robots.txt snapshot

← back to dailymail.co.uk · fetched 2026-07-07T20:35:12Z (1mo ago) · HTTP 200 · 5996 bytes · sha256 eb8299f3cbd42d8a · raw

final URL: https://www.dailymail.com/robots.txt

1# Robots.txt for https://www.dailymail.com/ updated 07/07/2026
2#
3#
4# Daily Mail content is made available for your personal and non-commercial
5# use subject to our Terms and Conditions of use:
6# https://www.dailymail.com/terms
7# Use of any device, tool, or process designed to data mine or scrape the content
8# using automated means is prohibited without prior written permission from
9# DMG MEDIA LIMITED. Prohibited uses include but are not limited to:
10# (1) text and data mining activities under Art. 4 of the EU Directive on Copyright in
11# the Digital Single Market;
12# (2) the development of any software, machine learning, artificial intelligence (AI),
13# and/or large language models (LLMs);
14# (3) creating or providing archived or cached data sets containing our content to others; and/or
15# (4) any commercial purposes.
16# For any licensing queries please contact [email protected].
17#
18#
19# All Robots
20User-agent: *
21#
22# Begin Standard Rules
23Disallow: /*readcommentshtml*
24Disallow: /*login?redirectPath=
25Disallow: /*logout?redirectPath=
26Disallow: /*nextThread.html
27Disallow: /*previousThread.html
28Disallow: /*questionId*
29Disallow: /*selectedImage*
30Disallow: /*threadIndex=*
31Disallow: /*topGallery*
32Disallow: /ce/item.cms*
33Disallow: /guide/*
34Disallow: /registration/*
35Disallow: /best-buys/clickout/*
36Disallow: /best-buys/m/clickout/*
37Disallow: /weather.html?latitude=*
38Disallow: /weather.html?old=*
39Disallow: /home/search.html*
40Disallow: /best-buys/mercury/out*
41Disallow: /api/infinite-list.html
42Disallow: /api/infinite-list.html
43Disallow: /mobileapps/*
44
45#
46# Disallow Money for Google News
47User-agent: Googlebot-News
48Disallow: /tmoney/*
49#
50# Allow Adsense
51User-agent: Mediapartners-Google
52Disallow:
53#
54#
55User-agent: CrystalSemanticsBot
56Disallow: /
57#
58User-agent: archive-it.org
59User-agent: archive.org_bot
60User-agent: ArchiveBot
61User-agent: Arquivo-web-crawler
62User-agent: Authory
63User-agent: babel
64User-agent: bl.uk_ldfc_bot
65User-agent: bl.uk_ldfc_renderbot
66User-agent: bne.es_bot
67User-agent: bnf.fr_bot
68User-agent: ecoresearch
69User-agent: ecoresearchCrawler
70User-agent: Europarchive.org
71User-agent: kb.dk_bot
72User-agent: Kulturarw3
73User-agent: LivelapBot
74User-agent: mediacloud
75User-agent: mirrorweb
76User-agent: MSIECrawler
77User-agent: nationalarchives.gov.uk
78User-agent: netarkivet.dk
79User-agent: netEstate NE Crawler
80User-agent: Nicecrawler
81User-agent: nl-israel_iaharvester2026
82User-agent: nlnbot
83User-agent: nlncrawler
84User-agent: PageFreezer
85User-agent: SafeDNS
86User-agent: SmarshBot
87User-agent: special_archiver
88User-agent: tkl.iis.u-tokyo.ac.jp
89User-agent: TurnitinBot
90User-agent: webarchiv.cz
91User-agent: WordPress.com mShots
92User-agent: XY-Archive-Compliance-Archiver
93User-agent: XY-Archive-Compliance-Crawler
94Disallow: /
95#
96User-agent: AhrefsBot
97User-agent: AI2Bot
98User-agent: Ai2Bot-Dolma
99User-agent: Amazonbot
100User-agent: amazon-kendra-
101User-agent: anthropic-ai
102User-agent: Applebot-Extended
103User-agent: bedrockbot
104User-agent: Bytespider
105User-agent: CloudflareBrowserRenderingCrawler
106User-agent: CCBot
107User-agent: ChatGLM-Spider
108User-agent: ChatGPT-User
109User-agent: ClaudeBot
110User-agent: Claude-SearchBot
111User-agent: Claude-User
112User-agent: Claude-Web
113User-agent: cohere-ai
114User-agent: Cotoyogi
115User-agent: DeepSeekBot
116User-agent: Diffbot
117User-agent: DuckAssistBot
118User-agent: EchoboxBot
119User-agent: FacebookBot
120User-agent: FriendlyCrawler
121User-agent: Gemini-Deep-Research
122User-agent: Google-CloudVertexBot
123User-agent: Google-Extended
124User-agent: GoogleOther-Image
125User-agent: GoogleOther-Video
126User-agent: GPTBot
127User-agent: Grok
128User-agent: iaskspider/2.0
129User-agent: ICC-Crawler
130User-agent: ImagesiftBot
131User-agent: img2dataset
132User-agent: ISSCyberRiskCrawler
133User-agent: Kangaroo Bot
134User-agent: KunatoCrawler
135User-agent: LinerBot
136User-agent: Meltwater
137User-agent: Meta-ExternalAgent
138User-agent: Meta-ExternalFetcher
139User-agent: MistralAI-User
140User-agent: OAI-Operator
141User-agent: OAI-SearchBot
142User-agent: omgili
143User-agent: omgilibot
144User-agent: PanguBot
145User-agent: PerplexityBot
146User-agent: Perplexity-User
147User-agent: PetalBot
148User-agent: QualifiedBot
149User-agent: Scrapy
150User-agent: Seekr
151User-agent: Sidetrade indexer bot
152User-agent: TaraGroup Intelligent Bot
153User-agent: TikTokSpider
154User-agent: Timpibot
155User-agent: VelenPublicWebCrawler
156User-agent: WARDBot
157User-agent: Webzio-Extended
158User-agent: wpbot
159User-agent: WRTNBot
160User-agent: YouBot
161Allow: /buyline/
162Allow: /*_.html
163Disallow: /
164#
165#Allow Google sandbox privacy testing
166User-agent: GoogleOther
167Allow: /.well-known/privacy-sandbox-attestations.json
168#
169# Sitemap Files
170Sitemap: https://www.dailymail.com/google-news-sitemap.xml
171Sitemap: https://www.dailymail.com/sitemap-channels.xml
172Sitemap: https://www.dailymail.com/sitemap-columnist.xml
173Sitemap: https://www.dailymail.com/sitemap-topics.xml
174Sitemap: https://www.dailymail.com/sitemap-videos-year~2021.xml
175Sitemap: https://www.dailymail.com/sitemap-facts-year~2021.xml
176Sitemap: https://www.dailymail.com/sitemap-articles-year~2021.xml
177Sitemap: https://www.dailymail.com/sitemap-videos-year~2022.xml
178Sitemap: https://www.dailymail.com/sitemap-articles-year~2022.xml
179Sitemap: https://www.dailymail.com/sitemap-facts-year~2022.xml
180Sitemap: https://www.dailymail.com/sitemap-videos-year~2023.xml
181Sitemap: https://www.dailymail.com/sitemap-articles-year~2023.xml
182Sitemap: https://www.dailymail.com/sitemap-facts-year~2023.xml
183Sitemap: https://www.dailymail.com/sitemap-videos-year~2024.xml
184Sitemap: https://www.dailymail.com/sitemap-articles-year~2024.xml
185Sitemap: https://www.dailymail.com/sitemap-facts-year~2024.xml
186Sitemap: https://www.dailymail.com/sitemap-videos-year~2025.xml
187Sitemap: https://www.dailymail.com/sitemap-articles-year~2025.xml
188Sitemap: https://www.dailymail.com/sitemap-facts-year~2025.xml
189Sitemap: https://www.dailymail.com/sitemap-videos-year~2026.xml
190Sitemap: https://www.dailymail.com/sitemap-articles-year~2026.xml
191Sitemap: https://www.dailymail.com/sitemap-facts-year~2026.xml
192Sitemap: https://www.dailymail.com/sitemap-authors.xml