NeuralCrawl

Bayer / robots.txt

← back to bayer.com · change detected 1mo ago (2026-09-22T10:23:37Z) · modified +11 −116

What changed

Diff old (2026-06-20T01:10:30Z) → new (2026-09-22T10:23:37Z)

@@ -1,75 +1,20 @@
1- #
2- # robots.txt
3- #
4- # This file is to prevent the crawling and indexing of certain parts
5- # of your site by web crawlers and spiders run by sites like Yahoo!
6- # and Google. By telling these "robots" where not to go on your site,
7- # you save bandwidth and server resources.
8- #
9- # This file will be ignored unless it is at the root of your host:
10- # Used: http://example.com/robots.txt
11- # Ignored: http://example.com/site/robots.txt
12- #
13- # For more information about the robots.txt standard, see:
14- # http://www.robotstxt.org/robotstxt.html
15-
16- #User-agent: python-requests/2.20.0
17- #Disallow: /
18-
191  User-agent: *
20- # CSS, JS, Images
21- Allow: /core/*.css$
22- Allow: /core/*.css?
23- Allow: /core/*.js$
24- Allow: /core/*.js?
25- Allow: /core/*.gif
26- Allow: /core/*.jpg
27- Allow: /core/*.jpeg
28- Allow: /core/*.png
29- Allow: /core/*.svg
30- Allow: /profiles/*.css$
31- Allow: /profiles/*.css?
32- Allow: /profiles/*.js$
33- Allow: /profiles/*.js?
34- Allow: /profiles/*.gif
35- Allow: /profiles/*.jpg
36- Allow: /profiles/*.jpeg
37- Allow: /profiles/*.png
38- Allow: /profiles/*.svg
39- # Directories
40- Disallow: /core/
41- Disallow: /profiles/
42- Disallow: /sites/default/files/notbeindexed/
43- # Files
44- Disallow: /README.txt
45- Disallow: /web.config
46- # Paths (clean URLs)
47- Disallow: /admin/
48- Disallow: /comment/reply/
49- Disallow: /filter/tips
50- Disallow: /node/add/
512  Disallow: /search/
52- Disallow: /user/register/
53- Disallow: /user/password/
54- Disallow: /user/login/
55- Disallow: /user/logout/
563  Disallow: /contact/
574  Disallow: /kontakt/
585  Disallow: /report/
59- Disallow: /event/
606  Disallow: /product/
61- Disallow: /uk/ua/v-med-news/overview
62- Disallow: /uk/ua/v-med-news/1
63- Disallow: /uk/ua/v-med-news/mednews231
64- Disallow: /uk/ua/v-med-news/anticoagulant-therapy
65- Disallow: /uk/ua/v-med-news/endometriom
66- Disallow: /uk/ua/v-med-news/dienogest
67- Disallow: /uk/ua/v-med-news/contraception
68- Disallow: /uk/ua/v-med-news/2
69- Disallow: /uk/ua/v-med-news/3
70- Disallow: /uk/ua/v-med-news/time-interview
7+ Disallow: /uk-ua/v-med-news/overview
8+ Disallow: /uk-ua/v-med-news/1
9+ Disallow: /uk-ua/v-med-news/mednews231
10+ Disallow: /uk-ua/v-med-news/anticoagulant-therapy
11+ Disallow: /uk-ua/v-med-news/endometriom
12+ Disallow: /uk-ua/v-med-news/dienogest
13+ Disallow: /uk-ua/v-med-news/contraception
14+ Disallow: /uk-ua/v-med-news/2
15+ Disallow: /uk-ua/v-med-news/3
16+ Disallow: /uk-ua/v-med-news/time-interview
7117  Disallow: /*/locations$
72- Disallow: /*cl0n3*
7318 
7419  # Paths (no clean URLs)
7520  Disallow: /index.php/admin/
@@ -77,10 +22,6 @@
7722  Disallow: /index.php/filter/tips
7823  Disallow: /index.php/node/add/
7924  Disallow: /index.php/search/
80- Disallow: /index.php/user/password/
81- Disallow: /index.php/user/register/
82- Disallow: /index.php/user/login/
83- Disallow: /index.php/user/logout/
8425  Disallow: /*?
8526  Disallow: /*?autotranslate=*
8627  Disallow: /en/data-privacy-information-for-managing-contingent-workforce
@@ -88,52 +29,6 @@
8829  Disallow: /*/webform/*
8930  Disallow: /en/cxpatients
9031  Disallow: /en/investors/document-download-tool?
91- Disallow: /de/investoren/document-download-tool?
32+ Disallow: /de-de/investoren/document-download-tool?
9233  Disallow: /node/*
9334  Sitemap: https://www.bayer.com/sitemap.xml
94-
95- # ..::------==------::..
96- # .::-==========================--:.
97- # .:-=============-----------============--:.
98- # .:-========--::.. ..::---=======-:.
99- # .:-=======-:. -++++++++=-. .::--=====-:.
100- # .-=======-. =##*+++++###: .:--=====-:
101- # .-======-. =##=.....+##= .:--====-:
102- # .-=====-: =###########. .:--===-:
103- # .-=====-. =##- :##* .:--===-.
104- # :=====-: =##*+++++*##- .:--===:
105- # -=====: :=========-. .:--==-
106- # .-=====. ... .::--=-.
107- # .-====-. .####: :::-==.
108- # -====- .##*+##: :::-=-.
109- # -====- .##* +##: :::---
110- # :====-. .###=---###: :::-=-
111- # .=====: .############: .:::-=.
112- # -====- .###. *##: ::::--
113- # =====: .::::-.
114- # :===== -++++++++=- -++- .=++. .+++. :++++++++++= .++++++++=: ::::-:
115- # -====- +##++++++###. =####+ .+##= -##*. -##*=======- .##*====+##* .::::-
116- # -====: +##-:::::*##. +##--##= :##*..*##- -##+:::::::. .##+....:### .::::-
117- # -====: +##########* =##- -##= +####*. -##########- .#########*: .::::-
118- # -====: +##: -##= =###****###= =##+ -##= .##+..=##*. .:::::
119- # :-===- +##*+++++###: =##=======+##= -##= -##*+++++++= .##+ .*##- .:::::
120- # .-==== -=========: .==- -==. .==: .==========- ==: -==: :::::.
121- # :-===: .......... .:::::.
122- # ::-==- =##########+ ::::::
123- # :--==: =##-........ .:::::.
124- # ::-==- =##*+++++++: ::::::
125- # ::-==- =##+=======: ::::::
126- # ::--=- =##=........ ::::::.
127- # :::-=- =##########+ ::::::.
128- # :::---. ::::::.
129- # ::::-=: .:::::::::. .::::::
130- # .:::---. =##########*. :::::::
131- # ::::---. =##- =##= :::::::.
132- # .:::::--: =###****###*. .:::::::.
133- # .:::::--:. =##=---*##+ .:::::::.
134- # .:::::::-:. =##- =##*. .::::::::.
135- # .:::::::::.. -**: :***: .::::::::::.
136- # .::::::::::::... ..::::::::::::.
137- # .::::::::::::::::::::::::::::::::::::::::.
138- # ..::::::::::::::::::::::::::::::..
139- # ....::::::::::::::....