NeuralCrawl

Bayer / robots.txt snapshot

← back to bayer.com · fetched 2026-06-20T01:10:30Z (15h ago) · HTTP 200 · 6757 bytes · sha256 dbce5409b9535f8e · raw

final URL: https://www.bayer.com/en/robots.txt

1#
2# robots.txt
3#
4# This file is to prevent the crawling and indexing of certain parts
5# of your site by web crawlers and spiders run by sites like Yahoo!
6# and Google. By telling these "robots" where not to go on your site,
7# you save bandwidth and server resources.
8#
9# This file will be ignored unless it is at the root of your host:
10# Used: http://example.com/robots.txt
11# Ignored: http://example.com/site/robots.txt
12#
13# For more information about the robots.txt standard, see:
14# http://www.robotstxt.org/robotstxt.html
15
16#User-agent: python-requests/2.20.0
17#Disallow: /
18
19User-agent: *
20# CSS, JS, Images
21Allow: /core/*.css$
22Allow: /core/*.css?
23Allow: /core/*.js$
24Allow: /core/*.js?
25Allow: /core/*.gif
26Allow: /core/*.jpg
27Allow: /core/*.jpeg
28Allow: /core/*.png
29Allow: /core/*.svg
30Allow: /profiles/*.css$
31Allow: /profiles/*.css?
32Allow: /profiles/*.js$
33Allow: /profiles/*.js?
34Allow: /profiles/*.gif
35Allow: /profiles/*.jpg
36Allow: /profiles/*.jpeg
37Allow: /profiles/*.png
38Allow: /profiles/*.svg
39# Directories
40Disallow: /core/
41Disallow: /profiles/
42Disallow: /sites/default/files/notbeindexed/
43# Files
44Disallow: /README.txt
45Disallow: /web.config
46# Paths (clean URLs)
47Disallow: /admin/
48Disallow: /comment/reply/
49Disallow: /filter/tips
50Disallow: /node/add/
51Disallow: /search/
52Disallow: /user/register/
53Disallow: /user/password/
54Disallow: /user/login/
55Disallow: /user/logout/
56Disallow: /contact/
57Disallow: /kontakt/
58Disallow: /report/
59Disallow: /event/
60Disallow: /product/
61Disallow: /uk/ua/v-med-news/overview
62Disallow: /uk/ua/v-med-news/1
63Disallow: /uk/ua/v-med-news/mednews231
64Disallow: /uk/ua/v-med-news/anticoagulant-therapy
65Disallow: /uk/ua/v-med-news/endometriom
66Disallow: /uk/ua/v-med-news/dienogest
67Disallow: /uk/ua/v-med-news/contraception
68Disallow: /uk/ua/v-med-news/2
69Disallow: /uk/ua/v-med-news/3
70Disallow: /uk/ua/v-med-news/time-interview
71Disallow: /*/locations$
72Disallow: /*cl0n3*
73
74# Paths (no clean URLs)
75Disallow: /index.php/admin/
76Disallow: /index.php/comment/reply/
77Disallow: /index.php/filter/tips
78Disallow: /index.php/node/add/
79Disallow: /index.php/search/
80Disallow: /index.php/user/password/
81Disallow: /index.php/user/register/
82Disallow: /index.php/user/login/
83Disallow: /index.php/user/logout/
84Disallow: /*?
85Disallow: /*?autotranslate=*
86Disallow: /en/data-privacy-information-for-managing-contingent-workforce
87Disallow: /*/form/*
88Disallow: /*/webform/*
89Disallow: /en/cxpatients
90Disallow: /en/investors/document-download-tool?
91Disallow: /de/investoren/document-download-tool?
92Disallow: /node/*
93Sitemap: https://www.bayer.com/sitemap.xml
94
95# ..::------==------::..
96# .::-==========================--:.
97# .:-=============-----------============--:.
98# .:-========--::.. ..::---=======-:.
99# .:-=======-:. -++++++++=-. .::--=====-:.
100# .-=======-. =##*+++++###: .:--=====-:
101# .-======-. =##=.....+##= .:--====-:
102# .-=====-: =###########. .:--===-:
103# .-=====-. =##- :##* .:--===-.
104# :=====-: =##*+++++*##- .:--===:
105# -=====: :=========-. .:--==-
106# .-=====. ... .::--=-.
107# .-====-. .####: :::-==.
108# -====- .##*+##: :::-=-.
109# -====- .##* +##: :::---
110# :====-. .###=---###: :::-=-
111# .=====: .############: .:::-=.
112# -====- .###. *##: ::::--
113# =====: .::::-.
114# :===== -++++++++=- -++- .=++. .+++. :++++++++++= .++++++++=: ::::-:
115# -====- +##++++++###. =####+ .+##= -##*. -##*=======- .##*====+##* .::::-
116# -====: +##-:::::*##. +##--##= :##*..*##- -##+:::::::. .##+....:### .::::-
117# -====: +##########* =##- -##= +####*. -##########- .#########*: .::::-
118# -====: +##: -##= =###****###= =##+ -##= .##+..=##*. .:::::
119# :-===- +##*+++++###: =##=======+##= -##= -##*+++++++= .##+ .*##- .:::::
120# .-==== -=========: .==- -==. .==: .==========- ==: -==: :::::.
121# :-===: .......... .:::::.
122# ::-==- =##########+ ::::::
123# :--==: =##-........ .:::::.
124# ::-==- =##*+++++++: ::::::
125# ::-==- =##+=======: ::::::
126# ::--=- =##=........ ::::::.
127# :::-=- =##########+ ::::::.
128# :::---. ::::::.
129# ::::-=: .:::::::::. .::::::
130# .:::---. =##########*. :::::::
131# ::::---. =##- =##= :::::::.
132# .:::::--: =###****###*. .:::::::.
133# .:::::--:. =##=---*##+ .:::::::.
134# .:::::::-:. =##- =##*. .::::::::.
135# .:::::::::.. -**: :***: .::::::::::.
136# .::::::::::::... ..::::::::::::.
137# .::::::::::::::::::::::::::::::::::::::::.
138# ..::::::::::::::::::::::::::::::..
139# ....::::::::::::::....