NeuralCrawl

Airbus / robots.txt snapshot

← back to airbus.com · fetched 2026-06-20T01:10:30Z (18h ago) · HTTP 200 · 6667 bytes · sha256 ab2f74133abae3cb · raw

final URL: https://www.airbus.com/robots.txt

1#
2# robots.txt
3#
4# This file is to prevent the crawling and indexing of certain parts
5# of your site by web crawlers and spiders run by sites like Yahoo!
6# and Google. By telling these "robots" where not to go on your site,
7# you save bandwidth and server resources.
8#
9# This file will be ignored unless it is at the root of your host:
10# Used: http://example.com/robots.txt
11# Ignored: http://example.com/site/robots.txt
12#
13# For more information about the robots.txt standard, see:
14# http://www.robotstxt.org/robotstxt.html
15
16User-agent: AlphaSeoBot
17User-agent: AlphaSeoBot-SA
18User-agent: BLEXBot
19User-agent: 008
20User-agent: Alexibot
21User-agent: AlvinetSpider
22User-agent: Antenne Hatena
23User-agent: ApocalXExplorerBot
24User-agent: asterias
25User-agent: BackDoorBot/1.0
26User-agent: BizInformation
27User-agent: Black Hole
28User-agent: BlowFish/1.0
29User-agent: BotALot
30User-agent: BuiltBotTough
31User-agent: Bullseye/1.0
32User-agent: BunnySlippers
33User-agent: Bytespider
34User-agent: Cegbfeieh
35User-agent: CheeseBot
36User-agent: CherryPicker
37User-agent: CherryPickerElite/1.0
38User-agent: CherryPickerSE/1.0
39User-agent: CopyRightCheck
40User-agent: cosmos
41User-agent: Crescent
42User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
43User-agent: DISCo Pump 3.1
44User-agent: DittoSpyder
45User-agent: dotbot
46User-agent: EmailCollector
47User-agent: EmailSiphon
48User-agent: EmailWolf
49User-agent: EroCrawler
50User-agent: Exabot
51User-agent: ExtractorPro
52User-agent: Flamingo_SearchEngine
53User-agent: Foobot
54User-agent: grapeshot
55User-agent: Harvest/1.5
56User-agent: hloader
57User-agent: httplib
58User-agent: HTTrack
59User-agent: HTTrack 3.0
60User-agent: humanlinks
61User-agent: Igentia
62User-agent: InfoNaviRobot
63User-agent: JennyBot
64User-agent: JikeSpider
65User-agent: Kenjin Spider
66User-agent: LexiBot
67User-agent: libWeb/clsHTTP
68User-agent: LinkextractorPro
69User-agent: LinkScan/8.1a Unix
70User-agent: LinkWalker
71User-agent: lwp-trivial
72User-agent: lwp-trivial/1.34
73User-agent: Mata Hari
74User-agent: Microsoft URL Control - 5.01.4511
75User-agent: Microsoft URL Control - 6.00.8169
76User-agent: MIIxpc
77User-agent: MIIxpc/4.2
78User-agent: Mister PiX
79User-agent: MLBot
80User-agent: moget
81User-agent: moget/2.1
82User-agent: MS Search 4.0 Robot
83User-agent: MS Search 5.0 Robot
84User-agent: Naverbot
85User-agent: NetAnts
86User-agent: NetAttache
87User-agent: NetMechanic
88User-agent: NICErsPRO
89User-agent: Offline Explorer
90User-agent: Openfind
91User-agent: OpenindexSpider
92User-agent: ProPowerBot/2.14
93User-agent: ProWebWalker
94User-agent: psbot
95User-agent: QuepasaCreep
96User-agent: QueryN Metasearch
97User-agent: RepoMonkey
98User-agent: RMA
99User-agent: SightupBot
100User-agent: SiteBot
101User-agent: SiteSnagger
102User-agent: SiteSucker
103User-agent: SpankBot
104User-agent: spanner
105User-agent: Speedy
106User-agent: suggybot
107User-agent: SuperBot
108User-agent: SuperBot/2.6
109User-agent: suzuran
110User-agent: Szukacz/1.4
111User-agent: Teleport
112User-agent: Telesoft
113User-agent: The Intraformant
114User-agent: TheNomad
115User-agent: TightTwatBot
116User-agent: Titan
117User-agent: toCrawl/UrlDispatcher
118User-agent: TosCrawler
119User-agent: trendictionbot
120User-agent: True_Robot
121User-agent: True_Robot/1.0
122User-agent: turingos
123User-agent: TurnitinBot
124User-agent: UrlPouls
125User-agent: URLy Warning
126User-agent: VCI
127User-agent: Web Image Collector
128User-agent: WebAuto
129User-agent: WebBandit
130User-agent: WebBandit/3.50
131User-agent: WebCopier
132User-agent: webcopy
133User-agent: WebEnhancer
134User-agent: WebmasterWorldForumBot
135User-agent: webmirror
136User-agent: WebReaper
137User-agent: WebSauger
138User-agent: website extractor
139User-agent: Website Quester
140User-agent: Webster Pro
141User-agent: WebStripper
142User-agent: WebStripper/2.02
143User-agent: WebZip
144User-agent: wget
145User-agent: WikioFeedBot
146User-agent: WinHTTrack
147User-agent: WWW-Collector-E
148User-agent: Xenu Link Sleuth/1.3.8
149User-agent: YisouSpider
150User-agent: yacy
151User-agent: YRSPider
152User-agent: Zeus
153User-agent: Zookabot
154Disallow: /
155
156User-agent: *
157# FIXME TWO LINES UNDER TO REMOVE ON MARCH 1st, LINE FOR TESTS IN PRODUCTION
158Disallow: /en/caching-tests/*
159Disallow: /fr/caching-tests/*
160# CSS, JS, Images
161Allow: /core/*.css$
162Allow: /core/*.css?
163Allow: /core/*.js$
164Allow: /core/*.js?
165Allow: /core/*.gif
166Allow: /core/*.jpg
167Allow: /core/*.jpeg
168Allow: /core/*.png
169Allow: /core/*.svg
170Allow: /profiles/*.css$
171Allow: /profiles/*.css?
172Allow: /profiles/*.js$
173Allow: /profiles/*.js?
174Allow: /profiles/*.gif
175Allow: /profiles/*.jpg
176Allow: /profiles/*.jpeg
177Allow: /profiles/*.png
178Allow: /profiles/*.svg
179# Directories
180Disallow: /core/
181Disallow: /profiles/
182# Files
183Disallow: /README.txt
184Disallow: /web.config
185# Paths (clean URLs)
186Disallow: /admin/
187Disallow: /comment/reply/
188Disallow: /filter/tips
189Disallow: /node/add/
190Disallow: /en/search*
191Disallow: /fr/search*
192Disallow: /de/search*
193Disallow: /es/search*
194Disallow: /zh-hans/search*
195Disallow: /ja/search*
196Disallow: /kk/search*
197Disallow: /ru/search*
198# Addition 20/03 new languages, part 1 (WEBFACT-4722)
199Disallow: /fi/search*
200Disallow: /pt-pt/search*
201Disallow: /pt-br/search*
202Disallow: /pl/search*
203# End 20/03 new languages, part 1
204Disallow: /en/asset-preview*
205Disallow: /fr/asset-preview*
206Disallow: /de/asset-preview*
207Disallow: /es/asset-preview*
208Disallow: /zh-hans/asset-preview*
209Disallow: /ja/asset-preview*
210Disallow: /kk/asset-preview*
211Disallow: /ru/asset-preview*
212# Addition 20/03 new languages, part 2 (WEBFACT-4722)
213Disallow: /fi/asset-preview*
214Disallow: /pt-pt/asset-preview*
215Disallow: /pt-br/asset-preview*
216Disallow: /pl/asset-preview*
217# End 20/03 new languages, part 2
218Disallow: /en/products-services/space/in-space-infrastructure/bartolomeo/request-your-payload-slot-on-bartolomeo/bartolomeo-form-confirmation
219Disallow: /en/innovation/the-airbus-nova-newsletter/the-nova-newsletter-subscription-confirmation
220Disallow: /en/terms-of-use
221Disallow: /en/thanks-for-signing-up-to-our-newsletter
222Disallow: /en/statement-on-accessibility
223Disallow: /de/die-h225-als-polizeihubschrauber
224# Paths (no clean URLs)
225Disallow: /index.php/admin/
226Disallow: /index.php/comment/reply/
227Disallow: /index.php/filter/tips
228Disallow: /index.php/node/add/
229Disallow: /index.php/search/
230Disallow: /index.php/user/password/
231Disallow: /index.php/user/register/
232Disallow: /index.php/user/login/
233Disallow: /index.php/user/logout/
234Disallow: /fr/opportunites-dinterim-et-formations-cqpm-chez-airbus-en-aeronautique
235Disallow: /*/node/
236# June 5th Disallowing these Code of Conduct
237Disallow: /sites/g/files/jlcbta136/files/2021-07/Airbus-Ethics-Compliance-Code-Conduct-EN_1.pdf
238Disallow: /sites/g/files/jlcbta136/files/2021-07/Airbus-Ethics-Compliance-Code-Conduct-FR.pdf
239Sitemap: https://www.airbus.com/sitemap.xml
240# Disallow search factes and other search filters to be indexed
241Disallow: /*/search?