NeuralCrawl

Galp / robots.txt snapshot

← back to galp.com · fetched 2026-06-20T01:10:30Z (15h ago) · HTTP 200 · 3982 bytes · sha256 76d2a7dabab6d8cf · raw

final URL: https://www.galp.com/robots.txt

1# Begin robots.txt file
2#/-----------------------------------------------\
3#| In single portal/domain situations, uncomment the sitmap line and enter domain name
4#\-----------------------------------------------/
5#Sitemap: http://www.DomainNamehere.com/sitemap.aspx
6
7
8User-agent: *
9Disallow: /admin/
10Disallow: /App_Browsers/
11Disallow: /App_Code/
12Disallow: /App_Data/
13Disallow: /App_GlobalResources/
14Disallow: /bin/
15Disallow: /Components/
16Disallow: /Config/
17Disallow: /contest/
18Disallow: /controls/
19Disallow: /DesktopModules/
20Disallow: /Documentation/
21Disallow: /HttpModules/
22Disallow: /images/
23Disallow: /Install/
24Disallow: /js/
25Disallow: /Portals/
26Disallow: /Providers/
27Disallow: /Resources/ContentRotator/
28Disallow: /Resources/ControlPanel/
29Disallow: /Resources/Dashboard/
30Disallow: /Resources/FeedBrowser/
31Disallow: /Resources/OpenForceAd/
32Disallow: /Resources/Search/
33Disallow: /Resources/Shared/
34Disallow: /Resources/SkinWidgets/
35Disallow: /Resources/TabStrip/
36Disallow: /Resources/Widgets/
37Disallow: /Activity-Feed/userId/ # Do not index user profiles
38
39User-agent: msnbot
40Disallow: /admin/
41Disallow: /App_Browsers/
42Disallow: /App_Code/
43Disallow: /App_Data/
44Disallow: /App_GlobalResources/
45Disallow: /bin/
46Disallow: /Components/
47Disallow: /Config/
48Disallow: /contest/
49Disallow: /controls/
50Disallow: /DesktopModules/
51Disallow: /Documentation/
52Disallow: /HttpModules/
53Disallow: /images/
54Disallow: /Install/
55Disallow: /js/
56Disallow: /Portals/
57Disallow: /Providers/
58Disallow: /Resources/ContentRotator/
59Disallow: /Resources/ControlPanel/
60Disallow: /Resources/Dashboard/
61Disallow: /Resources/FeedBrowser/
62Disallow: /Resources/OpenForceAd/
63Disallow: /Resources/Search/
64Disallow: /Resources/Shared/
65Disallow: /Resources/SkinWidgets/
66Disallow: /Resources/TabStrip/
67Disallow: /Resources/Widgets/
68Disallow: /Activity-Feed/userId/ # Do not index user profiles
69Crawl-delay: 5
70
71User-agent: Slurp
72Disallow: /*/ctl/ # Slurp permits *
73Disallow: /admin/
74Disallow: /App_Browsers/
75Disallow: /App_Code/
76Disallow: /App_Data/
77Disallow: /App_GlobalResources/
78Disallow: /bin/
79Disallow: /Components/
80Disallow: /Config/
81Disallow: /contest/
82Disallow: /controls/
83Disallow: /DesktopModules/
84Disallow: /Documentation/
85Disallow: /HttpModules/
86Disallow: /images/
87Disallow: /Install/
88Disallow: /js/
89Disallow: /Portals/
90Disallow: /Providers/
91Disallow: /Resources/ContentRotator/
92Disallow: /Resources/ControlPanel/
93Disallow: /Resources/Dashboard/
94Disallow: /Resources/FeedBrowser/
95Disallow: /Resources/OpenForceAd/
96Disallow: /Resources/Search/
97Disallow: /Resources/Shared/
98Disallow: /Resources/SkinWidgets/
99Disallow: /Resources/TabStrip/
100Disallow: /Resources/Widgets/
101Disallow: /Activity-Feed/userId/ # Do not index user profiles
102Crawl-delay: 5
103
104User-agent: Googlebot
105Disallow: /*/ctl/ # Googlebot permits *
106Disallow: /admin/
107Disallow: /App_Browsers/
108Disallow: /App_Code/
109Disallow: /App_Data/
110Disallow: /App_GlobalResources/
111Disallow: /bin/
112Disallow: /Components/
113Disallow: /Config/
114Disallow: /contest/
115Disallow: /controls/
116Disallow: /Documentation/
117Disallow: /HttpModules/
118Disallow: /Install/
119Disallow: /Providers/
120Disallow: /Activity-Feed/userId/ # Do not index user profiles
121
122User-agent: Yahoo Pipes 1.0
123Disallow: /admin/
124Disallow: /App_Browsers/
125Disallow: /App_Code/
126Disallow: /App_Data/
127Disallow: /App_GlobalResources/
128Disallow: /bin/
129Disallow: /Components/
130Disallow: /Config/
131Disallow: /contest/
132Disallow: /controls/
133Disallow: /DesktopModules/
134Disallow: /Documentation/
135Disallow: /HttpModules/
136Disallow: /images/
137Disallow: /Install/
138Disallow: /js/
139Disallow: /Portals/
140Disallow: /Providers/
141Disallow: /Resources/ContentRotator/
142Disallow: /Resources/ControlPanel/
143Disallow: /Resources/Dashboard/
144Disallow: /Resources/FeedBrowser/
145Disallow: /Resources/OpenForceAd/
146Disallow: /Resources/Search/
147Disallow: /Resources/Shared/
148Disallow: /Resources/SkinWidgets/
149Disallow: /Resources/TabStrip/
150Disallow: /Resources/Widgets/
151Disallow: /Activity-Feed/userId/ # Do not index user profiles
152
153
154# End of robots.txt file