NeuralCrawl

Iceland (Government.is) / robots.txt snapshot

← back to government.is · fetched 2026-10-19T14:06:06Z (18d ago) · HTTP 200 · 5671 bytes · sha256 35461a4e9ca4028d · raw

final URL: https://government.is/robots.txt

1# As a condition of accessing this website, you agree to abide by the following
2# content signals:
3
4# (a) If a Content-Signal = yes, you may collect content for the corresponding
5# use.
6# (b) If a Content-Signal = no, you may not collect content for the
7# corresponding use.
8# (c) If the website operator does not include a Content-Signal for a
9# corresponding use, the website operator neither grants nor restricts
10# permission via Content-Signal with respect to the corresponding use.
11
12# The content signals and their meanings are:
13
14# search: building a search index and providing search results (e.g., returning
15# hyperlinks and short excerpts from your website's contents). Search does not
16# include providing AI-generated search summaries.
17# ai-input: inputting content into one or more AI models (e.g., retrieval
18# augmented generation, grounding, or other real-time taking of content for
19# generative AI search answers).
20# ai-train: training or fine-tuning AI models.
21# use: how AI systems may consume the content (immediate, reference, or full).
22
23# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
24# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
25# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.
26
27# BEGIN Cloudflare Managed content
28
29
30User-agent: *
31Content-Signal: search=yes,ai-train=no,use=reference
32Allow: /
33
34# Training crawlers
35User-agent: Amazonbot
36Disallow: /
37
38User-agent: Applebot-Extended
39Disallow: /
40
41User-agent: Bytespider
42Disallow: /
43
44User-agent: CCBot
45Disallow: /
46
47User-agent: ClaudeBot
48Disallow: /
49
50User-agent: Diffbot
51Disallow: /
52
53User-agent: Google-Extended
54Disallow: /
55
56User-agent: GPTBot
57Disallow: /
58
59User-agent: omgili
60Disallow: /
61
62User-agent: anthropic-ai
63Disallow: /
64
65User-agent: Claude-Web
66Disallow: /
67
68User-agent: cohere-ai
69Disallow: /
70
71User-agent: MistralAI-Training
72Disallow: /
73
74User-agent: meta-externalagent
75Disallow: /
76
77User-agent: GoogleOther
78Disallow: /
79
80User-agent: Baiduspider
81Disallow: /
82
83User-agent: PetalBot
84Disallow: /
85
86User-agent: AwarioSmartBot
87User-agent: AwarioRssBot
88Disallow: /
89
90User-agent: Google-CloudVertexBot
91Disallow: /
92
93User-agent: QualifiedBot
94Disallow: /
95
96User-agent: Cotoyogi
97Disallow: /
98
99User-agent: ICC-Crawler
100Disallow: /
101
102User-agent: atlassian-bot
103Disallow: /
104
105User-agent: FishBot
106Disallow: /
107
108User-agent: BorderxBot
109Disallow: /
110
111User-agent: NavuBot
112Disallow: /
113
114User-agent: SemrushBot-SWA
115Disallow: /
116
117User-agent: WARDBot
118Disallow: /
119
120User-agent: magpie-crawler
121Disallow: /
122
123User-agent: KimiBot
124Disallow: /
125
126User-agent: CitibotSiteCrawler
127Disallow: /
128
129# END Cloudflare Managed Content
130
131User-agent: *
132Disallow: /page/a8cacb2b-3fa6-4b74-b51b-cf3f473a1933
133Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/destination-iceland$
134Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/destination-iceland?
135Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/destination-iceland/$
136Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/destination-iceland/?
137Disallow: /page/a17ab7d5-c496-47df-8b83-fd87b0003c42
138Disallow: /prufa$
139Disallow: /prufa?
140Disallow: /prufa/$
141Disallow: /prufa/?
142Disallow: /page/a286ef5a-b97c-4f58-96f6-fae663674dc1
143Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-services$
144Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-services?
145Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-services/$
146Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-services/?
147Disallow: /page/fcf5f9d8-fa0d-4b43-aa0d-0a014ebbedd2
148Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls$
149Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls?
150Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/$
151Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/?
152Disallow: /page/d382cd76-1f1c-41da-82bc-3c14593273d1
153Disallow: /search/bytags$
154Disallow: /search/bytags?
155Disallow: /search/bytags/$
156Disallow: /search/bytags/?
157Disallow: /page/e6bcbd6f-3827-4268-aa08-9b2568f4c936
158Disallow: /ministries/employees$
159Disallow: /ministries/employees?
160Disallow: /ministries/employees/$
161Disallow: /ministries/employees/?
162Disallow: /page/8583b52b-b0de-47c7-9397-5995b71c6761
163Disallow: /404$
164Disallow: /404?
165Disallow: /404/$
166Disallow: /404/?
167Disallow: /page/ec577559-aa02-45a6-829a-34405856097f
168Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-conferences$
169Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-conferences?
170Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-conferences/$
171Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-conferences/?
172Disallow: /page/838b7b42-f31e-4cb8-81a0-13b4fd7373a3
173Disallow: /topics/operations-and-state-assets/advisory-committee-on-anti-dumping-and-countervailing-duties$
174Disallow: /topics/operations-and-state-assets/advisory-committee-on-anti-dumping-and-countervailing-duties?
175Disallow: /topics/operations-and-state-assets/advisory-committee-on-anti-dumping-and-countervailing-duties/$
176Disallow: /topics/operations-and-state-assets/advisory-committee-on-anti-dumping-and-countervailing-duties/?
177Disallow: /page/a94a5941-a3a1-4e2e-ac24-6adf10e60a7d
178Disallow: /topics/foreign-affairs/visa-to-iceland$
179Disallow: /topics/foreign-affairs/visa-to-iceland?
180Disallow: /topics/foreign-affairs/visa-to-iceland/$
181Disallow: /topics/foreign-affairs/visa-to-iceland/?
182Disallow: /library$
183Disallow: /library?
184Disallow: /library/$
185Disallow: /library/?