NeuralCrawl

Iceland (Government.is) / robots.txt snapshot

← back to government.is · fetched 2026-06-20T01:10:30Z (18h ago) · HTTP 200 · 4733 bytes · sha256 bc11b2b2c53e9055 · raw

final URL: https://government.is/robots.txt

1# As a condition of accessing this website, you agree to abide by the following
2# content signals:
3
4# (a) If a Content-Signal = yes, you may collect content for the corresponding
5# use.
6# (b) If a Content-Signal = no, you may not collect content for the
7# corresponding use.
8# (c) If the website operator does not include a Content-Signal for a
9# corresponding use, the website operator neither grants nor restricts
10# permission via Content-Signal with respect to the corresponding use.
11
12# The content signals and their meanings are:
13
14# search: building a search index and providing search results (e.g., returning
15# hyperlinks and short excerpts from your website's contents). Search does not
16# include providing AI-generated search summaries.
17# ai-input: inputting content into one or more AI models (e.g., retrieval
18# augmented generation, grounding, or other real-time taking of content for
19# generative AI search answers).
20# ai-train: training or fine-tuning AI models.
21
22# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
23# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
24# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.
25
26# BEGIN Cloudflare Managed content
27
28User-agent: *
29Content-Signal: search=yes,ai-train=no
30Allow: /
31
32User-agent: Amazonbot
33Disallow: /
34
35User-agent: Applebot-Extended
36Disallow: /
37
38User-agent: Bytespider
39Disallow: /
40
41User-agent: CCBot
42Disallow: /
43
44User-agent: ClaudeBot
45Disallow: /
46
47User-agent: CloudflareBrowserRenderingCrawler
48Disallow: /
49
50User-agent: Google-Extended
51Disallow: /
52
53User-agent: GPTBot
54Disallow: /
55
56User-agent: meta-externalagent
57Disallow: /
58
59# END Cloudflare Managed Content
60
61User-agent: *
62Disallow: /page/a8cacb2b-3fa6-4b74-b51b-cf3f473a1933
63Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/destination-iceland$
64Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/destination-iceland?
65Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/destination-iceland/$
66Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/destination-iceland/?
67Disallow: /page/a17ab7d5-c496-47df-8b83-fd87b0003c42
68Disallow: /prufa$
69Disallow: /prufa?
70Disallow: /prufa/$
71Disallow: /prufa/?
72Disallow: /page/a286ef5a-b97c-4f58-96f6-fae663674dc1
73Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-services$
74Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-services?
75Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-services/$
76Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-services/?
77Disallow: /page/fcf5f9d8-fa0d-4b43-aa0d-0a014ebbedd2
78Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls$
79Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls?
80Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/$
81Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/?
82Disallow: /page/e6bcbd6f-3827-4268-aa08-9b2568f4c936
83Disallow: /ministries/employees$
84Disallow: /ministries/employees?
85Disallow: /ministries/employees/$
86Disallow: /ministries/employees/?
87Disallow: /page/8583b52b-b0de-47c7-9397-5995b71c6761
88Disallow: /404$
89Disallow: /404?
90Disallow: /404/$
91Disallow: /404/?
92Disallow: /page/ec577559-aa02-45a6-829a-34405856097f
93Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-conferences$
94Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-conferences?
95Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-conferences/$
96Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-conferences/?
97Disallow: /page/838b7b42-f31e-4cb8-81a0-13b4fd7373a3
98Disallow: /topics/operations-and-state-assets/advisory-committee-on-anti-dumping-and-countervailing-duties$
99Disallow: /topics/operations-and-state-assets/advisory-committee-on-anti-dumping-and-countervailing-duties?
100Disallow: /topics/operations-and-state-assets/advisory-committee-on-anti-dumping-and-countervailing-duties/$
101Disallow: /topics/operations-and-state-assets/advisory-committee-on-anti-dumping-and-countervailing-duties/?
102Disallow: /page/d382cd76-1f1c-41da-82bc-3c14593273d1
103Disallow: /search/bytags$
104Disallow: /search/bytags?
105Disallow: /search/bytags/$
106Disallow: /search/bytags/?
107Disallow: /page/a94a5941-a3a1-4e2e-ac24-6adf10e60a7d
108Disallow: /topics/foreign-affairs/visa-to-iceland$
109Disallow: /topics/foreign-affairs/visa-to-iceland?
110Disallow: /topics/foreign-affairs/visa-to-iceland/$
111Disallow: /topics/foreign-affairs/visa-to-iceland/?
112Disallow: /library$
113Disallow: /library?
114Disallow: /library/$
115Disallow: /library/?