NeuralCrawl

Iceland (Government.is) / robots.txt snapshot

← back to government.is · fetched 2026-08-04T06:35:30Z (1d ago) · HTTP 200 · 4831 bytes · sha256 dadec8a39bc91fd4 · raw

final URL: https://government.is/robots.txt

1# As a condition of accessing this website, you agree to abide by the following
2# content signals:
3
4# (a) If a Content-Signal = yes, you may collect content for the corresponding
5# use.
6# (b) If a Content-Signal = no, you may not collect content for the
7# corresponding use.
8# (c) If the website operator does not include a Content-Signal for a
9# corresponding use, the website operator neither grants nor restricts
10# permission via Content-Signal with respect to the corresponding use.
11
12# The content signals and their meanings are:
13
14# search: building a search index and providing search results (e.g., returning
15# hyperlinks and short excerpts from your website's contents). Search does not
16# include providing AI-generated search summaries.
17# ai-input: inputting content into one or more AI models (e.g., retrieval
18# augmented generation, grounding, or other real-time taking of content for
19# generative AI search answers).
20# ai-train: training or fine-tuning AI models.
21# use: how AI systems may consume the content (immediate, reference, or full).
22
23# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
24# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
25# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.
26
27# BEGIN Cloudflare Managed content
28
29User-agent: *
30Content-Signal: search=yes,ai-train=no,use=reference
31Allow: /
32
33User-agent: Amazonbot
34Disallow: /
35
36User-agent: Applebot-Extended
37Disallow: /
38
39User-agent: Bytespider
40Disallow: /
41
42User-agent: CCBot
43Disallow: /
44
45User-agent: ClaudeBot
46Disallow: /
47
48User-agent: CloudflareBrowserRenderingCrawler
49Disallow: /
50
51User-agent: Google-Extended
52Disallow: /
53
54User-agent: GPTBot
55Disallow: /
56
57User-agent: meta-externalagent
58Disallow: /
59
60# END Cloudflare Managed Content
61
62User-agent: *
63Disallow: /page/a8cacb2b-3fa6-4b74-b51b-cf3f473a1933
64Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/destination-iceland$
65Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/destination-iceland?
66Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/destination-iceland/$
67Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/destination-iceland/?
68Disallow: /page/a17ab7d5-c496-47df-8b83-fd87b0003c42
69Disallow: /prufa$
70Disallow: /prufa?
71Disallow: /prufa/$
72Disallow: /prufa/?
73Disallow: /page/a286ef5a-b97c-4f58-96f6-fae663674dc1
74Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-services$
75Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-services?
76Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-services/$
77Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-services/?
78Disallow: /page/fcf5f9d8-fa0d-4b43-aa0d-0a014ebbedd2
79Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls$
80Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls?
81Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/$
82Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/?
83Disallow: /page/e6bcbd6f-3827-4268-aa08-9b2568f4c936
84Disallow: /ministries/employees$
85Disallow: /ministries/employees?
86Disallow: /ministries/employees/$
87Disallow: /ministries/employees/?
88Disallow: /page/8583b52b-b0de-47c7-9397-5995b71c6761
89Disallow: /404$
90Disallow: /404?
91Disallow: /404/$
92Disallow: /404/?
93Disallow: /page/ec577559-aa02-45a6-829a-34405856097f
94Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-conferences$
95Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-conferences?
96Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-conferences/$
97Disallow: /ministries/ministry-for-foreign-affairs/honorary-consuls/consular-conferences/?
98Disallow: /page/838b7b42-f31e-4cb8-81a0-13b4fd7373a3
99Disallow: /topics/operations-and-state-assets/advisory-committee-on-anti-dumping-and-countervailing-duties$
100Disallow: /topics/operations-and-state-assets/advisory-committee-on-anti-dumping-and-countervailing-duties?
101Disallow: /topics/operations-and-state-assets/advisory-committee-on-anti-dumping-and-countervailing-duties/$
102Disallow: /topics/operations-and-state-assets/advisory-committee-on-anti-dumping-and-countervailing-duties/?
103Disallow: /page/d382cd76-1f1c-41da-82bc-3c14593273d1
104Disallow: /search/bytags$
105Disallow: /search/bytags?
106Disallow: /search/bytags/$
107Disallow: /search/bytags/?
108Disallow: /page/a94a5941-a3a1-4e2e-ac24-6adf10e60a7d
109Disallow: /topics/foreign-affairs/visa-to-iceland$
110Disallow: /topics/foreign-affairs/visa-to-iceland?
111Disallow: /topics/foreign-affairs/visa-to-iceland/$
112Disallow: /topics/foreign-affairs/visa-to-iceland/?
113Disallow: /library$
114Disallow: /library?
115Disallow: /library/$
116Disallow: /library/?