Letterboxd / robots.txt snapshot

← back to letterboxd.com · fetched 2026-06-20T01:10:31Z (18h ago) · HTTP 200 · 3402 bytes · sha256 8aba08fbd5831028 · raw

final URL: https://letterboxd.com/robots.txt

1	# Dark Visitors robots.txt from https://darkvisitors.com/robots-txt-builder
2
3	# AI Data Scraper
4	# https://darkvisitors.com/agents/ai2bot
5
6	User-agent: AI2Bot
7	Disallow: /
8
9	# AI Data Scraper
10	# https://darkvisitors.com/agents/applebot-extended
11
12	User-agent: Applebot-Extended
13	Disallow: /
14
15	# AI Data Scraper
16	# https://darkvisitors.com/agents/bytespider
17
18	User-agent: Bytespider
19	Disallow: /
20
21	# AI Data Scraper
22	# https://darkvisitors.com/agents/ccbot
23
24	User-agent: CCBot
25	Disallow: /
26
27	# AI Data Scraper
28	# https://darkvisitors.com/agents/claudebot
29
30	User-agent: ClaudeBot
31	Disallow: /
32
33	# AI Data Scraper
34	# https://darkvisitors.com/agents/diffbot
35
36	User-agent: Diffbot
37	Disallow: /
38
39	# AI Data Scraper
40	# https://darkvisitors.com/agents/cohere-training-data-crawler
41
42	User-agent: cohere-training-data-crawler
43	Disallow: /
44
45	# AI Data Scraper
46	# https://darkvisitors.com/agents/cotoyogi
47
48	User-agent: Cotoyogi
49	Disallow: /
50
51	# AI Data Scraper
52	# https://darkvisitors.com/agents/datenbank-crawler
53
54	User-agent: Datenbank Crawler
55	Disallow: /
56
57	# AI Data Scraper
58	# https://darkvisitors.com/agents/facebookbot
59
60	User-agent: FacebookBot
61	Disallow: /
62
63	# AI Data Scraper
64	# https://darkvisitors.com/agents/factset-spyderbot
65
66	User-agent: Factset_spyderbot
67	Disallow: /
68
69	# AI Data Scraper
70	# https://darkvisitors.com/agents/google-extended
71
72	User-agent: Google-Extended
73	Disallow: /
74
75	# AI Data Scraper
76	# https://darkvisitors.com/agents/gptbot
77
78	User-agent: GPTBot
79	Disallow: /
80
81	# AI Data Scraper
82	# https://darkvisitors.com/agents/icc-crawler
83
84	User-agent: ICC-Crawler
85	Disallow: /
86
87	# AI Data Scraper
88	# https://darkvisitors.com/agents/kangaroo-bot
89
90	User-agent: Kangaroo Bot
91	Disallow: /
92
93	# AI Data Scraper
94	# https://darkvisitors.com/agents/meta-externalagent
95
96	User-agent: Meta-ExternalAgent
97	Disallow: /
98
99	# AI Data Scraper
100	# https://darkvisitors.com/agents/netestate-imprint-crawler
101
102	User-agent: netEstate Imprint Crawler
103	Disallow: /
104
105	# AI Data Scraper
106	# https://darkvisitors.com/agents/omgili
107
108	User-agent: omgili
109	Disallow: /
110
111	# AI Data Scraper
112	# https://darkvisitors.com/agents/pangubot
113
114	User-agent: PanguBot
115	Disallow: /
116
117	# AI Data Scraper
118	# https://darkvisitors.com/agents/petalbot
119
120	User-agent: PetalBot
121	Disallow: /
122
123	# AI Data Scraper
124	# https://darkvisitors.com/agents/semrushbot-ocob
125
126	User-agent: SemrushBot-OCOB
127	Disallow: /
128
129	# AI Data Scraper
130	# https://darkvisitors.com/agents/timpibot
131
132	User-agent: Timpibot
133	Disallow: /
134
135	# AI Data Scraper
136	# https://darkvisitors.com/agents/webzio-extended
137
138	User-agent: Webzio-Extended
139	Disallow: /
140
141	# Indexes content for Algolia search engine - which
142	# https://radar.cloudflare.com/bots/directory/algolia
143
144	User-Agent: Algolia Crawler
145	Disallow: /
146
147	# All other crawlers
148	User-agent: *
149	Disallow: //by/ # sorting options
150	Disallow: //popular/this/ # popularity sorting options
151	Disallow: //on/ # availability options
152	Disallow: //tag/ # Members? tag lists
153	Disallow: //genre/ # Films by genre
154	Disallow: //country/ # Films by country
155	Disallow: //language/ # Films by language
156	Disallow: //decade/ # Films by decade
157	Disallow: /films/year/* # Films by year
158	Disallow: /films//year/ # Films by year
159	Disallow: /films//size/large/ # Films with large posters (and therefore stats)
160	Disallow: //friends/ # stuff grouped for users' friends