NeuralCrawl

X (Twitter) / robots.txt snapshot

← back to x.com · fetched 2026-11-05T19:53:47Z (1d ago) · HTTP 200 · 3039 bytes · sha256 b54dd4fefb8786b6 · raw

final URL: https://x.com/robots.txt

1# Google / Bing Search Engine Robots
2# ==================================
3# Shared group so Googlebot and Bingbot always get the same rules.
4User-agent: Googlebot
5User-agent: Bingbot
6
7Allow: /*?s=
8Allow: /*?t=
9Allow: /*?ref_src=
10Allow: /hashtag/*?src=
11Allow: /search?q=%23
12Allow: /i/api/
13# Same length as the /*/likes and /*/media rules; RFC 9309 breaks the tie in
14# favor of Allow, so /i/api/ fetches stay crawlable.
15Allow: /i/api/*
16Disallow: /*?lang=en-ss
17Allow: /*?lang=
18Disallow: /search/realtime
19Disallow: /search/users
20Disallow: /search/*/grid
21
22Disallow: /*/analytics
23Disallow: /*/followers
24Disallow: /*/following
25Disallow: /*/verified_followers
26
27Disallow: /account/deactivated
28Disallow: /settings/deactivated
29
30# Only `*` and `$` are metacharacters here (RFC 9309); regex classes match
31# literally. /photo is anchored so /status/*/photo/1 media pages stay open.
32Disallow: /*/likes
33Disallow: /*/likes?
34Disallow: /*/retweets
35Disallow: /*/retweets?
36Disallow: /*/media
37Disallow: /*/media?
38Disallow: /*/photo$
39Disallow: /*/photo?
40Allow: /*?
41
42User-agent: facebookexternalhit
43
44Allow: /*?lang=
45Allow: /*?s=
46Allow: /*&s=
47Allow: /*?t=
48Allow: /*?ref_src=
49Allow: /hashtag/*?src=
50Allow: /search?*cashtagRestId=
51Allow: /i/api/
52Allow: /i/api/*
53Disallow: /search?q=
54Disallow: /search/realtime
55Disallow: /search/users
56Disallow: /search/*/grid
57
58Disallow: /*?
59Disallow: /*/followers
60Disallow: /*/following
61Disallow: /*/verified_followers
62
63Disallow: /account/deactivated
64Disallow: /settings/deactivated
65
66Disallow: /*/likes
67Disallow: /*/likes?
68Disallow: /*/retweets
69Disallow: /*/retweets?
70Disallow: /*/media
71Disallow: /*/media?
72Disallow: /*/photo$
73Disallow: /*/photo?
74
75User-Agent: Google-Extended
76Disallow: *
77
78User-Agent: FacebookBot
79Disallow: *
80
81User-agent: Discordbot
82Disallow: *
83
84# Meta AI / product crawlers (product tokens; versions like /1.1 are ignored
85# under RFC 9309 token matching).
86User-agent: meta-webindexer
87Disallow: *
88
89User-agent: meta-externalagent
90Disallow: *
91
92User-agent: meta-externalads
93Disallow: *
94
95User-agent: meta-externalfetcher
96Disallow: *
97
98# Shared AI training preference (IETF draft-kelley-aipref-training-preference-signal),
99# honored by Microsoft, Google, OpenAI, and Anthropic. This is a usage policy
100# token, not a crawler: it does not change crawl access, and `User-agent: *`
101# never matches it.
102User-agent: __usage_ai-training
103Disallow: /
104
105# Every bot that might possibly read and respect this file
106# ========================================================
107User-agent: *
108Disallow: /
109
110
111# WHAT-4882 - Keep notification-email links (/i/u) out of search results.
112# Named crawlers stay un-blocked on purpose: they must crawl /i/u to see its
113# X-Robots-Tag noindex (the robots.txt Noindex directive died in 2019).
114Disallow: /i/u
115
116# Wait 1 second between successive requests. See ONBOARD-2698 for details.
117Crawl-delay: 1
118
119# Independent of user agent. Links in the sitemap are full URLs using https://
120# and need to match the protocol of the sitemap.
121Sitemap: https://x.com/sitemap.xml
122Sitemap: https://x.com/sitemaps/finance0