NeuralCrawl

X (Twitter) / robots.txt snapshot

← back to x.com · fetched 2026-06-20T01:10:30Z (18h ago) · HTTP 200 · 2545 bytes · sha256 2e162da07003732f · raw

final URL: https://x.com/robots.txt

1# Google Search Engine Robot
2# ==========================
3User-agent: Googlebot
4
5Allow: /*?s=
6Allow: /*?t=
7Allow: /*?ref_src=
8Allow: /hashtag/*?src=
9Allow: /search?q=%23
10Allow: /i/api/
11Disallow: /*?lang=en-ss
12Allow: /*?lang=
13Disallow: /search/realtime
14Disallow: /search/users
15Disallow: /search/*/grid
16
17Disallow: /*/analytics
18Disallow: /*/followers
19Disallow: /*/following
20
21Disallow: /account/deactivated
22Disallow: /settings/deactivated
23
24Disallow: /[_0-9a-zA-Z]+/status/[0-9]+/likes
25Disallow: /[_0-9a-zA-Z]+/status/[0-9]+/retweets
26Disallow: /[_0-9a-zA-Z]+/likes
27Disallow: /[_0-9a-zA-Z]+/media
28Disallow: /[_0-9a-zA-Z]+/photo
29Allow: /*?
30
31User-agent: Bingbot
32
33Allow: /*?s=
34Allow: /*?t=
35Allow: /*?ref_src=
36Allow: /hashtag/*?src=
37Allow: /search?q=%23
38Allow: /i/api/
39Disallow: /*?lang=en-ss
40Allow: /*?lang=
41Disallow: /search/realtime
42Disallow: /search/users
43Disallow: /search/*/grid
44
45Disallow: /*/analytics
46Disallow: /*/followers
47Disallow: /*/following
48
49Disallow: /account/deactivated
50Disallow: /settings/deactivated
51
52Disallow: /[_0-9a-zA-Z]+/status/[0-9]+/likes
53Disallow: /[_0-9a-zA-Z]+/status/[0-9]+/retweets
54Disallow: /[_0-9a-zA-Z]+/likes
55Disallow: /[_0-9a-zA-Z]+/media
56Disallow: /[_0-9a-zA-Z]+/photo
57Allow: /*?
58
59User-agent: facebookexternalhit
60
61Allow: /*?lang=
62Allow: /*?s=
63Allow: /*?t=
64Allow: /*?ref_src=
65Allow: /hashtag/*?src=
66Allow: /search?q=%23
67Allow: /search?*cashtagRestId=
68Allow: /i/api/
69Disallow: /search/realtime
70Disallow: /search/users
71Disallow: /search/*/grid
72
73Disallow: /*?
74Disallow: /*/followers
75Disallow: /*/following
76
77Disallow: /account/deactivated
78Disallow: /settings/deactivated
79
80Disallow: /[_0-9a-zA-Z]+/status/[0-9]+/likes
81Disallow: /[_0-9a-zA-Z]+/status/[0-9]+/retweets
82Disallow: /[_0-9a-zA-Z]+/likes
83Disallow: /[_0-9a-zA-Z]+/media
84Disallow: /[_0-9a-zA-Z]+/photo
85
86User-Agent: Google-Extended
87Disallow: *
88
89User-Agent: FacebookBot
90Disallow: *
91
92User-agent: Discordbot
93Disallow: *
94
95# Every bot that might possibly read and respect this file
96# ========================================================
97User-agent: *
98Disallow: /
99
100
101# WHAT-4882 - Block indexing of links in notification emails. This applies to all bots.
102# =====================================================================================
103Disallow: /i/u
104Noindex: /i/u
105
106# Wait 1 second between successive requests. See ONBOARD-2698 for details.
107Crawl-delay: 1
108
109# Independent of user agent. Links in the sitemap are full URLs using https:// and need to match
110# the protocol of the sitemap.
111Sitemap: https://x.com/sitemap.xml
112Sitemap: https://twitter.com/sitemap.xml