NeuralCrawl

X (Twitter) / robots.txt

← back to x.com · change detected 28d ago (2026-07-07T02:22:00Z) · modified +39 −21

What changed

Diff old (2026-06-20T01:10:30Z) → new (2026-07-07T02:22:00Z)

@@ -8,6 +8,9 @@
88  Allow: /hashtag/*?src=
99  Allow: /search?q=%23
1010  Allow: /i/api/
11+ # Same length as the /*/likes and /*/media rules; RFC 9309 breaks the tie in
12+ # favor of Allow, so /i/api/ fetches stay crawlable.
13+ Allow: /i/api/*
1114  Disallow: /*?lang=en-ss
1215  Allow: /*?lang=
1316  Disallow: /search/realtime
@@ -17,15 +20,21 @@
1720  Disallow: /*/analytics
1821  Disallow: /*/followers
1922  Disallow: /*/following
23+ Disallow: /*/verified_followers
2024 
2125  Disallow: /account/deactivated
2226  Disallow: /settings/deactivated
2327 
24- Disallow: /[_0-9a-zA-Z]+/status/[0-9]+/likes
25- Disallow: /[_0-9a-zA-Z]+/status/[0-9]+/retweets
26- Disallow: /[_0-9a-zA-Z]+/likes
27- Disallow: /[_0-9a-zA-Z]+/media
28- Disallow: /[_0-9a-zA-Z]+/photo
28+ # Only `*` and `$` are metacharacters here (RFC 9309); regex classes match
29+ # literally. /photo is anchored so /status/*/photo/1 media pages stay open.
30+ Disallow: /*/status/*/likes
31+ Disallow: /*/status/*/retweets
32+ Disallow: /*/likes
33+ Disallow: /*/likes?
34+ Disallow: /*/media
35+ Disallow: /*/media?
36+ Disallow: /*/photo$
37+ Disallow: /*/photo?
2938  Allow: /*?
3039 
3140  User-agent: Bingbot
@@ -36,6 +45,7 @@
3645  Allow: /hashtag/*?src=
3746  Allow: /search?q=%23
3847  Allow: /i/api/
48+ Allow: /i/api/*
3949  Disallow: /*?lang=en-ss
4050  Allow: /*?lang=
4151  Disallow: /search/realtime
@@ -45,15 +55,19 @@
4555  Disallow: /*/analytics
4656  Disallow: /*/followers
4757  Disallow: /*/following
58+ Disallow: /*/verified_followers
4859 
4960  Disallow: /account/deactivated
5061  Disallow: /settings/deactivated
5162 
52- Disallow: /[_0-9a-zA-Z]+/status/[0-9]+/likes
53- Disallow: /[_0-9a-zA-Z]+/status/[0-9]+/retweets
54- Disallow: /[_0-9a-zA-Z]+/likes
55- Disallow: /[_0-9a-zA-Z]+/media
56- Disallow: /[_0-9a-zA-Z]+/photo
63+ Disallow: /*/status/*/likes
64+ Disallow: /*/status/*/retweets
65+ Disallow: /*/likes
66+ Disallow: /*/likes?
67+ Disallow: /*/media
68+ Disallow: /*/media?
69+ Disallow: /*/photo$
70+ Disallow: /*/photo?
5771  Allow: /*?
5872 
5973  User-agent: facebookexternalhit
@@ -66,6 +80,7 @@
6680  Allow: /search?q=%23
6781  Allow: /search?*cashtagRestId=
6882  Allow: /i/api/
83+ Allow: /i/api/*
6984  Disallow: /search/realtime
7085  Disallow: /search/users
7186  Disallow: /search/*/grid
@@ -73,15 +88,19 @@
7388  Disallow: /*?
7489  Disallow: /*/followers
7590  Disallow: /*/following
91+ Disallow: /*/verified_followers
7692 
7793  Disallow: /account/deactivated
7894  Disallow: /settings/deactivated
7995 
80- Disallow: /[_0-9a-zA-Z]+/status/[0-9]+/likes
81- Disallow: /[_0-9a-zA-Z]+/status/[0-9]+/retweets
82- Disallow: /[_0-9a-zA-Z]+/likes
83- Disallow: /[_0-9a-zA-Z]+/media
84- Disallow: /[_0-9a-zA-Z]+/photo
96+ Disallow: /*/status/*/likes
97+ Disallow: /*/status/*/retweets
98+ Disallow: /*/likes
99+ Disallow: /*/likes?
100+ Disallow: /*/media
101+ Disallow: /*/media?
102+ Disallow: /*/photo$
103+ Disallow: /*/photo?
85104 
86105  User-Agent: Google-Extended
87106  Disallow: *
@@ -98,15 +117,14 @@
98117  Disallow: /
99118 
100119 
101- # WHAT-4882 - Block indexing of links in notification emails. This applies to all bots.
102- # =====================================================================================
120+ # WHAT-4882 - Keep notification-email links (/i/u) out of search results.
121+ # Named crawlers stay un-blocked on purpose: they must crawl /i/u to see its
122+ # X-Robots-Tag noindex (the robots.txt Noindex directive died in 2019).
103123  Disallow: /i/u
104- Noindex: /i/u
105124 
106125  # Wait 1 second between successive requests. See ONBOARD-2698 for details.
107126  Crawl-delay: 1
108127 
109- # Independent of user agent. Links in the sitemap are full URLs using https:// and need to match
110- # the protocol of the sitemap.
128+ # Independent of user agent. Links in the sitemap are full URLs using https://
129+ # and need to match the protocol of the sitemap.
111130  Sitemap: https://x.com/sitemap.xml
112- Sitemap: https://twitter.com/sitemap.xml