NeuralCrawl

X (Twitter) / robots.txt snapshot

← back to x.com · fetched 2026-10-30T11:48:22Z (8d ago) · HTTP 200 · 2691 bytes · sha256 8343de4a7e647615 · raw

final URL: https://x.com/robots.txt

1# Google / Bing Search Engine Robots
2# ==================================
3# Shared group so Googlebot and Bingbot always get the same rules.
4User-agent: Googlebot
5User-agent: Bingbot
6
7Allow: /*?s=
8Allow: /*?t=
9Allow: /*?ref_src=
10Allow: /hashtag/*?src=
11Allow: /search?q=%23
12Allow: /i/api/
13# Same length as the /*/likes and /*/media rules; RFC 9309 breaks the tie in
14# favor of Allow, so /i/api/ fetches stay crawlable.
15Allow: /i/api/*
16Disallow: /*?lang=en-ss
17Allow: /*?lang=
18Disallow: /search/realtime
19Disallow: /search/users
20Disallow: /search/*/grid
21
22Disallow: /*/analytics
23Disallow: /*/followers
24Disallow: /*/following
25Disallow: /*/verified_followers
26
27Disallow: /account/deactivated
28Disallow: /settings/deactivated
29
30# Only `*` and `$` are metacharacters here (RFC 9309); regex classes match
31# literally. /photo is anchored so /status/*/photo/1 media pages stay open.
32Disallow: /*/likes
33Disallow: /*/likes?
34Disallow: /*/retweets
35Disallow: /*/retweets?
36Disallow: /*/media
37Disallow: /*/media?
38Disallow: /*/photo$
39Disallow: /*/photo?
40Allow: /*?
41
42User-agent: facebookexternalhit
43
44Allow: /*?lang=
45Allow: /*?s=
46Allow: /*&s=
47Allow: /*?t=
48Allow: /*?ref_src=
49Allow: /hashtag/*?src=
50Allow: /search?*cashtagRestId=
51Allow: /i/api/
52Allow: /i/api/*
53Disallow: /search?q=
54Disallow: /search/realtime
55Disallow: /search/users
56Disallow: /search/*/grid
57
58Disallow: /*?
59Disallow: /*/followers
60Disallow: /*/following
61Disallow: /*/verified_followers
62
63Disallow: /account/deactivated
64Disallow: /settings/deactivated
65
66Disallow: /*/likes
67Disallow: /*/likes?
68Disallow: /*/retweets
69Disallow: /*/retweets?
70Disallow: /*/media
71Disallow: /*/media?
72Disallow: /*/photo$
73Disallow: /*/photo?
74
75User-Agent: Google-Extended
76Disallow: *
77
78User-Agent: FacebookBot
79Disallow: *
80
81User-agent: Discordbot
82Disallow: *
83
84# Meta AI / product crawlers (product tokens; versions like /1.1 are ignored
85# under RFC 9309 token matching).
86User-agent: meta-webindexer
87Disallow: *
88
89User-agent: meta-externalagent
90Disallow: *
91
92User-agent: meta-externalads
93Disallow: *
94
95User-agent: meta-externalfetcher
96Disallow: *
97
98# Every bot that might possibly read and respect this file
99# ========================================================
100User-agent: *
101Disallow: /
102
103
104# WHAT-4882 - Keep notification-email links (/i/u) out of search results.
105# Named crawlers stay un-blocked on purpose: they must crawl /i/u to see its
106# X-Robots-Tag noindex (the robots.txt Noindex directive died in 2019).
107Disallow: /i/u
108
109# Wait 1 second between successive requests. See ONBOARD-2698 for details.
110Crawl-delay: 1
111
112# Independent of user agent. Links in the sitemap are full URLs using https://
113# and need to match the protocol of the sitemap.
114Sitemap: https://x.com/sitemap.xml