NeuralCrawl

X (Twitter) / robots.txt snapshot

← back to x.com · fetched 2026-08-13T21:57:57Z (1mo ago) · HTTP 200 · 2706 bytes · sha256 08c99bcf1a79c3da · raw

final URL: https://x.com/robots.txt

1# Google / Bing Search Engine Robots
2# ==================================
3# Shared group so Googlebot and Bingbot always get the same rules.
4User-agent: Googlebot
5User-agent: Bingbot
6
7Allow: /*?s=
8Allow: /*?t=
9Allow: /*?ref_src=
10Allow: /hashtag/*?src=
11Allow: /search?q=%23
12Allow: /i/api/
13# Same length as the /*/likes and /*/media rules; RFC 9309 breaks the tie in
14# favor of Allow, so /i/api/ fetches stay crawlable.
15Allow: /i/api/*
16Disallow: /*?lang=en-ss
17Allow: /*?lang=
18Disallow: /search/realtime
19Disallow: /search/users
20Disallow: /search/*/grid
21
22Disallow: /*/analytics
23Disallow: /*/followers
24Disallow: /*/following
25Disallow: /*/verified_followers
26
27Disallow: /account/deactivated
28Disallow: /settings/deactivated
29
30# Only `*` and `$` are metacharacters here (RFC 9309); regex classes match
31# literally. /photo is anchored so /status/*/photo/1 media pages stay open.
32Disallow: /*/status/*/likes
33Disallow: /*/status/*/retweets
34Disallow: /*/likes
35Disallow: /*/likes?
36Disallow: /*/media
37Disallow: /*/media?
38Disallow: /*/photo$
39Disallow: /*/photo?
40Allow: /*?
41
42User-agent: facebookexternalhit
43
44Allow: /*?lang=
45Allow: /*?s=
46Allow: /*?t=
47Allow: /*?ref_src=
48Allow: /hashtag/*?src=
49Allow: /search?*cashtagRestId=
50Allow: /i/api/
51Allow: /i/api/*
52Disallow: /search?q=
53Disallow: /search/realtime
54Disallow: /search/users
55Disallow: /search/*/grid
56
57Disallow: /*?
58Disallow: /*/followers
59Disallow: /*/following
60Disallow: /*/verified_followers
61
62Disallow: /account/deactivated
63Disallow: /settings/deactivated
64
65Disallow: /*/status/*/likes
66Disallow: /*/status/*/retweets
67Disallow: /*/likes
68Disallow: /*/likes?
69Disallow: /*/media
70Disallow: /*/media?
71Disallow: /*/photo$
72Disallow: /*/photo?
73
74User-Agent: Google-Extended
75Disallow: *
76
77User-Agent: FacebookBot
78Disallow: *
79
80User-agent: Discordbot
81Disallow: *
82
83# Meta AI / product crawlers (product tokens; versions like /1.1 are ignored
84# under RFC 9309 token matching).
85User-agent: meta-webindexer
86Disallow: *
87
88User-agent: meta-externalagent
89Disallow: *
90
91User-agent: meta-externalads
92Disallow: *
93
94User-agent: meta-externalfetcher
95Disallow: *
96
97# Every bot that might possibly read and respect this file
98# ========================================================
99User-agent: *
100Disallow: /
101
102
103# WHAT-4882 - Keep notification-email links (/i/u) out of search results.
104# Named crawlers stay un-blocked on purpose: they must crawl /i/u to see its
105# X-Robots-Tag noindex (the robots.txt Noindex directive died in 2019).
106Disallow: /i/u
107
108# Wait 1 second between successive requests. See ONBOARD-2698 for details.
109Crawl-delay: 1
110
111# Independent of user agent. Links in the sitemap are full URLs using https://
112# and need to match the protocol of the sitemap.
113Sitemap: https://x.com/sitemap.xml