NeuralCrawl

ChatGPT / robots.txt snapshot

← back to chatgpt.com · fetched 2026-06-26T13:23:15Z (3h ago) · HTTP 200 · 3682 bytes · sha256 bef0f070f071ef3d · raw

final URL: https://chatgpt.com/robots.txt

1# https://www.robotstxt.org/robotstxt.html
2
3User-agent: CCBot
4Disallow: /
5
6User-agent: img2dataset
7Disallow: /
8
9User-agent: Google-Extended
10Disallow: /
11
12User-agent: anthropic-ai
13Disallow: /
14
15User-agent: Claude-Web
16Disallow: /
17
18User-agent: Omgilibot
19Disallow: /
20
21User-agent: Omgili
22Disallow: /
23
24User-agent: FacebookBot
25Disallow: /
26
27User-agent: Bytespider
28Disallow: /
29
30User-agent: magpie-crawler
31Disallow: /
32
33User-agent: PerplexityBot
34Disallow: /
35
36User-agent: Perplexity‑User
37Disallow: /
38
39# General rules for all other bots
40User-agent: *
41
42# Place allows first to avoid bots skipping after Disallow: /
43# Allow exactly the homepage
44Allow: /$
45# Allow the homepage with any query parameters
46Allow: /?*
47Allow: /api/share/og/
48Allow: /g/
49Allow: /s/
50Allow: /gg/v/
51Allow: /share/
52Allow: /canvas/shared/
53Allow: /*/images
54Allow: /images
55Allow: /*/library
56Allow: /library
57Allow: /favicon.ico
58Allow: /assets/favicon
59Allow: /cdn/assets/favicon
60Allow: /cdn/assets/
61Allow: /auth/
62Allow: /gpts$
63Allow: /codex
64Allow: /*/codex
65Allow: /search$
66Allow: /backend-anon/
67Allow: /public-api/
68Allow: /sitemap.xml
69Allow: /marketing-sitemap.xml
70Allow: /images-sitemap.xml
71Allow: /writing-tools-sitemap.xml
72Allow: /football-sitemap.xml
73Allow: /100chats
74Allow: /api/public_content/
75Allow: /backend-api/public_content/
76Allow: /?ref=dotcom
77# Static Landing Pages
78Allow: /overview
79Allow: /*/overview
80Allow: /features
81Allow: /*/features
82Allow: /apps
83Allow: /*/apps
84Allow: /finances
85Allow: /*/finances
86Allow: /shopping
87Allow: /*/shopping
88Allow: /use-cases
89Allow: /*/use-cases
90Allow: /learn
91Allow: /*/learn
92Allow: /business
93Allow: /*/business
94Allow: /pricing
95Allow: /*/pricing
96Allow: /download
97Allow: /*/download
98Allow: /students
99Allow: /*/college-students
100Allow: /college-students
101Allow: /contact-sales
102Allow: /*/contact-sales
103Allow: /100chats-project
104Allow: /*/100chats-project
105Allow: /merchants
106Allow: /*/merchants
107Allow: /parent-resources
108Allow: /*/parent-resources
109Allow: /atlas
110Allow: /*/atlas
111Allow: /plans
112Allow: /*/plans
113Allow: /translate
114Allow: /*/translate
115Allow: /writing
116Allow: /*/writing
117Allow: /futures
118Allow: /*/futures
119Allow: /football
120Allow: /*/football
121# Exact locale specific homepages
122Allow: /am/$
123Allow: /ar/$
124Allow: /bg-BG/$
125Allow: /bn-BD/$
126Allow: /bs-BA/$
127Allow: /ca-ES/$
128Allow: /cs-CZ/$
129Allow: /da-DK/$
130Allow: /de-DE/$
131Allow: /el-GR/$
132Allow: /es-ES/$
133Allow: /es-419/$
134Allow: /et-EE/$
135Allow: /fi-FI/$
136Allow: /fr-FR/$
137Allow: /fr-CA/$
138Allow: /gu-IN/$
139Allow: /hi-IN/$
140Allow: /hr-HR/$
141Allow: /hu-HU/$
142Allow: /hy-AM/$
143Allow: /id-ID/$
144Allow: /is-IS/$
145Allow: /it-IT/$
146Allow: /ja-JP/$
147Allow: /ka-GE/$
148Allow: /kk/$
149Allow: /kn-IN/$
150Allow: /ko-KR/$
151Allow: /lt/$
152Allow: /lv-LV/$
153Allow: /mk-MK/$
154Allow: /ml/$
155Allow: /mn/$
156Allow: /mr-IN/$
157Allow: /ms-MY/$
158Allow: /my-MM/$
159Allow: /nb-NO/$
160Allow: /nl-NL/$
161Allow: /pa/$
162Allow: /pl-PL/$
163Allow: /pt-BR/$
164Allow: /pt-PT/$
165Allow: /ro-RO/$
166Allow: /ru-RU/$
167Allow: /sk-SK/$
168Allow: /sl-SI/$
169Allow: /so-SO/$
170Allow: /sq-AL/$
171Allow: /sr-RS/$
172Allow: /sv-SE/$
173Allow: /sw-TZ/$
174Allow: /ta-IN/$
175Allow: /te-IN/$
176Allow: /th-TH/$
177Allow: /tl/$
178Allow: /tr-TR/$
179Allow: /uk-UA/$
180Allow: /ur/$
181Allow: /vi-VN/$
182Allow: /zh-CN/$
183Allow: /zh-TW/$
184Allow: /zh-HK/$
185
186# Now block everything else
187Disallow: /
188
189# Specific disallows (redundant for some bots, but still useful for those that respect precedence)
190Disallow: /auth/logout
191Disallow: /auth/login?*
192Disallow: /backend-anon/sentinel/*
193Disallow: /backend-anon/conversation$
194Disallow: /account-link/*
195
196Sitemap: https://chatgpt.com/sitemap.xml
197Sitemap: https://chatgpt.com/marketing-sitemap.xml
198Sitemap: https://chatgpt.com/images-sitemap.xml
199Sitemap: https://chatgpt.com/writing-tools-sitemap.xml
200Sitemap: https://chatgpt.com/football-sitemap.xml