NeuralCrawl

ChatGPT / robots.txt snapshot

← back to chatgpt.com · fetched 2026-08-07T01:28:26Z (3mo ago) · HTTP 200 · 4246 bytes · sha256 efcfb1150524df95 · raw

final URL: https://chatgpt.com/robots.txt

1# https://www.robotstxt.org/robotstxt.html
2
3User-agent: CCBot
4Disallow: /
5
6User-agent: img2dataset
7Disallow: /
8
9User-agent: Google-Extended
10Disallow: /
11
12# Prevent shared code blocks and GPTs from being used as Google Ads destinations
13User-agent: AdsBot-Google
14User-agent: AdsBot-Google-Mobile
15Disallow: /s/cb_
16Disallow: /g/
17
18User-agent: anthropic-ai
19Disallow: /
20
21User-agent: Claude-Web
22Disallow: /
23
24User-agent: Omgilibot
25Disallow: /
26
27User-agent: Omgili
28Disallow: /
29
30User-agent: FacebookBot
31Disallow: /
32
33User-agent: Bytespider
34Disallow: /
35
36User-agent: magpie-crawler
37Disallow: /
38
39User-agent: PerplexityBot
40Disallow: /
41
42User-agent: Perplexity‑User
43Disallow: /
44
45# General rules for all other bots
46User-agent: *
47
48# Place allows first to avoid bots skipping after Disallow: /
49# Allow exactly the homepage
50Allow: /$
51# Allow the homepage with any query parameters
52Allow: /?*
53Allow: /api/share/og/
54Allow: /g/
55Allow: /s/
56Allow: /gg/v/
57Allow: /share/
58Allow: /canvas/shared/
59Allow: /*/images
60Allow: /images
61Allow: /*/library
62Allow: /library
63Allow: /favicon.ico
64Allow: /assets/favicon
65Allow: /cdn/assets/favicon
66Allow: /cdn/assets/
67Allow: /auth/
68Allow: /gpts$
69Allow: /codex
70Allow: /*/codex
71Allow: /search$
72Allow: /backend-anon/
73Allow: /public-api/
74Allow: /sitemap.xml
75Allow: /marketing-sitemap.xml
76Allow: /images-sitemap.xml
77Allow: /writing-tools-sitemap.xml
78Allow: /football-sitemap.xml
79Allow: /acquisition-landing-pages-sitemap.xml
80Allow: /100chats
81Allow: /api/public_content/
82Allow: /backend-api/public_content/
83Allow: /?ref=dotcom
84# Static Landing Pages
85Allow: /overview
86Allow: /*/overview
87Allow: /features
88Allow: /*/features
89Allow: /health$
90Allow: /health/$
91Allow: /*/health$
92Allow: /*/health/$
93Allow: /voice$
94Allow: /voice/$
95Allow: /*/voice$
96Allow: /*/voice/$
97Allow: /apps
98Allow: /*/apps
99Allow: /finances
100Allow: /*/finances
101Allow: /shopping
102Allow: /*/shopping
103Allow: /use-cases
104Allow: /*/use-cases
105Allow: /learn
106Allow: /*/learn
107Allow: /business
108Allow: /*/business
109Allow: /pricing
110Allow: /*/pricing
111Allow: /download
112Allow: /*/download
113Allow: /students
114Allow: /*/college-students
115Allow: /college-students
116Allow: /contact-sales
117Allow: /*/contact-sales
118Allow: /100chats-project
119Allow: /*/100chats-project
120Allow: /merchants
121Allow: /*/merchants
122Allow: /parent-resources
123Allow: /*/parent-resources
124Allow: /atlas
125Allow: /*/atlas
126Allow: /plans
127Allow: /*/plans
128Allow: /plugins
129Allow: /translate
130Allow: /*/translate
131Allow: /writing
132Allow: /*/writing
133Allow: /ask/
134Allow: /futures
135Allow: /*/futures
136Allow: /football
137Allow: /*/football
138Allow: /remote
139Allow: /*/remote
140Allow: /work
141Allow: /*/work
142Allow: /import-to-chatgpt
143Allow: /*/import-to-chatgpt
144# Exact locale specific homepages
145Allow: /am/$
146Allow: /ar/$
147Allow: /bg-BG/$
148Allow: /bn-BD/$
149Allow: /bs-BA/$
150Allow: /ca-ES/$
151Allow: /cs-CZ/$
152Allow: /da-DK/$
153Allow: /de-DE/$
154Allow: /el-GR/$
155Allow: /es-ES/$
156Allow: /es-419/$
157Allow: /et-EE/$
158Allow: /fi-FI/$
159Allow: /fr-FR/$
160Allow: /fr-CA/$
161Allow: /gu-IN/$
162Allow: /hi-IN/$
163Allow: /hr-HR/$
164Allow: /hu-HU/$
165Allow: /hy-AM/$
166Allow: /id-ID/$
167Allow: /is-IS/$
168Allow: /it-IT/$
169Allow: /ja-JP/$
170Allow: /ka-GE/$
171Allow: /kk/$
172Allow: /kn-IN/$
173Allow: /ko-KR/$
174Allow: /lt/$
175Allow: /lv-LV/$
176Allow: /mk-MK/$
177Allow: /ml/$
178Allow: /mn/$
179Allow: /mr-IN/$
180Allow: /ms-MY/$
181Allow: /my-MM/$
182Allow: /nb-NO/$
183Allow: /nl-NL/$
184Allow: /pa/$
185Allow: /pl-PL/$
186Allow: /pt-BR/$
187Allow: /pt-PT/$
188Allow: /ro-RO/$
189Allow: /ru-RU/$
190Allow: /sk-SK/$
191Allow: /sl-SI/$
192Allow: /so-SO/$
193Allow: /sq-AL/$
194Allow: /sr-RS/$
195Allow: /sv-SE/$
196Allow: /sw-TZ/$
197Allow: /ta-IN/$
198Allow: /te-IN/$
199Allow: /th-TH/$
200Allow: /tl/$
201Allow: /tr-TR/$
202Allow: /uk-UA/$
203Allow: /ur/$
204Allow: /vi-VN/$
205Allow: /zh-CN/$
206Allow: /zh-TW/$
207Allow: /zh-HK/$
208
209# Now block everything else
210Disallow: /
211
212# Specific disallows (redundant for some bots, but still useful for those that respect precedence)
213Disallow: /auth/logout
214Disallow: /auth/login?*
215Disallow: /backend-anon/sentinel/*
216Disallow: /backend-anon/conversation$
217Disallow: /account-link/*
218
219Sitemap: https://chatgpt.com/sitemap.xml
220Sitemap: https://chatgpt.com/marketing-sitemap.xml
221Sitemap: https://chatgpt.com/images-sitemap.xml
222Sitemap: https://chatgpt.com/writing-tools-sitemap.xml
223Sitemap: https://chatgpt.com/football-sitemap.xml
224Sitemap: https://chatgpt.com/acquisition-landing-pages-sitemap.xml