NeuralCrawl

ChatGPT / robots.txt snapshot

← back to chatgpt.com · fetched 2026-08-15T22:38:38Z (2mo ago) · HTTP 200 · 4248 bytes · sha256 bf2737d7d596e36a · raw

final URL: https://chatgpt.com/robots.txt

1# https://www.robotstxt.org/robotstxt.html
2
3User-agent: CCBot
4Disallow: /
5
6User-agent: img2dataset
7Disallow: /
8
9User-agent: Google-Extended
10Disallow: /
11
12# Prevent shared pages and GPTs from being used as Google Ads destinations
13User-agent: AdsBot-Google
14User-agent: AdsBot-Google-Mobile
15Disallow: /s/
16Disallow: /g/
17
18User-agent: anthropic-ai
19Disallow: /
20
21User-agent: Claude-Web
22Disallow: /
23
24User-agent: Omgilibot
25Disallow: /
26
27User-agent: Omgili
28Disallow: /
29
30User-agent: FacebookBot
31Disallow: /
32
33User-agent: Bytespider
34Disallow: /
35
36User-agent: magpie-crawler
37Disallow: /
38
39User-agent: PerplexityBot
40Disallow: /
41
42User-agent: Perplexity‑User
43Disallow: /
44
45# General rules for all other bots
46User-agent: *
47
48# Place allows first to avoid bots skipping after Disallow: /
49# Allow exactly the homepage
50Allow: /$
51# Allow the homepage with any query parameters
52Allow: /?*
53Allow: /api/share/og/
54Allow: /g/
55Allow: /s/
56Allow: /gg/v/
57Allow: /m/
58Allow: /share/
59Allow: /canvas/shared/
60Allow: /*/images
61Allow: /images
62Allow: /*/library
63Allow: /library
64Allow: /favicon.ico
65Allow: /assets/favicon
66Allow: /cdn/assets/favicon
67Allow: /cdn/assets/
68Allow: /auth/
69Allow: /gpts$
70Allow: /codex
71Allow: /*/codex
72Allow: /search$
73Allow: /backend-anon/
74Allow: /public-api/
75Allow: /sitemap.xml
76Allow: /marketing-sitemap.xml
77Allow: /images-sitemap.xml
78Allow: /writing-tools-sitemap.xml
79Allow: /football-sitemap.xml
80Allow: /acquisition-landing-pages-sitemap.xml
81Allow: /100chats
82Allow: /api/public_content/
83Allow: /backend-api/public_content/
84Allow: /?ref=dotcom
85# Static Landing Pages
86Allow: /overview
87Allow: /*/overview
88Allow: /features
89Allow: /*/features
90Allow: /health$
91Allow: /health/$
92Allow: /*/health$
93Allow: /*/health/$
94Allow: /voice$
95Allow: /voice/$
96Allow: /*/voice$
97Allow: /*/voice/$
98Allow: /apps
99Allow: /*/apps
100Allow: /finances
101Allow: /*/finances
102Allow: /shopping
103Allow: /*/shopping
104Allow: /use-cases
105Allow: /*/use-cases
106Allow: /learn
107Allow: /*/learn
108Allow: /business
109Allow: /*/business
110Allow: /pricing
111Allow: /*/pricing
112Allow: /download
113Allow: /*/download
114Allow: /students
115Allow: /*/college-students
116Allow: /college-students
117Allow: /contact-sales
118Allow: /*/contact-sales
119Allow: /100chats-project
120Allow: /*/100chats-project
121Allow: /merchants
122Allow: /*/merchants
123Allow: /parent-resources
124Allow: /*/parent-resources
125Allow: /atlas
126Allow: /*/atlas
127Allow: /plans
128Allow: /*/plans
129Allow: /plugins
130Allow: /translate
131Allow: /*/translate
132Allow: /writing
133Allow: /*/writing
134Allow: /ask/
135Allow: /futures
136Allow: /*/futures
137Allow: /football
138Allow: /*/football
139Allow: /remote
140Allow: /*/remote
141Allow: /work
142Allow: /*/work
143Allow: /import-to-chatgpt
144Allow: /*/import-to-chatgpt
145# Exact locale specific homepages
146Allow: /am/$
147Allow: /ar/$
148Allow: /bg-BG/$
149Allow: /bn-BD/$
150Allow: /bs-BA/$
151Allow: /ca-ES/$
152Allow: /cs-CZ/$
153Allow: /da-DK/$
154Allow: /de-DE/$
155Allow: /el-GR/$
156Allow: /es-ES/$
157Allow: /es-419/$
158Allow: /et-EE/$
159Allow: /fi-FI/$
160Allow: /fr-FR/$
161Allow: /fr-CA/$
162Allow: /gu-IN/$
163Allow: /hi-IN/$
164Allow: /hr-HR/$
165Allow: /hu-HU/$
166Allow: /hy-AM/$
167Allow: /id-ID/$
168Allow: /is-IS/$
169Allow: /it-IT/$
170Allow: /ja-JP/$
171Allow: /ka-GE/$
172Allow: /kk/$
173Allow: /kn-IN/$
174Allow: /ko-KR/$
175Allow: /lt/$
176Allow: /lv-LV/$
177Allow: /mk-MK/$
178Allow: /ml/$
179Allow: /mn/$
180Allow: /mr-IN/$
181Allow: /ms-MY/$
182Allow: /my-MM/$
183Allow: /nb-NO/$
184Allow: /nl-NL/$
185Allow: /pa/$
186Allow: /pl-PL/$
187Allow: /pt-BR/$
188Allow: /pt-PT/$
189Allow: /ro-RO/$
190Allow: /ru-RU/$
191Allow: /sk-SK/$
192Allow: /sl-SI/$
193Allow: /so-SO/$
194Allow: /sq-AL/$
195Allow: /sr-RS/$
196Allow: /sv-SE/$
197Allow: /sw-TZ/$
198Allow: /ta-IN/$
199Allow: /te-IN/$
200Allow: /th-TH/$
201Allow: /tl/$
202Allow: /tr-TR/$
203Allow: /uk-UA/$
204Allow: /ur/$
205Allow: /vi-VN/$
206Allow: /zh-CN/$
207Allow: /zh-TW/$
208Allow: /zh-HK/$
209
210# Now block everything else
211Disallow: /
212
213# Specific disallows (redundant for some bots, but still useful for those that respect precedence)
214Disallow: /auth/logout
215Disallow: /auth/login?*
216Disallow: /backend-anon/sentinel/*
217Disallow: /backend-anon/conversation$
218Disallow: /account-link/*
219
220Sitemap: https://chatgpt.com/sitemap.xml
221Sitemap: https://chatgpt.com/marketing-sitemap.xml
222Sitemap: https://chatgpt.com/images-sitemap.xml
223Sitemap: https://chatgpt.com/writing-tools-sitemap.xml
224Sitemap: https://chatgpt.com/football-sitemap.xml
225Sitemap: https://chatgpt.com/acquisition-landing-pages-sitemap.xml