NeuralCrawl

ChatGPT / robots.txt snapshot

← back to chatgpt.com · fetched 2026-10-01T01:22:44Z (1mo ago) · HTTP 200 · 4388 bytes · sha256 a6628f6c9c3b373e · raw

final URL: https://chatgpt.com/robots.txt

1# https://www.robotstxt.org/robotstxt.html
2
3User-agent: CCBot
4Disallow: /
5
6User-agent: img2dataset
7Disallow: /
8
9User-agent: Google-Extended
10Disallow: /
11
12# Prevent shared pages and GPTs from being used as Google Ads destinations
13User-agent: AdsBot-Google
14User-agent: AdsBot-Google-Mobile
15Disallow: /s/
16Disallow: /g/
17
18User-agent: anthropic-ai
19Disallow: /
20
21User-agent: Claude-Web
22Disallow: /
23
24User-agent: Omgilibot
25Disallow: /
26
27User-agent: Omgili
28Disallow: /
29
30User-agent: FacebookBot
31Disallow: /
32
33User-agent: Bytespider
34Disallow: /
35
36User-agent: magpie-crawler
37Disallow: /
38
39User-agent: PerplexityBot
40Disallow: /
41
42User-agent: Perplexity‑User
43Disallow: /
44
45# General rules for all other bots
46User-agent: *
47
48# Place allows first to avoid bots skipping after Disallow: /
49# Allow exactly the homepage
50Allow: /$
51# Allow the homepage with any query parameters
52Allow: /?*
53Allow: /api/share/og/
54Allow: /g/
55Allow: /s/
56Allow: /gg/v/
57Allow: /m/
58Allow: /share/
59Allow: /maps$
60Allow: /maps/
61Allow: /*/maps$
62Allow: /*/maps/
63Allow: /canvas/shared/
64Allow: /*/images
65Allow: /images
66Allow: /*/library
67Allow: /library
68Allow: /favicon.ico
69Allow: /assets/favicon
70Allow: /cdn/assets/favicon
71Allow: /cdn/assets/
72Allow: /auth/
73Allow: /gpts$
74Allow: /codex
75Allow: /*/codex
76Allow: /search$
77Allow: /backend-anon/
78Allow: /public-api/
79Allow: /sitemap.xml
80Allow: /marketing-sitemap.xml
81Allow: /images-sitemap.xml
82Allow: /writing-tools-sitemap.xml
83Allow: /football-sitemap.xml
84Allow: /acquisition-landing-pages-sitemap.xml
85Allow: /100chats
86Allow: /api/public_content/
87Allow: /backend-api/public_content/
88Allow: /?ref=dotcom
89# Static Landing Pages
90Allow: /overview
91Allow: /*/overview
92Allow: /features
93Allow: /*/features
94Allow: /health$
95Allow: /health/$
96Allow: /*/health$
97Allow: /*/health/$
98Allow: /voice$
99Allow: /voice/$
100Allow: /*/voice$
101Allow: /*/voice/$
102Allow: /apps
103Allow: /*/apps
104Allow: /finances
105Allow: /*/finances
106Allow: /shopping
107Allow: /*/shopping
108Allow: /use-cases
109Allow: /*/use-cases
110Allow: /learn
111Allow: /*/learn
112Allow: /business
113Allow: /*/business
114Allow: /pricing
115Allow: /*/pricing
116Allow: /download
117Allow: /*/download
118Allow: /students
119Allow: /students/2026
120Allow: /*/college-students
121Allow: /college-students
122Allow: /contact-sales
123Allow: /*/contact-sales
124Allow: /100chats-project
125Allow: /*/100chats-project
126Allow: /merchants
127Allow: /*/merchants
128Allow: /parent-resources
129Allow: /*/parent-resources
130Allow: /plans
131Allow: /*/plans
132Allow: /plugins
133Allow: /translate
134Allow: /*/translate
135Allow: /writing
136Allow: /*/writing
137Allow: /ask/
138Allow: /futures
139Allow: /*/futures
140Allow: /football
141Allow: /*/football
142Allow: /remote
143Allow: /*/remote
144Allow: /work
145Allow: /*/work
146Allow: /dots
147Allow: /*/dots
148Allow: /sites
149Allow: /*/sites
150Allow: /space
151Allow: /*/space
152Allow: /import-to-chatgpt
153Allow: /*/import-to-chatgpt
154# Exact locale specific homepages
155Allow: /am/$
156Allow: /ar/$
157Allow: /bg-BG/$
158Allow: /bn-BD/$
159Allow: /bs-BA/$
160Allow: /ca-ES/$
161Allow: /cs-CZ/$
162Allow: /da-DK/$
163Allow: /de-DE/$
164Allow: /el-GR/$
165Allow: /es-ES/$
166Allow: /es-419/$
167Allow: /et-EE/$
168Allow: /fi-FI/$
169Allow: /fr-FR/$
170Allow: /fr-CA/$
171Allow: /gu-IN/$
172Allow: /hi-IN/$
173Allow: /hr-HR/$
174Allow: /hu-HU/$
175Allow: /hy-AM/$
176Allow: /id-ID/$
177Allow: /is-IS/$
178Allow: /it-IT/$
179Allow: /ja-JP/$
180Allow: /ka-GE/$
181Allow: /kk/$
182Allow: /kn-IN/$
183Allow: /ko-KR/$
184Allow: /lt/$
185Allow: /lv-LV/$
186Allow: /mk-MK/$
187Allow: /ml/$
188Allow: /mn/$
189Allow: /mr-IN/$
190Allow: /ms-MY/$
191Allow: /my-MM/$
192Allow: /nb-NO/$
193Allow: /nl-NL/$
194Allow: /pa/$
195Allow: /pl-PL/$
196Allow: /pt-BR/$
197Allow: /pt-PT/$
198Allow: /ro-RO/$
199Allow: /ru-RU/$
200Allow: /sk-SK/$
201Allow: /sl-SI/$
202Allow: /so-SO/$
203Allow: /sq-AL/$
204Allow: /sr-RS/$
205Allow: /sv-SE/$
206Allow: /sw-TZ/$
207Allow: /ta-IN/$
208Allow: /te-IN/$
209Allow: /th-TH/$
210Allow: /tl/$
211Allow: /tr-TR/$
212Allow: /uk-UA/$
213Allow: /ur/$
214Allow: /vi-VN/$
215Allow: /zh-CN/$
216Allow: /zh-TW/$
217Allow: /zh-HK/$
218
219# Now block everything else
220Disallow: /
221
222# Specific disallows (redundant for some bots, but still useful for those that respect precedence)
223Disallow: /auth/logout
224Disallow: /auth/login?*
225Disallow: /backend-anon/sentinel/*
226Disallow: /backend-anon/conversation$
227Disallow: /account-link/*
228
229Sitemap: https://chatgpt.com/sitemap.xml
230Sitemap: https://chatgpt.com/marketing-sitemap.xml
231Sitemap: https://chatgpt.com/images-sitemap.xml
232Sitemap: https://chatgpt.com/writing-tools-sitemap.xml
233Sitemap: https://chatgpt.com/football-sitemap.xml
234Sitemap: https://chatgpt.com/acquisition-landing-pages-sitemap.xml