CBS News / robots.txt snapshot

← back to cbsnews.com · fetched 2026-06-20T11:49:13Z (7h ago) · HTTP 200 · 950 bytes · sha256 3183f29f832838e6 · raw

final URL: manual:file

1	# www.robotstxt.org/
2	# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449
3
4	User-agent: *
5
6	Disallow: /search/
7	Disallow: /index.php/*
8	Disallow: /election-results-data/*
9	Disallow: /common
10	Disallow: /htdocs/*
11	Disallow: /user/login*
12	Disallow: /app/?*
13	Disallow: /network
14	Disallow: /assets
15	Disallow: /stories
16	Disallow: /sections
17	Disallow: */feed/
18
19	# PER CBS-N ENG FINAL ROUTES DOC
20
21	Disallow: /1318
22	Disallow: /1319
23	Disallow: /1328
24	Disallow: /1344
25	Disallow: /1770
26	Disallow: /2240
27	Disallow: /2300
28	Disallow: /2303
29	Disallow: /2304
30	Disallow: /2305
31	Disallow: /2306
32	Disallow: /2307
33	Disallow: /2308
34	Disallow: /2332
35	Disallow: /2741
36	Disallow: /2994
37	Disallow: /8301
38	Disallow: /8324
39	Disallow: /8601
40	Disallow: /8614
41	Disallow: /8618
42	Disallow: /9742
43	Disallow: /9744
44
45	Sitemap: https://www.cbsnews.com/xml-sitemap/index.xml
46
47	User-agent: GPTBot
48	Disallow: /
49	User-agent: MAZBot
50	Disallow: /
51	User-agent: panscient.com
52	Disallow: /
53	User-agent: proximic
54	Disallow: