NeuralCrawl

S&P Global / robots.txt snapshot

← back to spglobal.com · fetched 2026-07-01T06:18:40Z (1mo ago) · HTTP 200 · 9766 bytes · sha256 2516fd6aa4dc5afb · raw

final URL: https://www.spglobal.com/robots.txt

1User-agent: *
2Allow: /spdji/*/multimedia-center/?p
3Allow: /sustainable1/en/scores/results?cid=*
4Disallow: /en-test/*
5Disallow: /ratings/en-test/*
6Disallow: /market-intelligence/en-test/*
7Disallow: /commodity-insights/en-test/*
8Disallow: /sustainable1/en-test/*
9Disallow: /en/homepage-v2
10Disallow: /en/search
11Disallow: /market-intelligence/en/search
12Disallow: /ratings/en/search
13Disallow: prod.aws.platts.com
14Disallow: prod-us.aws.platts.com
15Disallow: prod-us1.aws.platts.com
16Disallow: prod-us2.aws.platts.com
17Disallow: prod-us3.aws.platts.com
18Disallow: prod-uk.aws.platts.com
19Disallow: prod-uk1.aws.platts.com
20Disallow: prod-uk2.aws.platts.com
21Disallow: prod-uk3.aws.platts.com
22Disallow: prod-ap.aws.platts.com
23Disallow: prod-ap1.aws.platts.com
24Disallow: prod-ap2.aws.platts.com
25Disallow: /platts/en/our-methodology/corrections/*
26Disallow: /commodity-insights/en/our-methodology/subscriber-notes/*
27Disallow: /commodity-insights/en/our-methodology/price-symbols/*
28Disallow: /commodity-insights/en/our-methodology/corrections/*
29Disallow: /platts/*/search-results*
30Disallow: /platts/*_contact_entries/external*
31Disallow: /platts/*/about/_pressreleases
32Disallow: /commodity-insights/en/search-results*
33Disallow: /commodity-insights/*/search-results*
34Disallow: /commodity-insights/*_contact_entries/external*
35Disallow: /commodity-insights/*/about/_pressreleases
36Disallow: prod.aws.platts.com
37Disallow: www-prod.platts.com.es
38Disallow: www-prod.platts.cn
39Disallow: www-prod.platts.kr
40Disallow: www-prod.platts.ru
41Disallow: /platts/PlattsContent/_assets/_files/en/dimensions-pro/platts-product-management-release-notes.pdf
42Disallow: /commodity-insights/PlattsContent/_assets/_files/en/dimensions-pro/platts-product-management-release-notes.pdf
43Disallow: /commodity-insights/*/Home/ChangeLanguage/*
44Disallow: /spdji/*/rss/rss-details/
45Disallow: /spdji/*/idsfactsheet*/
46Disallow: /spdji/*/idsenhancedfactsheet*/
47Disallow: /spdji/*/idsexport*/
48Disallow: /spdji/*/multimedia-center/*?
49Disallow: /spdji/*/client-services/
50Disallow: /spdji/*/documents/client-services/
51Disallow: /spdji/*/client-services
52Disallow: /spdji/*/documents/clientservices/
53Disallow: /spdji/*/supplemental-data/*?
54Disallow: /spdji/*/search/
55Disallow: /spdji/*/util/redesign/index-specialization-ri-solr-json.dot
56Disallow: /spdji/*/util/redesign/press-room/get-pr-news-announcements-solr-json.dot
57Disallow: /spdji/*/util/redesign/index-family-ri-solr-json.dot
58Disallow: /*pagenotfound.html
59Disallow: /*PageNotFound.html
60Disallow: /*error.html
61Disallow: /*Error.html
62Disallow: /*forms/
63Disallow: /*.ashx
64Disallow: /*?noheader
65Disallow: /ratings/en/regulatory/consolidated-search-landing/
66Disallow: /ratings/en/regulatory/consolidated-search-entity/
67Disallow: /ratings/en/regulatory/consolidated-search-insights/
68Disallow: /ratings/es/regulatory/consolidated-search-landing/
69Disallow: /ratings/es/regulatory/consolidated-search-entity/
70Disallow: /ratings/es/regulatory/consolidated-search-insights/
71Disallow: /ratings/pt/regulatory/consolidated-search-landing/
72Disallow: /ratings/pt/regulatory/consolidated-search-entity/
73Disallow: /ratings/pt/regulatory/consolidated-search-insights/
74Disallow: /ratings/jp/regulatory/consolidated-search-landing/
75Disallow: /ratings/jp/regulatory/consolidated-search-entity/
76Disallow: /ratings/ru/regulatory/consolidated-search-landing/
77Disallow: /ratings/ru/regulatory/consolidated-search-entity/
78Disallow: /ratings/en/regulatory/instrument-details/identifier-search/searchType/CINS/searchTerm/
79Disallow: /ratings/en/regulatory/instrument-details/identifier-search/searchType/CUSIP9/searchTerm/
80Disallow: /ratings/en/regulatory/instrument-details/identifier-search/searchType/ISIN/searchTerm/
81Disallow: /ratings/es/regulatory/instrument-details/identifier-search/searchType/CINS/searchTerm/
82Disallow: /ratings/es/regulatory/instrument-details/identifier-search/searchType/CUSIP9/searchTerm/
83Disallow: /ratings/es/regulatory/instrument-details/identifier-search/searchType/ISIN/searchTerm/
84Disallow: /ratings/pt/regulatory/instrument-details/identifier-search/searchType/CINS/searchTerm/
85Disallow: /ratings/pt/regulatory/instrument-details/identifier-search/searchType/CUSIP9/searchTerm/
86Disallow: /ratings/pt/regulatory/instrument-details/identifier-search/searchType/ISIN/searchTerm/
87Disallow: /ratings/jp/regulatory/instrument-details/identifier-search/searchType/CINS/searchTerm/
88Disallow: /ratings/jp/regulatory/instrument-details/identifier-search/searchType/CUSIP9/searchTerm/
89Disallow: /ratings/jp/regulatory/instrument-details/identifier-search/searchType/ISIN/searchTerm/
90Disallow: /ratings/ru/regulatory/instrument-details/identifier-search/searchType/CINS/searchTerm/
91Disallow: /ratings/ru/regulatory/instrument-details/identifier-search/searchType/CUSIP9/searchTerm/
92Disallow: /ratings/ru/regulatory/instrument-details/identifier-search/searchType/ISIN/searchTerm/
93Disallow: /*utm_*
94Disallow: /*source=*
95Disallow: *?gclid=*
96Disallow: /*catSelected=*
97Disallow: /*search?author=*
98Disallow: /*?uid=*
99Disallow: /*en?vregion=*
100Disallow: /*search?q=*
101Disallow: /*?ref=*
102Disallow: /*?Ref=*
103Disallow: /*?query=*
104Disallow: /*?mod=*
105Disallow: /*?jumpid=*
106Disallow: /en?aad=*
107Disallow: /*?go=*
108Disallow: /*?initialSessionID=*
109Disallow: /*?currency=*
110Disallow: /*?*kw=
111Disallow: /energy/en-test/*
112Disallow: /energy/en/our-methodology/subscriber-notes/*
113Disallow: /energy/en/our-methodology/price-symbols/*
114Disallow: /energy/en/our-methodology/corrections/*
115Disallow: /energy/en/search-results*
116Disallow: /energy/*/search-results*
117Disallow: /energy/*_contact_entries/external*
118Disallow: /energy/*/about/_pressreleases
119Disallow: /energy/PlattsContent/_assets/_files/en/dimensions-pro/platts-product-management-release-notes.pdf
120Disallow: /energy/*/Home/ChangeLanguage/*
121Disallow: /content/dam/spglobal/corporate/en/documents/education/*
122Sitemap: https://www.spglobal.com/en/sitemap.xml
123Sitemap: https://www.spglobal.com/en/sitemap-index.xml
124Sitemap: https://www.spglobal.com/ratings/en/sitemap.xml
125Sitemap: https://www.spglobal.com/ratings/sitemaps/sitemap.xml
126Sitemap: https://www.spglobal.com/ratings/sitemaps/entities-sitemap-index.xml
127Sitemap: https://www.spglobal.com/ratings/sitemaps/articles-sitemap-index.xml
128Sitemap: https://www.spglobal.com/ratings/sitemaps/cmt-sitemap-index.xml
129Sitemap: https://www.spglobal.com/ratings/sitemaps/news-sitemap.xml
130Sitemap: https://brazil.ratings.spglobal.com/ratings/pt/sitemaps/entities-sitemap-index.xml
131Sitemap: https://brazil.ratings.spglobal.com/ratings/pt/sitemaps/articles-sitemap-index.xml
132Sitemap: https://brazil.ratings.spglobal.com/ratings/pt/sitemaps/news-sitemap.xml
133Sitemap: https://brazil.ratings.spglobal.com/ratings/pt/sitemaps/sitemap.xml
134Sitemap: https://mexico.ratings.spglobal.com/ratings/es/sitemaps/sitemap.xml
135Sitemap: https://mexico.ratings.spglobal.com/ratings/es/sitemaps/news-sitemap.xml
136Sitemap: https://mexico.ratings.spglobal.com/ratings/es/sitemaps/entities-sitemap-index.xml
137Sitemap: https://mexico.ratings.spglobal.com/ratings/es/sitemaps/articles-sitemap-index.xml
138Sitemap: https://uruguay.ratings.spglobal.com/ratings/es/sitemaps/sitemap.xml
139Sitemap: https://uruguay.ratings.spglobal.com/ratings/es/sitemaps/news-sitemap.xml
140Sitemap: https://uruguay.ratings.spglobal.com/ratings/es/sitemaps/entities-sitemap-index.xml
141Sitemap: https://uruguay.ratings.spglobal.com/ratings/es/sitemaps/articles-sitemap-index.xml
142Sitemap: https://argentina.ratings.spglobal.com/ratings/es/sitemaps/sitemap.xml
143Sitemap: https://argentina.ratings.spglobal.com/ratings/es/sitemaps/news-sitemap.xml
144Sitemap: https://argentina.ratings.spglobal.com/ratings/es/sitemaps/entities-sitemap-index.xml
145Sitemap: https://argentina.ratings.spglobal.com/ratings/es/sitemaps/articles-sitemap-index.xml
146Sitemap: https://www.spglobal.com/sustainable1/en/sitemap.xml
147Sitemap: https://www.spglobal.com/market-intelligence/en/sitemap.xml
148Sitemap: https://www.spglobal.com/market-intelligence/es/sitemap.xml
149Sitemap: https://www.spglobal.com/market-intelligence/pt/sitemap.xml
150Sitemap: https://www.spglobal.com/market-intelligence/ko/sitemap.xml
151Sitemap: https://www.spglobal.com/market-intelligence/zh/sitemap.xml
152Sitemap: https://www.spglobal.com/energy/en.sitemap.news-research-videos-sitemap.xml
153Sitemap: https://www.spglobal.com/energy/en.sitemap.news-research-podcasts-sitemap.xml
154Sitemap: https://www.spglobal.com/energy/en.sitemap.news-research-latest-news-crude-oil-sitemap.xml
155Sitemap: https://www.spglobal.com/energy/en.sitemap.news-research-latest-news-natural-gas-sitemap.xml
156Sitemap: https://www.spglobal.com/energy/en.sitemap.news-research-latest-news-sitemap.xml
157Sitemap: https://www.spglobal.com/energy/en/sitemap.xml
158Sitemap: https://www.spglobal.com/energy/en/sitemap-index.xml
159Sitemap: https://www.spglobal.com/energy/es/sitemap.xml
160Sitemap: https://www.spglobal.com/energy/pt/sitemap.xml
161Sitemap: https://www.spglobal.com/energy/zh/sitemap.xml
162Sitemap: https://www.spglobal.com/energy/ko/sitemap.xml
163Sitemap: https://www.spglobal.com/spdji/en/sitemap/
164Sitemap: https://www.spglobal.com/commodityinsights/en/ci/sitemap.xml
165Sitemap: https://www.spglobal.com/en/enterprise/sitemap.xml
166Sitemap: https://www.spglobal.com/esg/s1/sitemap.xml
167Sitemap: https://www.spglobal.com/marketintelligence/en/mi/sitemap.xml
168Sitemap: https://www.spglobal.com/spdji/es/sitemap/
169Sitemap: https://www.spglobal.com/spdji/pt/sitemap/
170Sitemap: https://www.spglobal.com/spdji/kr/sitemap/
171Sitemap: https://www.spglobal.com/spdji/jp/sitemap/
172Sitemap: https://www.spglobal.com/spdji/tc/sitemap/
173Sitemap: https://www.spglobal.com/spdji/zh/sitemap/
174User-agent: Twitterbot
175Allow: /spdji/*
176User-agent: Googlebot-Image
177Disallow: /spdji/*
178User-agent: PetalBot
179Crawl-delay: 10
180User-agent: GPTBot
181Crawl-delay: 10
182User-agent: YisouSpider
183Crawl-delay: 10