NeuralCrawl

S&P Global / robots.txt snapshot

← back to spglobal.com · fetched 2026-06-20T01:10:31Z (18h ago) · HTTP 200 · 9997 bytes · sha256 e8554f5e08bab2be · raw

final URL: https://www.spglobal.com/robots.txt

1User-agent: *
2Allow: /spdji/*/multimedia-center/?p
3Allow: /sustainable1/en/scores/results?cid=*
4Disallow: /en-test/*
5Disallow: /ratings/en-test/*
6Disallow: /market-intelligence/en-test/*
7Disallow: /commodity-insights/en-test/*
8Disallow: /automotive-insights/en-test/*
9Disallow: /sustainable1/en-test/*
10Disallow: /en/homepage-v2
11Disallow: /en/search
12Disallow: /market-intelligence/en/search
13Disallow: /ratings/en/search
14Disallow: /engineering/en/search
15Disallow: /mobility/en/search
16Disallow: prod.aws.platts.com
17Disallow: prod-us.aws.platts.com
18Disallow: prod-us1.aws.platts.com
19Disallow: prod-us2.aws.platts.com
20Disallow: prod-us3.aws.platts.com
21Disallow: prod-uk.aws.platts.com
22Disallow: prod-uk1.aws.platts.com
23Disallow: prod-uk2.aws.platts.com
24Disallow: prod-uk3.aws.platts.com
25Disallow: prod-ap.aws.platts.com
26Disallow: prod-ap1.aws.platts.com
27Disallow: prod-ap2.aws.platts.com
28Disallow: /platts/en/our-methodology/corrections/*
29Disallow: /commodity-insights/en/our-methodology/subscriber-notes/*
30Disallow: /commodity-insights/en/our-methodology/price-symbols/*
31Disallow: /commodity-insights/en/our-methodology/corrections/*
32Disallow: /platts/*/search-results*
33Disallow: /platts/*_contact_entries/external*
34Disallow: /platts/*/about/_pressreleases
35Disallow: /commodity-insights/en/search-results*
36Disallow: /commodity-insights/*/search-results*
37Disallow: /commodity-insights/*_contact_entries/external*
38Disallow: /commodity-insights/*/about/_pressreleases
39Disallow: prod.aws.platts.com
40Disallow: www-prod.platts.com.es
41Disallow: www-prod.platts.cn
42Disallow: www-prod.platts.kr
43Disallow: www-prod.platts.ru
44Disallow: /platts/PlattsContent/_assets/_files/en/dimensions-pro/platts-product-management-release-notes.pdf
45Disallow: /commodity-insights/PlattsContent/_assets/_files/en/dimensions-pro/platts-product-management-release-notes.pdf
46Disallow: /commodity-insights/*/Home/ChangeLanguage/*
47Disallow: /spdji/*/rss/rss-details/
48Disallow: /spdji/*/idsfactsheet*/
49Disallow: /spdji/*/idsenhancedfactsheet*/
50Disallow: /spdji/*/idsexport*/
51Disallow: /spdji/*/multimedia-center/*?
52Disallow: /spdji/*/client-services/
53Disallow: /spdji/*/documents/client-services/
54Disallow: /spdji/*/client-services
55Disallow: /spdji/*/documents/clientservices/
56Disallow: /spdji/*/supplemental-data/*?
57Disallow: /spdji/*/search/
58Disallow: /spdji/*/util/redesign/index-specialization-ri-solr-json.dot
59Disallow: /spdji/*/util/redesign/press-room/get-pr-news-announcements-solr-json.dot
60Disallow: /spdji/*/util/redesign/index-family-ri-solr-json.dot
61Disallow: /*pagenotfound.html
62Disallow: /*PageNotFound.html
63Disallow: /*error.html
64Disallow: /*Error.html
65Disallow: /*forms/
66Disallow: /*.ashx
67Disallow: /*?noheader
68Disallow: /ratings/en/regulatory/consolidated-search-landing/
69Disallow: /ratings/en/regulatory/consolidated-search-entity/
70Disallow: /ratings/en/regulatory/consolidated-search-insights/
71Disallow: /ratings/es/regulatory/consolidated-search-landing/
72Disallow: /ratings/es/regulatory/consolidated-search-entity/
73Disallow: /ratings/es/regulatory/consolidated-search-insights/
74Disallow: /ratings/pt/regulatory/consolidated-search-landing/
75Disallow: /ratings/pt/regulatory/consolidated-search-entity/
76Disallow: /ratings/pt/regulatory/consolidated-search-insights/
77Disallow: /ratings/jp/regulatory/consolidated-search-landing/
78Disallow: /ratings/jp/regulatory/consolidated-search-entity/
79Disallow: /ratings/ru/regulatory/consolidated-search-landing/
80Disallow: /ratings/ru/regulatory/consolidated-search-entity/
81Disallow: /ratings/en/regulatory/instrument-details/identifier-search/searchType/CINS/searchTerm/
82Disallow: /ratings/en/regulatory/instrument-details/identifier-search/searchType/CUSIP9/searchTerm/
83Disallow: /ratings/en/regulatory/instrument-details/identifier-search/searchType/ISIN/searchTerm/
84Disallow: /ratings/es/regulatory/instrument-details/identifier-search/searchType/CINS/searchTerm/
85Disallow: /ratings/es/regulatory/instrument-details/identifier-search/searchType/CUSIP9/searchTerm/
86Disallow: /ratings/es/regulatory/instrument-details/identifier-search/searchType/ISIN/searchTerm/
87Disallow: /ratings/pt/regulatory/instrument-details/identifier-search/searchType/CINS/searchTerm/
88Disallow: /ratings/pt/regulatory/instrument-details/identifier-search/searchType/CUSIP9/searchTerm/
89Disallow: /ratings/pt/regulatory/instrument-details/identifier-search/searchType/ISIN/searchTerm/
90Disallow: /ratings/jp/regulatory/instrument-details/identifier-search/searchType/CINS/searchTerm/
91Disallow: /ratings/jp/regulatory/instrument-details/identifier-search/searchType/CUSIP9/searchTerm/
92Disallow: /ratings/jp/regulatory/instrument-details/identifier-search/searchType/ISIN/searchTerm/
93Disallow: /ratings/ru/regulatory/instrument-details/identifier-search/searchType/CINS/searchTerm/
94Disallow: /ratings/ru/regulatory/instrument-details/identifier-search/searchType/CUSIP9/searchTerm/
95Disallow: /ratings/ru/regulatory/instrument-details/identifier-search/searchType/ISIN/searchTerm/
96Disallow: /*utm_*
97Disallow: /*source=*
98Disallow: *?gclid=*
99Disallow: /*catSelected=*
100Disallow: /*search?author=*
101Disallow: /*?uid=*
102Disallow: /*en?vregion=*
103Disallow: /*search?q=*
104Disallow: /*?ref=*
105Disallow: /*?Ref=*
106Disallow: /*?query=*
107Disallow: /*?mod=*
108Disallow: /*?jumpid=*
109Disallow: /en?aad=*
110Disallow: /*?go=*
111Disallow: /*?initialSessionID=*
112Disallow: /*?currency=*
113Disallow: /*?*kw=
114Disallow: /energy/en-test/*
115Disallow: /energy/en/our-methodology/subscriber-notes/*
116Disallow: /energy/en/our-methodology/price-symbols/*
117Disallow: /energy/en/our-methodology/corrections/*
118Disallow: /energy/en/search-results*
119Disallow: /energy/*/search-results*
120Disallow: /energy/*_contact_entries/external*
121Disallow: /energy/*/about/_pressreleases
122Disallow: /energy/PlattsContent/_assets/_files/en/dimensions-pro/platts-product-management-release-notes.pdf
123Disallow: /energy/*/Home/ChangeLanguage/*
124Disallow: /content/dam/spglobal/corporate/en/documents/education/*
125Sitemap: https://www.spglobal.com/en/sitemap.xml
126Sitemap: https://www.spglobal.com/en/sitemap-index.xml
127Sitemap: https://www.spglobal.com/ratings/en/sitemap.xml
128Sitemap: https://www.spglobal.com/ratings/sitemaps/sitemap.xml
129Sitemap: https://www.spglobal.com/ratings/sitemaps/entities-sitemap-index.xml
130Sitemap: https://www.spglobal.com/ratings/sitemaps/articles-sitemap-index.xml
131Sitemap: https://www.spglobal.com/ratings/sitemaps/cmt-sitemap-index.xml
132Sitemap: https://www.spglobal.com/ratings/sitemaps/news-sitemap.xml
133Sitemap: https://brazil.ratings.spglobal.com/ratings/pt/sitemaps/entities-sitemap-index.xml
134Sitemap: https://brazil.ratings.spglobal.com/ratings/pt/sitemaps/articles-sitemap-index.xml
135Sitemap: https://brazil.ratings.spglobal.com/ratings/pt/sitemaps/news-sitemap.xml
136Sitemap: https://brazil.ratings.spglobal.com/ratings/pt/sitemaps/sitemap.xml
137Sitemap: https://mexico.ratings.spglobal.com/ratings/es/sitemaps/sitemap.xml
138Sitemap: https://mexico.ratings.spglobal.com/ratings/es/sitemaps/news-sitemap.xml
139Sitemap: https://mexico.ratings.spglobal.com/ratings/es/sitemaps/entities-sitemap-index.xml
140Sitemap: https://mexico.ratings.spglobal.com/ratings/es/sitemaps/articles-sitemap-index.xml
141Sitemap: https://uruguay.ratings.spglobal.com/ratings/es/sitemaps/sitemap.xml
142Sitemap: https://uruguay.ratings.spglobal.com/ratings/es/sitemaps/news-sitemap.xml
143Sitemap: https://uruguay.ratings.spglobal.com/ratings/es/sitemaps/entities-sitemap-index.xml
144Sitemap: https://uruguay.ratings.spglobal.com/ratings/es/sitemaps/articles-sitemap-index.xml
145Sitemap: https://argentina.ratings.spglobal.com/ratings/es/sitemaps/sitemap.xml
146Sitemap: https://argentina.ratings.spglobal.com/ratings/es/sitemaps/news-sitemap.xml
147Sitemap: https://argentina.ratings.spglobal.com/ratings/es/sitemaps/entities-sitemap-index.xml
148Sitemap: https://argentina.ratings.spglobal.com/ratings/es/sitemaps/articles-sitemap-index.xml
149Sitemap: https://www.spglobal.com/sustainable1/en/sitemap.xml
150Sitemap: https://www.spglobal.com/market-intelligence/en/sitemap.xml
151Sitemap: https://www.spglobal.com/market-intelligence/es/sitemap.xml
152Sitemap: https://www.spglobal.com/market-intelligence/pt/sitemap.xml
153Sitemap: https://www.spglobal.com/market-intelligence/ko/sitemap.xml
154Sitemap: https://www.spglobal.com/market-intelligence/zh/sitemap.xml
155Sitemap: https://www.spglobal.com/energy/en.sitemap.news-research-videos-sitemap.xml
156Sitemap: https://www.spglobal.com/energy/en.sitemap.news-research-podcasts-sitemap.xml
157Sitemap: https://www.spglobal.com/energy/en.sitemap.news-research-latest-news-crude-oil-sitemap.xml
158Sitemap: https://www.spglobal.com/energy/en.sitemap.news-research-latest-news-natural-gas-sitemap.xml
159Sitemap: https://www.spglobal.com/energy/en.sitemap.news-research-latest-news-sitemap.xml
160Sitemap: https://www.spglobal.com/energy/en/sitemap.xml
161Sitemap: https://www.spglobal.com/energy/en/sitemap-index.xml
162Sitemap: https://www.spglobal.com/energy/es/sitemap.xml
163Sitemap: https://www.spglobal.com/energy/pt/sitemap.xml
164Sitemap: https://www.spglobal.com/energy/zh/sitemap.xml
165Sitemap: https://www.spglobal.com/energy/ko/sitemap.xml
166Sitemap: https://www.spglobal.com/spdji/en/sitemap/
167Sitemap: https://www.spglobal.com/mobility/en/sitemap.xml
168Sitemap: https://www.spglobal.com/automotive-insights/en/sitemap.xml
169Sitemap: https://www.spglobal.com/commodityinsights/en/ci/sitemap.xml
170Sitemap: https://www.spglobal.com/en/enterprise/sitemap.xml
171Sitemap: https://www.spglobal.com/esg/s1/sitemap.xml
172Sitemap: https://www.spglobal.com/marketintelligence/en/mi/sitemap.xml
173Sitemap: https://www.spglobal.com/spdji/es/sitemap/
174Sitemap: https://www.spglobal.com/spdji/pt/sitemap/
175Sitemap: https://www.spglobal.com/spdji/kr/sitemap/
176Sitemap: https://www.spglobal.com/spdji/jp/sitemap/
177Sitemap: https://www.spglobal.com/spdji/tc/sitemap/
178Sitemap: https://www.spglobal.com/spdji/zh/sitemap/
179User-agent: Twitterbot
180Allow: /spdji/*
181User-agent: Googlebot-Image
182Disallow: /spdji/*
183User-agent: PetalBot
184Crawl-delay: 10
185User-agent: GPTBot
186Crawl-delay: 10
187User-agent: YisouSpider
188Crawl-delay: 10