Cemex / robots.txt
← back to cemex.com · change detected 1mo ago (2026-09-15T02:23:28Z) · modified +46 −2
What changed
- Added Allow: /documents/ for *
- Added Allow: /favicon.ico for *
- Added Allow: /image/* for *
- Added Allow: /o/web/image/* for *
- Added Allow: /sitemap.xml for *
- Added Disallow: */web for *
- Added Disallow: /*?p_p_lifecycle=* for *
- Added Disallow: /*p_p_auth=* for *
- Added Disallow: /*p_p_id=* for *
- Added Disallow: /*p_p_lifecycle=* for *
- Added Disallow: /*p_p_mode=* for *
- Added Disallow: /*p_p_state=* for *
- Added Disallow: /api/jsonws/ for *
- Added Disallow: /c/portal/ for *
- Added Disallow: /configuration_admin/ for *
- Added Disallow: /control_panel/ for *
- Added Disallow: /d/ for *
- Added Disallow: /delegate/ for *
- Added Disallow: /drafts/ for *
- Added Disallow: /group/ for *
- Added Disallow: /group/*/control_panel/ for *
- Added Disallow: /group/control_panel/ for *
- Added Disallow: /instance/ for *
- Added Disallow: /o/ for *
- Added Disallow: /o/api/ for *
- Added Disallow: /o/portal/ for *
- Added Disallow: /portal/ for *
- Added Disallow: /preview/ for *
- Added Disallow: /search/ for *
- Added Disallow: /server-admin/ for *
- Added Disallow: /staging/ for *
- Added Disallow: /system/ for *
- Added Disallow: /user/ for *
- Added Disallow: /web/*/control_panel/ for *
- Added Disallow: /workflow/ for *
- Added Disallow: /~/control_panel/ for *
- Removed Disallow: /en-US/ for *
- Removed Disallow: /es-MX/ for *
Diff old (2026-06-20T01:10:30Z) → new (2026-09-15T02:23:28Z)
| @@ -3,6 +3,29 @@ | ||
| 3 | 3 | |
| 4 | 4 | User-Agent: * |
| 5 | 5 | |
| 6 | + # --- Disallow internal, administrative, or dynamic URLs --- | |
| 7 | + Disallow: /c/ | |
| 8 | + Disallow: /delegate/ | |
| 9 | + Disallow: /d/ | |
| 10 | + Disallow: /group/*/control_panel/ | |
| 11 | + Disallow: /web/*/control_panel/ | |
| 12 | + Disallow: /~/control_panel/ | |
| 13 | + Disallow: /api/jsonws/ | |
| 14 | + Disallow: /c/portal/ | |
| 15 | + Disallow: /group/control_panel/ | |
| 16 | + Disallow: /api/jsonws/ | |
| 17 | + Disallow: /o/ | |
| 18 | + Disallow: /o/portal/ | |
| 19 | + Disallow: /o/api/ | |
| 20 | + Disallow: /*p_p_id=* | |
| 21 | + Disallow: /*p_p_lifecycle=* | |
| 22 | + Disallow: /*p_p_state=* | |
| 23 | + Disallow: /*p_p_mode=* | |
| 24 | + Disallow: /*p_p_auth=* | |
| 25 | + Disallow: /*?p_p_lifecycle=* | |
| 26 | + Disallow: /portal/ | |
| 27 | + Disallow: /user/ | |
| 28 | + Disallow: /search/ | |
| 6 | 29 | Disallow: /web/ |
| 7 | 30 | Disallow: /asset-publisher/ |
| 8 | 31 | Disallow: /-/asset_publisher/ |
| @@ -15,8 +38,29 @@ | ||
| 15 | 38 | Disallow: /pt/ |
| 16 | 39 | Disallow: /de/ |
| 17 | 40 | Disallow: /c/ |
| 18 | - Disallow: /es-MX/ | |
| 19 | - Disallow: /en-US/ | |
| 41 | + | |
| 42 | + | |
| 43 | + # --- Block temporary or preview URLs --- | |
| 44 | + Disallow: /preview/ | |
| 45 | + Disallow: /staging/ | |
| 46 | + Disallow: /drafts/ | |
| 47 | + Disallow: /workflow/ | |
| 48 | + | |
| 49 | + # --- Block common Liferay admin paths --- | |
| 50 | + Disallow: /control_panel/ | |
| 51 | + Disallow: /system/ | |
| 52 | + Disallow: /configuration_admin/ | |
| 53 | + Disallow: /instance/ | |
| 54 | + Disallow: /server-admin/ | |
| 55 | + Disallow: */web | |
| 56 | + Disallow: /group/ | |
| 57 | + | |
| 58 | + # --- Allow robots to crawl the sitemap and assets --- | |
| 59 | + Allow: /sitemap.xml | |
| 60 | + Allow: /documents/ | |
| 61 | + Allow: /image/* | |
| 62 | + Allow: /o/web/image/* | |
| 63 | + Allow: /favicon.ico | |
| 20 | 64 | |
| 21 | 65 | Sitemap: https://www.cemex.com/sitemap.xml |
| 22 | 66 |