Snowflake / robots.txt snapshot
← back to snowflake.com · fetched 2026-10-19T07:56:10Z (19d ago) · HTTP 200 · 2751 bytes · sha256 8f7044965766cc80 · raw
final URL: https://www.snowflake.com/robots.txt
| 1 | # ---------------------------------------------------------------------- |
| 2 | # Global default for all other crawlers (Applicable to both WordPress & AEM if paths match) |
| 3 | # ---------------------------------------------------------------------- |
| 4 | |
| 5 | User-agent: * |
| 6 | Allow: / |
| 7 | # Disallow WordPress admin area |
| 8 | Disallow: /wp-admin/ |
| 9 | # Disallow common search query parameters to prevent duplicate content |
| 10 | Disallow: *?s= |
| 11 | Disallow: *&s= |
| 12 | |
| 13 | # ---------------------------------------------------------------------- |
| 14 | # AEM (Adobe Experience Manager) specific rules |
| 15 | # ---------------------------------------------------------------------- |
| 16 | |
| 17 | # Standard AEM authoring, system, and temporary paths |
| 18 | Disallow: /apps/ |
| 19 | Disallow: /bin/ |
| 20 | Disallow: /crx/ |
| 21 | Disallow: /etc/ |
| 22 | Disallow: /etc/clientlibs/ |
| 23 | Disallow: /libs/ |
| 24 | Disallow: /system/ |
| 25 | Disallow: /tmp/ |
| 26 | Disallow: /var/ |
| 27 | # Disallow JCR content exposure |
| 28 | Disallow: /jcr:content/ |
| 29 | # Disallow JCR content exposure in subpages |
| 30 | Disallow: /*/_jcr_content/ |
| 31 | # Common AEM dispatcher cache invalidation path |
| 32 | Disallow: /_/ |
| 33 | # Disallow AEM's infinity.json selectors |
| 34 | Disallow: /*.infinity.json$ |
| 35 | Disallow: /*.tidy.json$ |
| 36 | Disallow: /*.sysview.xml$ |
| 37 | # If you have specific feeds to allow, add Allow rules before this |
| 38 | Disallow: /*.feed.xml$ |
| 39 | |
| 40 | # AEM DAM rules |
| 41 | # Allow specific legal PDFs in English |
| 42 | Allow: /content/dam/snowflake-site/en/legal/*.pdf |
| 43 | |
| 44 | # Disallow other PDFs within the main snowflake-site DAM path |
| 45 | Disallow: /content/dam/snowflake-site/*.pdf |
| 46 | |
| 47 | # Allow crawling of the general DAM path (if it contains browsable content or other allowed assets) |
| 48 | Allow: /content/dam/snowflake-site/ |
| 49 | |
| 50 | # ---------------------------------------------------------------------- |
| 51 | # WordPress specific rules |
| 52 | # ---------------------------------------------------------------------- |
| 53 | |
| 54 | # Disallow theme directory |
| 55 | Disallow: /wp-content/themes/snowflake/ |
| 56 | |
| 57 | # Allow WordPress AJAX handler (important for some plugin/theme functionality) |
| 58 | Allow: /wp-admin/admin-ajax.php |
| 59 | |
| 60 | # Disallow other common WordPress paths that don't need indexing |
| 61 | Disallow: /wp-includes/ |
| 62 | Disallow: /wp-content/plugins/ |
| 63 | Disallow: /wp-content/cache/ |
| 64 | # WordPress REST API, usually not for direct crawling unless specific endpoints are public |
| 65 | Disallow: /wp-json/ |
| 66 | Disallow: /xmlrpc.php |
| 67 | Disallow: /readme.html |
| 68 | Disallow: /license.txt |
| 69 | Disallow: /trackback/ |
| 70 | # If you want specific feeds crawled, use Allow rules. Otherwise, sitemap is preferred. |
| 71 | Disallow: /feed/ |
| 72 | Disallow: */feed/$ |
| 73 | Disallow: */trackback/$ |
| 74 | Disallow: /*?replytocom= |
| 75 | |
| 76 | # ---------------------------------------------------------------------- |
| 77 | # Sitemap Directives |
| 78 | # ---------------------------------------------------------------------- |
| 79 | |
| 80 | Sitemap: https://www.snowflake.com/sitemap_index.xml |
| 81 | Sitemap: https://www.snowflake.com/content/snowflake-site/global.sitemap.xml |