USA Today / robots.txt
← back to usatoday.com · change detected 14d ago (2026-09-04T16:59:44Z) · modified +6 −12
What changed
- Added Allow: /branded-story/ for ChatGPT Operator
- Removed Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ for ChatGPT Operator
- Removed Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ for ChatGPT Operator
- 🤖 AIAdded Allow: /branded-story/ for ChatGPT-User
- 🤖 AIRemoved Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ for ChatGPT-User
- 🤖 AIRemoved Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ for ChatGPT-User
- 🤖 AIAdded Allow: /branded-story/ for Google-Extended
- 🤖 AIRemoved Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ for Google-Extended
- 🤖 AIRemoved Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ for Google-Extended
- 🤖 AIAdded Allow: /branded-story/ for GPTBot
- 🤖 AIRemoved Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ for GPTBot
- 🤖 AIRemoved Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ for GPTBot
- 🤖 AIAdded Allow: /branded-story/ for OAI-SearchBot
- 🤖 AIRemoved Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ for OAI-SearchBot
- 🤖 AIRemoved Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ for OAI-SearchBot
- 🤖 AIAdded Allow: /branded-story/ for PerplexityBot
- 🤖 AIRemoved Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ for PerplexityBot
- 🤖 AIRemoved Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ for PerplexityBot
Diff old (2026-08-27T20:19:03Z) → new (2026-09-04T16:59:43Z)
| @@ -35,11 +35,10 @@ | ||
| 35 | 35 | Allow: /contributor-content/ |
| 36 | 36 | Allow: /story/special/contributor-content/ |
| 37 | 37 | Allow: /money/money-management/ |
| 38 | + Allow: /branded-story/ | |
| 38 | 39 | Allow: /shopping/ |
| 39 | 40 | Allow: /story/shopping/ |
| 40 | 41 | Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/ |
| 41 | - Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ | |
| 42 | - Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ | |
| 43 | 42 | |
| 44 | 43 | User-agent: Googlebot-News |
| 45 | 44 | Disallow: /story/sponsor-story/ |
| @@ -55,11 +54,10 @@ | ||
| 55 | 54 | Allow: /contributor-content/ |
| 56 | 55 | Allow: /story/special/contributor-content/ |
| 57 | 56 | Allow: /money/money-management/ |
| 57 | + Allow: /branded-story/ | |
| 58 | 58 | Allow: /shopping/ |
| 59 | 59 | Allow: /story/shopping/ |
| 60 | 60 | Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/ |
| 61 | - Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ | |
| 62 | - Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ | |
| 63 | 61 | |
| 64 | 62 | User-agent: CCBot |
| 65 | 63 | Disallow: / |
| @@ -69,11 +67,10 @@ | ||
| 69 | 67 | Allow: /contributor-content/ |
| 70 | 68 | Allow: /story/special/contributor-content/ |
| 71 | 69 | Allow: /money/money-management/ |
| 70 | + Allow: /branded-story/ | |
| 72 | 71 | Allow: /shopping/ |
| 73 | 72 | Allow: /story/shopping/ |
| 74 | 73 | Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/ |
| 75 | - Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ | |
| 76 | - Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ | |
| 77 | 74 | |
| 78 | 75 | User-agent: DataForSeoBot |
| 79 | 76 | Disallow: / |
| @@ -116,11 +113,10 @@ | ||
| 116 | 113 | Allow: /contributor-content/ |
| 117 | 114 | Allow: /story/special/contributor-content/ |
| 118 | 115 | Allow: /money/money-management/ |
| 116 | + Allow: /branded-story/ | |
| 119 | 117 | Allow: /shopping/ |
| 120 | 118 | Allow: /story/shopping/ |
| 121 | 119 | Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/ |
| 122 | - Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ | |
| 123 | - Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ | |
| 124 | 120 | |
| 125 | 121 | User-agent: Scrapy |
| 126 | 122 | Disallow: / |
| @@ -142,11 +138,10 @@ | ||
| 142 | 138 | Allow: /contributor-content/ |
| 143 | 139 | Allow: /story/special/contributor-content/ |
| 144 | 140 | Allow: /money/money-management/ |
| 141 | + Allow: /branded-story/ | |
| 145 | 142 | Allow: /shopping/ |
| 146 | 143 | Allow: /story/shopping/ |
| 147 | 144 | Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/ |
| 148 | - Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ | |
| 149 | - Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ | |
| 150 | 145 | |
| 151 | 146 | User-agent: Bytespider |
| 152 | 147 | Disallow: / |
| @@ -378,11 +373,10 @@ | ||
| 378 | 373 | Allow: /contributor-content/ |
| 379 | 374 | Allow: /story/special/contributor-content/ |
| 380 | 375 | Allow: /money/money-management/ |
| 376 | + Allow: /branded-story/ | |
| 381 | 377 | Allow: /shopping/ |
| 382 | 378 | Allow: /story/shopping/ |
| 383 | 379 | Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/ |
| 384 | - Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ | |
| 385 | - Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ | |
| 386 | 380 | |
| 387 | 381 | User-agent: ChatGPT-PageFetcher |
| 388 | 382 | Disallow: / |