USA Today / robots.txt
← back to usatoday.com · change detected 22d ago (2026-08-27T20:19:04Z) · modified +12 −0
What changed
- Added Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ for ChatGPT Operator
- Added Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ for ChatGPT Operator
- 🤖 AIAdded Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ for ChatGPT-User
- 🤖 AIAdded Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ for ChatGPT-User
- 🤖 AIAdded Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ for Google-Extended
- 🤖 AIAdded Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ for Google-Extended
- 🤖 AIAdded Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ for GPTBot
- 🤖 AIAdded Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ for GPTBot
- 🤖 AIAdded Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ for OAI-SearchBot
- 🤖 AIAdded Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ for OAI-SearchBot
- 🤖 AIAdded Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ for PerplexityBot
- 🤖 AIAdded Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ for PerplexityBot
Diff old (2026-08-11T21:11:35Z) → new (2026-08-27T20:19:03Z)
| @@ -38,6 +38,8 @@ | ||
| 38 | 38 | Allow: /shopping/ |
| 39 | 39 | Allow: /story/shopping/ |
| 40 | 40 | Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/ |
| 41 | + Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ | |
| 42 | + Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ | |
| 41 | 43 | |
| 42 | 44 | User-agent: Googlebot-News |
| 43 | 45 | Disallow: /story/sponsor-story/ |
| @@ -56,6 +58,8 @@ | ||
| 56 | 58 | Allow: /shopping/ |
| 57 | 59 | Allow: /story/shopping/ |
| 58 | 60 | Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/ |
| 61 | + Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ | |
| 62 | + Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ | |
| 59 | 63 | |
| 60 | 64 | User-agent: CCBot |
| 61 | 65 | Disallow: / |
| @@ -68,6 +72,8 @@ | ||
| 68 | 72 | Allow: /shopping/ |
| 69 | 73 | Allow: /story/shopping/ |
| 70 | 74 | Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/ |
| 75 | + Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ | |
| 76 | + Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ | |
| 71 | 77 | |
| 72 | 78 | User-agent: DataForSeoBot |
| 73 | 79 | Disallow: / |
| @@ -113,6 +119,8 @@ | ||
| 113 | 119 | Allow: /shopping/ |
| 114 | 120 | Allow: /story/shopping/ |
| 115 | 121 | Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/ |
| 122 | + Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ | |
| 123 | + Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ | |
| 116 | 124 | |
| 117 | 125 | User-agent: Scrapy |
| 118 | 126 | Disallow: / |
| @@ -137,6 +145,8 @@ | ||
| 137 | 145 | Allow: /shopping/ |
| 138 | 146 | Allow: /story/shopping/ |
| 139 | 147 | Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/ |
| 148 | + Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ | |
| 149 | + Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ | |
| 140 | 150 | |
| 141 | 151 | User-agent: Bytespider |
| 142 | 152 | Disallow: / |
| @@ -371,6 +381,8 @@ | ||
| 371 | 381 | Allow: /shopping/ |
| 372 | 382 | Allow: /story/shopping/ |
| 373 | 383 | Allow: /story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/ |
| 384 | + Allow: /story/sponsor-story/real-estate-ausa/2026/08/21/buy-now-or-wait-until-fall-tips-for-homebuyers-in-2026/91389661007/ | |
| 385 | + Allow: /story/sponsor-story/address-usa/2026/08/21/should-sellers-accept-the-first-offer-what-to-consider-beyond-price/91390104007/ | |
| 374 | 386 | |
| 375 | 387 | User-agent: ChatGPT-PageFetcher |
| 376 | 388 | Disallow: / |