Airbnb / robots.txt
← back to airbnb.com · change detected 1mo ago (2026-09-26T05:35:36Z) · modified +24 −134
What changed
- 🤖 AIBlocked AI2Bot (Allen Institute for AI) entirely
- 🤖 AIBlocked Applebot-Extended (Apple) entirely
- Blocked user-agent ‘MistralAI-Training’ entirely
- Blocked user-agent ‘Webzio-Extended’ entirely
- 🤖 AIBlocked ClaudeBot (Anthropic) entirely
- 🤖 AIRemoved Allow: /.well-known/amphtml/apikey.pub for ClaudeBot
- 🤖 AIRemoved Allow: /calendar/ical/ for ClaudeBot
- 🤖 AIRemoved Disallow: /*/skeleton for ClaudeBot
- 🤖 AIRemoved Disallow: /*/sw_skeleton for ClaudeBot
- 🤖 AIRemoved Disallow: /*/things-to-do for ClaudeBot
- 🤖 AIRemoved Disallow: /.well-known/assetlinks.json for ClaudeBot
- 🤖 AIRemoved Disallow: /500 for ClaudeBot
- 🤖 AIRemoved Disallow: /account for ClaudeBot
- 🤖 AIRemoved Disallow: /alumni for ClaudeBot
- 🤖 AIRemoved Disallow: /api/v1/trebuchet for ClaudeBot
- 🤖 AIRemoved Disallow: /api/v2/google_place_photos for ClaudeBot
- 🤖 AIRemoved Disallow: /api/v3/LandingPageStaysQuery for ClaudeBot
- 🤖 AIRemoved Disallow: /associates/click for ClaudeBot
- 🤖 AIRemoved Disallow: /book/ for ClaudeBot
- 🤖 AIRemoved Disallow: /calendar/ for ClaudeBot
- 🤖 AIRemoved Disallow: /contact_host for ClaudeBot
- 🤖 AIRemoved Disallow: /disaster/lookup for ClaudeBot
- 🤖 AIRemoved Disallow: /email/unsubscribe for ClaudeBot
- 🤖 AIRemoved Disallow: /embeddable for ClaudeBot
- 🤖 AIRemoved Disallow: /experiences/ for ClaudeBot
- 🤖 AIRemoved Disallow: /experiences/*/book for ClaudeBot
- 🤖 AIRemoved Disallow: /experiences/*?*modal for ClaudeBot
- 🤖 AIRemoved Disallow: /experiences/*?*scheduled_id for ClaudeBot
- 🤖 AIRemoved Disallow: /external_link? for ClaudeBot
- 🤖 AIRemoved Disallow: /fix-it for ClaudeBot
- 🤖 AIRemoved Disallow: /fixit for ClaudeBot
- 🤖 AIRemoved Disallow: /forgot_password for ClaudeBot
- 🤖 AIRemoved Disallow: /google_place_photo for ClaudeBot
- 🤖 AIRemoved Disallow: /groups for ClaudeBot
- 🤖 AIRemoved Disallow: /guidebooks for ClaudeBot
- 🤖 AIRemoved Disallow: /help/ for ClaudeBot
- 🤖 AIRemoved Disallow: /help/feedback for ClaudeBot
- 🤖 AIRemoved Disallow: /help/search for ClaudeBot
- 🤖 AIRemoved Disallow: /home/dashboard for ClaudeBot
- 🤖 AIRemoved Disallow: /inbox for ClaudeBot
- 🤖 AIRemoved Disallow: /login_with_redirect for ClaudeBot
- 🤖 AIRemoved Disallow: /logout for ClaudeBot
- 🤖 AIRemoved Disallow: /manage-listing for ClaudeBot
- 🤖 AIRemoved Disallow: /messaging/ajax_already_messaged/ for ClaudeBot
- 🤖 AIRemoved Disallow: /my_listings for ClaudeBot
- 🤖 AIRemoved Disallow: /oauth_connect for ClaudeBot
- 🤖 AIRemoved Disallow: /payments/book for ClaudeBot
- 🤖 AIRemoved Disallow: /reservation for ClaudeBot
- 🤖 AIRemoved Disallow: /rooms/ for ClaudeBot
- 🤖 AIRemoved Disallow: /rooms/*/amenities for ClaudeBot
- ... and 84 more change(s)
Diff old (2026-06-20T01:10:30Z) → new (2026-09-26T05:35:36Z)
| @@ -412,71 +412,6 @@ | ||
| 412 | 412 | Disallow: /s/*/* |
| 413 | 413 | Disallow: /things-to-do/places |
| 414 | 414 | |
| 415 | - User-agent: GPTBot | |
| 416 | - Allow: /calendar/ical/ | |
| 417 | - Allow: /.well-known/amphtml/apikey.pub | |
| 418 | - Disallow: /.well-known/assetlinks.json | |
| 419 | - Disallow: /*/skeleton | |
| 420 | - Disallow: /*/sw_skeleton | |
| 421 | - Disallow: /500 | |
| 422 | - Disallow: /account | |
| 423 | - Disallow: /alumni | |
| 424 | - Disallow: /api/v1/trebuchet | |
| 425 | - Disallow: /associates/click | |
| 426 | - Disallow: /book/ | |
| 427 | - Disallow: /calendar/ | |
| 428 | - Disallow: /contact_host | |
| 429 | - Disallow: /disaster/lookup | |
| 430 | - Disallow: /email/unsubscribe | |
| 431 | - Disallow: /embeddable | |
| 432 | - Disallow: /experiences/*?*scheduled_id | |
| 433 | - Disallow: /experiences/*?*modal | |
| 434 | - Disallow: /experiences/*/book | |
| 435 | - Disallow: /external_link? | |
| 436 | - Disallow: /fix-it | |
| 437 | - Disallow: /fixit | |
| 438 | - Disallow: /forgot_password | |
| 439 | - Disallow: /google_place_photo | |
| 440 | - Disallow: /api/v2/google_place_photos | |
| 441 | - Disallow: /groups | |
| 442 | - Disallow: /guidebooks | |
| 443 | - Disallow: /help/feedback | |
| 444 | - Disallow: /help/search | |
| 445 | - Disallow: /help/search | |
| 446 | - Disallow: /home/dashboard | |
| 447 | - Disallow: /inbox | |
| 448 | - Disallow: /login_with_redirect | |
| 449 | - Disallow: /logout | |
| 450 | - Disallow: /manage-listing | |
| 451 | - Disallow: /messaging/ajax_already_messaged/ | |
| 452 | - Disallow: /my_listings | |
| 453 | - Disallow: /oauth_connect | |
| 454 | - Disallow: /payments/book | |
| 455 | - Disallow: /reservation | |
| 456 | - Disallow: /rooms/*/amenities | |
| 457 | - Disallow: /rooms/*/enhanced-cleaning | |
| 458 | - Disallow: /rooms/*/house-rules | |
| 459 | - Disallow: /rooms/*/location | |
| 460 | - Disallow: /rooms/*/photos | |
| 461 | - Disallow: /rooms/*/reviews | |
| 462 | - Disallow: /rooms/*/safety | |
| 463 | - Disallow: /rooms/*?viralityEntryPoint | |
| 464 | - Disallow: /rooms/*/cancellation-policy | |
| 465 | - Disallow: /rooms/*/description | |
| 466 | - Disallow: /s/guidebooks | |
| 467 | - Disallow: /signed_out_modal.json | |
| 468 | - Disallow: /signup_modal | |
| 469 | - Disallow: /stories | |
| 470 | - Disallow: /trips/upcoming | |
| 471 | - Disallow: /trips/v1/ | |
| 472 | - Disallow: /update-your-browser | |
| 473 | - Disallow: /users/*/listings | |
| 474 | - Disallow: /users/show | |
| 475 | - Disallow: /users/profile | |
| 476 | - Disallow: /api/v3/LandingPageStaysQuery | |
| 477 | - Disallow: /s/*/* | |
| 478 | - Disallow: /things-to-do/places | |
| 479 | - | |
| 480 | 415 | User-agent: meta-externalagent |
| 481 | 416 | Allow: /calendar/ical/ |
| 482 | 417 | Allow: /.well-known/amphtml/apikey.pub |
| @@ -676,75 +611,6 @@ | ||
| 676 | 611 | Disallow: /rooms/ |
| 677 | 612 | Disallow: /experiences/ |
| 678 | 613 | |
| 679 | - User-agent: ClaudeBot | |
| 680 | - Allow: /calendar/ical/ | |
| 681 | - Allow: /.well-known/amphtml/apikey.pub | |
| 682 | - Disallow: /.well-known/assetlinks.json | |
| 683 | - Disallow: /*/skeleton | |
| 684 | - Disallow: /*/sw_skeleton | |
| 685 | - Disallow: /500 | |
| 686 | - Disallow: /account | |
| 687 | - Disallow: /alumni | |
| 688 | - Disallow: /api/v1/trebuchet | |
| 689 | - Disallow: /associates/click | |
| 690 | - Disallow: /book/ | |
| 691 | - Disallow: /calendar/ | |
| 692 | - Disallow: /contact_host | |
| 693 | - Disallow: /disaster/lookup | |
| 694 | - Disallow: /email/unsubscribe | |
| 695 | - Disallow: /embeddable | |
| 696 | - Disallow: /experiences/*?*scheduled_id | |
| 697 | - Disallow: /experiences/*?*modal | |
| 698 | - Disallow: /experiences/*/book | |
| 699 | - Disallow: /external_link? | |
| 700 | - Disallow: /fix-it | |
| 701 | - Disallow: /fixit | |
| 702 | - Disallow: /forgot_password | |
| 703 | - Disallow: /google_place_photo | |
| 704 | - Disallow: /api/v2/google_place_photos | |
| 705 | - Disallow: /groups | |
| 706 | - Disallow: /guidebooks | |
| 707 | - Disallow: /help/feedback | |
| 708 | - Disallow: /help/search | |
| 709 | - Disallow: /help/search | |
| 710 | - Disallow: /home/dashboard | |
| 711 | - Disallow: /inbox | |
| 712 | - Disallow: /login_with_redirect | |
| 713 | - Disallow: /logout | |
| 714 | - Disallow: /manage-listing | |
| 715 | - Disallow: /messaging/ajax_already_messaged/ | |
| 716 | - Disallow: /my_listings | |
| 717 | - Disallow: /oauth_connect | |
| 718 | - Disallow: /payments/book | |
| 719 | - Disallow: /reservation | |
| 720 | - Disallow: /rooms/*/amenities | |
| 721 | - Disallow: /rooms/*/enhanced-cleaning | |
| 722 | - Disallow: /rooms/*/house-rules | |
| 723 | - Disallow: /rooms/*/location | |
| 724 | - Disallow: /rooms/*/photos | |
| 725 | - Disallow: /rooms/*/reviews | |
| 726 | - Disallow: /rooms/*/safety | |
| 727 | - Disallow: /rooms/*?viralityEntryPoint | |
| 728 | - Disallow: /rooms/*/cancellation-policy | |
| 729 | - Disallow: /rooms/*/description | |
| 730 | - Disallow: /s/guidebooks | |
| 731 | - Disallow: /signed_out_modal.json | |
| 732 | - Disallow: /signup_modal | |
| 733 | - Disallow: /stories | |
| 734 | - Disallow: /trips/upcoming | |
| 735 | - Disallow: /trips/v1/ | |
| 736 | - Disallow: /update-your-browser | |
| 737 | - Disallow: /users/*/listings | |
| 738 | - Disallow: /users/show | |
| 739 | - Disallow: /users/profile | |
| 740 | - Disallow: /api/v3/LandingPageStaysQuery | |
| 741 | - Disallow: /s/*/* | |
| 742 | - Disallow: /things-to-do/places | |
| 743 | - Disallow: /rooms/ | |
| 744 | - Disallow: /experiences/ | |
| 745 | - Disallow: /help/ | |
| 746 | - Disallow: /*/things-to-do | |
| 747 | - | |
| 748 | 614 | User-agent: cohere-ai |
| 749 | 615 | Allow: /calendar/ical/ |
| 750 | 616 | Allow: /.well-known/amphtml/apikey.pub |
| @@ -820,6 +686,30 @@ | ||
| 820 | 686 | Allow: /.well-known/amphtml/apikey.pub |
| 821 | 687 | Disallow: / |
| 822 | 688 | |
| 689 | + | |
| 690 | + User-agent: GPTBot | |
| 691 | + Disallow: / | |
| 692 | + | |
| 693 | + | |
| 694 | + User-agent: ClaudeBot | |
| 695 | + Disallow: / | |
| 696 | + | |
| 697 | + | |
| 698 | + User-agent: Applebot-Extended | |
| 699 | + Disallow: / | |
| 700 | + | |
| 701 | + | |
| 702 | + User-agent: Webzio-Extended | |
| 703 | + Disallow: / | |
| 704 | + | |
| 705 | + | |
| 706 | + User-agent: AI2Bot | |
| 707 | + Disallow: / | |
| 708 | + | |
| 709 | + | |
| 710 | + User-agent: MistralAI-Training | |
| 711 | + Disallow: / | |
| 712 | + | |
| 823 | 713 | User-agent: * |
| 824 | 714 | Allow: /calendar/ical/ |
| 825 | 715 | Allow: /.well-known/amphtml/apikey.pub |