Facebook / robots.txt
← back to facebook.com · change detected 19d ago (2026-10-18T13:46:13Z) · modified +18 −282
What changed
- 🤖 AIBlocked Amazonbot (Amazon) entirely
- 🤖 AIRemoved Allow: /ajax/bootloader-endpoint/ for Amazonbot
- 🤖 AIRemoved Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet for Amazonbot
- 🤖 AIRemoved Allow: /careers/ for Amazonbot
- 🤖 AIRemoved Allow: /safetycheck/ for Amazonbot
- 🤖 AIRemoved Disallow: /*/plugins/* for Amazonbot
- 🤖 AIRemoved Disallow: /?*next= for Amazonbot
- 🤖 AIRemoved Disallow: /a/bz? for Amazonbot
- 🤖 AIRemoved Disallow: /ajax/ for Amazonbot
- 🤖 AIRemoved Disallow: /album.php for Amazonbot
- 🤖 AIRemoved Disallow: /business/*&categories for Amazonbot
- 🤖 AIRemoved Disallow: /business/*?categories for Amazonbot
- 🤖 AIRemoved Disallow: /business/help/search*&query= for Amazonbot
- 🤖 AIRemoved Disallow: /business/help/search*?query= for Amazonbot
- 🤖 AIRemoved Disallow: /business/news*&tags for Amazonbot
- 🤖 AIRemoved Disallow: /business/news*?tags for Amazonbot
- 🤖 AIRemoved Disallow: /business/search*&q= for Amazonbot
- 🤖 AIRemoved Disallow: /business/search*?q= for Amazonbot
- 🤖 AIRemoved Disallow: /checkpoint/ for Amazonbot
- 🤖 AIRemoved Disallow: /contact_importer/ for Amazonbot
- 🤖 AIRemoved Disallow: /dialog/ for Amazonbot
- 🤖 AIRemoved Disallow: /fbml/ajax/dialog/ for Amazonbot
- 🤖 AIRemoved Disallow: /feeds/ for Amazonbot
- 🤖 AIRemoved Disallow: /file_download.php for Amazonbot
- 🤖 AIRemoved Disallow: /job_application/ for Amazonbot
- 🤖 AIRemoved Disallow: /l.php for Amazonbot
- 🤖 AIRemoved Disallow: /login.php*&next= for Amazonbot
- 🤖 AIRemoved Disallow: /login.php/?next= for Amazonbot
- 🤖 AIRemoved Disallow: /login.php?next= for Amazonbot
- 🤖 AIRemoved Disallow: /login/*&next= for Amazonbot
- 🤖 AIRemoved Disallow: /login/?next= for Amazonbot
- 🤖 AIRemoved Disallow: /login/device-based/regular/login/*&next= for Amazonbot
- 🤖 AIRemoved Disallow: /login/device-based/regular/login/?next= for Amazonbot
- 🤖 AIRemoved Disallow: /meta-ai-answers/ for Amazonbot
- 🤖 AIRemoved Disallow: /moments_app/ for Amazonbot
- 🤖 AIRemoved Disallow: /p.php for Amazonbot
- 🤖 AIRemoved Disallow: /photos.php for Amazonbot
- 🤖 AIRemoved Disallow: /plugins/ for Amazonbot
- 🤖 AIRemoved Disallow: /share.php for Amazonbot
- 🤖 AIRemoved Disallow: /share/ for Amazonbot
- 🤖 AIRemoved Disallow: /sharer.php for Amazonbot
- 🤖 AIRemoved Disallow: /sharer/ for Amazonbot
- 🤖 AIRemoved Disallow: /tr/ for Amazonbot
- 🤖 AIRemoved Disallow: /tr? for Amazonbot
- 🤖 AIRemoved Disallow: /ufi/reaction/profile/browser/ for Amazonbot
- 🤖 AIRemoved Disallow: /x/oauth/ for Amazonbot
- 🤖 AIBlocked Applebot-Extended (Apple) entirely
- 🤖 AIRemoved Allow: /ajax/bootloader-endpoint/ for Applebot-Extended
- 🤖 AIRemoved Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet for Applebot-Extended
- 🤖 AIRemoved Allow: /careers/ for Applebot-Extended
- ... and 226 more change(s)
Diff old (2026-10-15T06:25:59Z) → new (2026-10-18T13:46:13Z)
| @@ -5,7 +5,25 @@ | ||
| 5 | 5 | # All authorized user-agents listed on this page must comply with Meta’s |
| 6 | 6 | # Automated Data Collection Terms available at: |
| 7 | 7 | # https://www.facebook.com/legal/automated_data_collection_terms |
| 8 | + User-agent: Amazonbot | |
| 9 | + Disallow: / | |
| 10 | + | |
| 11 | + User-agent: Applebot-Extended | |
| 12 | + Disallow: / | |
| 13 | + | |
| 14 | + User-agent: Brightbot | |
| 15 | + Disallow: / | |
| 16 | + | |
| 17 | + User-agent: ClaudeBot | |
| 18 | + Disallow: / | |
| 19 | + | |
| 8 | 20 | User-agent: Google-Extended |
| 21 | + Disallow: / | |
| 22 | + | |
| 23 | + User-agent: GPTBot | |
| 24 | + Disallow: / | |
| 25 | + | |
| 26 | + User-agent: PerplexityBot | |
| 9 | 27 | Disallow: / |
| 10 | 28 | |
| 11 | 29 | User-agent: PetalBot |
| @@ -1022,288 +1040,6 @@ | ||
| 1022 | 1040 | Allow: /careers/ |
| 1023 | 1041 | Allow: /safetycheck/ |
| 1024 | 1042 | |
| 1025 | - User-agent: Amazonbot | |
| 1026 | - Disallow: /*/plugins/* | |
| 1027 | - Disallow: /?*next= | |
| 1028 | - Disallow: /a/bz? | |
| 1029 | - Disallow: /ajax/ | |
| 1030 | - Disallow: /album.php | |
| 1031 | - Disallow: /business/*&categories | |
| 1032 | - Disallow: /business/*?categories | |
| 1033 | - Disallow: /business/help/search*&query= | |
| 1034 | - Disallow: /business/help/search*?query= | |
| 1035 | - Disallow: /business/news*&tags | |
| 1036 | - Disallow: /business/news*?tags | |
| 1037 | - Disallow: /business/search*&q= | |
| 1038 | - Disallow: /business/search*?q= | |
| 1039 | - Disallow: /checkpoint/ | |
| 1040 | - Disallow: /contact_importer/ | |
| 1041 | - Disallow: /dialog/ | |
| 1042 | - Disallow: /fbml/ajax/dialog/ | |
| 1043 | - Disallow: /feeds/ | |
| 1044 | - Disallow: /file_download.php | |
| 1045 | - Disallow: /job_application/ | |
| 1046 | - Disallow: /l.php | |
| 1047 | - Disallow: /login.php*&next= | |
| 1048 | - Disallow: /login.php/?next= | |
| 1049 | - Disallow: /login.php?next= | |
| 1050 | - Disallow: /login/*&next= | |
| 1051 | - Disallow: /login/?next= | |
| 1052 | - Disallow: /login/device-based/regular/login/*&next= | |
| 1053 | - Disallow: /login/device-based/regular/login/?next= | |
| 1054 | - Disallow: /meta-ai-answers/ | |
| 1055 | - Disallow: /moments_app/ | |
| 1056 | - Disallow: /p.php | |
| 1057 | - Disallow: /photos.php | |
| 1058 | - Disallow: /plugins/ | |
| 1059 | - Disallow: /share.php | |
| 1060 | - Disallow: /share/ | |
| 1061 | - Disallow: /sharer.php | |
| 1062 | - Disallow: /sharer/ | |
| 1063 | - Disallow: /tr/ | |
| 1064 | - Disallow: /tr? | |
| 1065 | - Disallow: /ufi/reaction/profile/browser/ | |
| 1066 | - Disallow: /x/oauth/ | |
| 1067 | - Allow: /ajax/bootloader-endpoint/ | |
| 1068 | - Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet | |
| 1069 | - Allow: /careers/ | |
| 1070 | - Allow: /safetycheck/ | |
| 1071 | - | |
| 1072 | - User-agent: Applebot-Extended | |
| 1073 | - Disallow: /*/plugins/* | |
| 1074 | - Disallow: /?*next= | |
| 1075 | - Disallow: /a/bz? | |
| 1076 | - Disallow: /ajax/ | |
| 1077 | - Disallow: /album.php | |
| 1078 | - Disallow: /business/*&categories | |
| 1079 | - Disallow: /business/*?categories | |
| 1080 | - Disallow: /business/help/search*&query= | |
| 1081 | - Disallow: /business/help/search*?query= | |
| 1082 | - Disallow: /business/news*&tags | |
| 1083 | - Disallow: /business/news*?tags | |
| 1084 | - Disallow: /business/search*&q= | |
| 1085 | - Disallow: /business/search*?q= | |
| 1086 | - Disallow: /checkpoint/ | |
| 1087 | - Disallow: /contact_importer/ | |
| 1088 | - Disallow: /dialog/ | |
| 1089 | - Disallow: /fbml/ajax/dialog/ | |
| 1090 | - Disallow: /feeds/ | |
| 1091 | - Disallow: /file_download.php | |
| 1092 | - Disallow: /job_application/ | |
| 1093 | - Disallow: /l.php | |
| 1094 | - Disallow: /login.php*&next= | |
| 1095 | - Disallow: /login.php/?next= | |
| 1096 | - Disallow: /login.php?next= | |
| 1097 | - Disallow: /login/*&next= | |
| 1098 | - Disallow: /login/?next= | |
| 1099 | - Disallow: /login/device-based/regular/login/*&next= | |
| 1100 | - Disallow: /login/device-based/regular/login/?next= | |
| 1101 | - Disallow: /meta-ai-answers/ | |
| 1102 | - Disallow: /moments_app/ | |
| 1103 | - Disallow: /p.php | |
| 1104 | - Disallow: /photos.php | |
| 1105 | - Disallow: /plugins/ | |
| 1106 | - Disallow: /share.php | |
| 1107 | - Disallow: /share/ | |
| 1108 | - Disallow: /sharer.php | |
| 1109 | - Disallow: /sharer/ | |
| 1110 | - Disallow: /tr/ | |
| 1111 | - Disallow: /tr? | |
| 1112 | - Disallow: /ufi/reaction/profile/browser/ | |
| 1113 | - Disallow: /x/oauth/ | |
| 1114 | - Allow: /ajax/bootloader-endpoint/ | |
| 1115 | - Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet | |
| 1116 | - Allow: /careers/ | |
| 1117 | - Allow: /safetycheck/ | |
| 1118 | - | |
| 1119 | - User-agent: Brightbot | |
| 1120 | - Disallow: /*/plugins/* | |
| 1121 | - Disallow: /?*next= | |
| 1122 | - Disallow: /a/bz? | |
| 1123 | - Disallow: /ajax/ | |
| 1124 | - Disallow: /album.php | |
| 1125 | - Disallow: /business/*&categories | |
| 1126 | - Disallow: /business/*?categories | |
| 1127 | - Disallow: /business/help/search*&query= | |
| 1128 | - Disallow: /business/help/search*?query= | |
| 1129 | - Disallow: /business/news*&tags | |
| 1130 | - Disallow: /business/news*?tags | |
| 1131 | - Disallow: /business/search*&q= | |
| 1132 | - Disallow: /business/search*?q= | |
| 1133 | - Disallow: /checkpoint/ | |
| 1134 | - Disallow: /contact_importer/ | |
| 1135 | - Disallow: /dialog/ | |
| 1136 | - Disallow: /fbml/ajax/dialog/ | |
| 1137 | - Disallow: /feeds/ | |
| 1138 | - Disallow: /file_download.php | |
| 1139 | - Disallow: /job_application/ | |
| 1140 | - Disallow: /l.php | |
| 1141 | - Disallow: /login.php*&next= | |
| 1142 | - Disallow: /login.php/?next= | |
| 1143 | - Disallow: /login.php?next= | |
| 1144 | - Disallow: /login/*&next= | |
| 1145 | - Disallow: /login/?next= | |
| 1146 | - Disallow: /login/device-based/regular/login/*&next= | |
| 1147 | - Disallow: /login/device-based/regular/login/?next= | |
| 1148 | - Disallow: /meta-ai-answers/ | |
| 1149 | - Disallow: /moments_app/ | |
| 1150 | - Disallow: /p.php | |
| 1151 | - Disallow: /photos.php | |
| 1152 | - Disallow: /plugins/ | |
| 1153 | - Disallow: /share.php | |
| 1154 | - Disallow: /share/ | |
| 1155 | - Disallow: /sharer.php | |
| 1156 | - Disallow: /sharer/ | |
| 1157 | - Disallow: /tr/ | |
| 1158 | - Disallow: /tr? | |
| 1159 | - Disallow: /ufi/reaction/profile/browser/ | |
| 1160 | - Disallow: /x/oauth/ | |
| 1161 | - Allow: /ajax/bootloader-endpoint/ | |
| 1162 | - Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet | |
| 1163 | - Allow: /careers/ | |
| 1164 | - Allow: /safetycheck/ | |
| 1165 | - | |
| 1166 | - User-agent: ClaudeBot | |
| 1167 | - Disallow: /*/plugins/* | |
| 1168 | - Disallow: /?*next= | |
| 1169 | - Disallow: /a/bz? | |
| 1170 | - Disallow: /ajax/ | |
| 1171 | - Disallow: /album.php | |
| 1172 | - Disallow: /business/*&categories | |
| 1173 | - Disallow: /business/*?categories | |
| 1174 | - Disallow: /business/help/search*&query= | |
| 1175 | - Disallow: /business/help/search*?query= | |
| 1176 | - Disallow: /business/news*&tags | |
| 1177 | - Disallow: /business/news*?tags | |
| 1178 | - Disallow: /business/search*&q= | |
| 1179 | - Disallow: /business/search*?q= | |
| 1180 | - Disallow: /checkpoint/ | |
| 1181 | - Disallow: /contact_importer/ | |
| 1182 | - Disallow: /dialog/ | |
| 1183 | - Disallow: /fbml/ajax/dialog/ | |
| 1184 | - Disallow: /feeds/ | |
| 1185 | - Disallow: /file_download.php | |
| 1186 | - Disallow: /job_application/ | |
| 1187 | - Disallow: /l.php | |
| 1188 | - Disallow: /login.php*&next= | |
| 1189 | - Disallow: /login.php/?next= | |
| 1190 | - Disallow: /login.php?next= | |
| 1191 | - Disallow: /login/*&next= | |
| 1192 | - Disallow: /login/?next= | |
| 1193 | - Disallow: /login/device-based/regular/login/*&next= | |
| 1194 | - Disallow: /login/device-based/regular/login/?next= | |
| 1195 | - Disallow: /meta-ai-answers/ | |
| 1196 | - Disallow: /moments_app/ | |
| 1197 | - Disallow: /p.php | |
| 1198 | - Disallow: /photos.php | |
| 1199 | - Disallow: /plugins/ | |
| 1200 | - Disallow: /share.php | |
| 1201 | - Disallow: /share/ | |
| 1202 | - Disallow: /sharer.php | |
| 1203 | - Disallow: /sharer/ | |
| 1204 | - Disallow: /tr/ | |
| 1205 | - Disallow: /tr? | |
| 1206 | - Disallow: /ufi/reaction/profile/browser/ | |
| 1207 | - Disallow: /x/oauth/ | |
| 1208 | - Allow: /ajax/bootloader-endpoint/ | |
| 1209 | - Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet | |
| 1210 | - Allow: /careers/ | |
| 1211 | - Allow: /safetycheck/ | |
| 1212 | - | |
| 1213 | - User-agent: GPTBot | |
| 1214 | - Disallow: /*/plugins/* | |
| 1215 | - Disallow: /?*next= | |
| 1216 | - Disallow: /a/bz? | |
| 1217 | - Disallow: /ajax/ | |
| 1218 | - Disallow: /album.php | |
| 1219 | - Disallow: /business/*&categories | |
| 1220 | - Disallow: /business/*?categories | |
| 1221 | - Disallow: /business/help/search*&query= | |
| 1222 | - Disallow: /business/help/search*?query= | |
| 1223 | - Disallow: /business/news*&tags | |
| 1224 | - Disallow: /business/news*?tags | |
| 1225 | - Disallow: /business/search*&q= | |
| 1226 | - Disallow: /business/search*?q= | |
| 1227 | - Disallow: /checkpoint/ | |
| 1228 | - Disallow: /contact_importer/ | |
| 1229 | - Disallow: /dialog/ | |
| 1230 | - Disallow: /fbml/ajax/dialog/ | |
| 1231 | - Disallow: /feeds/ | |
| 1232 | - Disallow: /file_download.php | |
| 1233 | - Disallow: /job_application/ | |
| 1234 | - Disallow: /l.php | |
| 1235 | - Disallow: /login.php*&next= | |
| 1236 | - Disallow: /login.php/?next= | |
| 1237 | - Disallow: /login.php?next= | |
| 1238 | - Disallow: /login/*&next= | |
| 1239 | - Disallow: /login/?next= | |
| 1240 | - Disallow: /login/device-based/regular/login/*&next= | |
| 1241 | - Disallow: /login/device-based/regular/login/?next= | |
| 1242 | - Disallow: /meta-ai-answers/ | |
| 1243 | - Disallow: /moments_app/ | |
| 1244 | - Disallow: /p.php | |
| 1245 | - Disallow: /photos.php | |
| 1246 | - Disallow: /plugins/ | |
| 1247 | - Disallow: /share.php | |
| 1248 | - Disallow: /share/ | |
| 1249 | - Disallow: /sharer.php | |
| 1250 | - Disallow: /sharer/ | |
| 1251 | - Disallow: /tr/ | |
| 1252 | - Disallow: /tr? | |
| 1253 | - Disallow: /ufi/reaction/profile/browser/ | |
| 1254 | - Disallow: /x/oauth/ | |
| 1255 | - Allow: /ajax/bootloader-endpoint/ | |
| 1256 | - Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet | |
| 1257 | - Allow: /careers/ | |
| 1258 | - Allow: /safetycheck/ | |
| 1259 | - | |
| 1260 | - User-agent: PerplexityBot | |
| 1261 | - Disallow: /*/plugins/* | |
| 1262 | - Disallow: /?*next= | |
| 1263 | - Disallow: /a/bz? | |
| 1264 | - Disallow: /ajax/ | |
| 1265 | - Disallow: /album.php | |
| 1266 | - Disallow: /business/*&categories | |
| 1267 | - Disallow: /business/*?categories | |
| 1268 | - Disallow: /business/help/search*&query= | |
| 1269 | - Disallow: /business/help/search*?query= | |
| 1270 | - Disallow: /business/news*&tags | |
| 1271 | - Disallow: /business/news*?tags | |
| 1272 | - Disallow: /business/search*&q= | |
| 1273 | - Disallow: /business/search*?q= | |
| 1274 | - Disallow: /checkpoint/ | |
| 1275 | - Disallow: /contact_importer/ | |
| 1276 | - Disallow: /dialog/ | |
| 1277 | - Disallow: /fbml/ajax/dialog/ | |
| 1278 | - Disallow: /feeds/ | |
| 1279 | - Disallow: /file_download.php | |
| 1280 | - Disallow: /job_application/ | |
| 1281 | - Disallow: /l.php | |
| 1282 | - Disallow: /login.php*&next= | |
| 1283 | - Disallow: /login.php/?next= | |
| 1284 | - Disallow: /login.php?next= | |
| 1285 | - Disallow: /login/*&next= | |
| 1286 | - Disallow: /login/?next= | |
| 1287 | - Disallow: /login/device-based/regular/login/*&next= | |
| 1288 | - Disallow: /login/device-based/regular/login/?next= | |
| 1289 | - Disallow: /meta-ai-answers/ | |
| 1290 | - Disallow: /moments_app/ | |
| 1291 | - Disallow: /p.php | |
| 1292 | - Disallow: /photos.php | |
| 1293 | - Disallow: /plugins/ | |
| 1294 | - Disallow: /share.php | |
| 1295 | - Disallow: /share/ | |
| 1296 | - Disallow: /sharer.php | |
| 1297 | - Disallow: /sharer/ | |
| 1298 | - Disallow: /tr/ | |
| 1299 | - Disallow: /tr? | |
| 1300 | - Disallow: /ufi/reaction/profile/browser/ | |
| 1301 | - Disallow: /x/oauth/ | |
| 1302 | - Allow: /ajax/bootloader-endpoint/ | |
| 1303 | - Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet | |
| 1304 | - Allow: /careers/ | |
| 1305 | - Allow: /safetycheck/ | |
| 1306 | - | |
| 1307 | 1043 | User-agent: * |
| 1308 | 1044 | Disallow: / |
| 1309 | 1045 |