NeuralCrawl

Facebook / robots.txt

← back to facebook.com · change detected 2d ago (2026-08-02T18:01:38Z) · modified +329 −21

What changed

Diff old (2026-07-27T15:48:23Z) → new (2026-08-02T18:01:38Z)

@@ -5,27 +5,6 @@
55  # All authorized user-agents listed on this page must comply with Meta’s
66  # Automated Data Collection Terms available at:
77  # https://www.facebook.com/legal/automated_data_collection_terms
8- User-agent: Amazonbot
9- Disallow: /
10-
11- User-agent: Applebot-Extended
12- Disallow: /
13-
14- User-agent: Brightbot
15- Disallow: /
16-
17- User-agent: ClaudeBot
18- Disallow: /
19-
20- User-agent: Google-Extended
21- Disallow: /
22-
23- User-agent: GPTBot
24- Disallow: /
25-
26- User-agent: PerplexityBot
27- Disallow: /
28-
298  User-agent: PetalBot
309  Disallow: /
3110 
@@ -942,6 +921,335 @@
942921  Allow: /careers/
943922  Allow: /safetycheck/
944923 
924+ User-agent: Amazonbot
925+ Disallow: /*/plugins/*
926+ Disallow: /?*next=
927+ Disallow: /a/bz?
928+ Disallow: /ajax/
929+ Disallow: /album.php
930+ Disallow: /business/*&categories
931+ Disallow: /business/*?categories
932+ Disallow: /business/help/search*&query=
933+ Disallow: /business/help/search*?query=
934+ Disallow: /business/news*&tags
935+ Disallow: /business/news*?tags
936+ Disallow: /business/search*&q=
937+ Disallow: /business/search*?q=
938+ Disallow: /checkpoint/
939+ Disallow: /contact_importer/
940+ Disallow: /dialog/
941+ Disallow: /fbml/ajax/dialog/
942+ Disallow: /feeds/
943+ Disallow: /file_download.php
944+ Disallow: /job_application/
945+ Disallow: /l.php
946+ Disallow: /login.php*&next=
947+ Disallow: /login.php/?next=
948+ Disallow: /login.php?next=
949+ Disallow: /login/*&next=
950+ Disallow: /login/?next=
951+ Disallow: /login/device-based/regular/login/*&next=
952+ Disallow: /login/device-based/regular/login/?next=
953+ Disallow: /meta-ai-answers/
954+ Disallow: /moments_app/
955+ Disallow: /p.php
956+ Disallow: /photos.php
957+ Disallow: /plugins/
958+ Disallow: /share.php
959+ Disallow: /share/
960+ Disallow: /sharer.php
961+ Disallow: /sharer/
962+ Disallow: /tr/
963+ Disallow: /tr?
964+ Disallow: /ufi/reaction/profile/browser/
965+ Disallow: /x/oauth/
966+ Allow: /ajax/bootloader-endpoint/
967+ Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
968+ Allow: /careers/
969+ Allow: /safetycheck/
970+
971+ User-agent: Applebot-Extended
972+ Disallow: /*/plugins/*
973+ Disallow: /?*next=
974+ Disallow: /a/bz?
975+ Disallow: /ajax/
976+ Disallow: /album.php
977+ Disallow: /business/*&categories
978+ Disallow: /business/*?categories
979+ Disallow: /business/help/search*&query=
980+ Disallow: /business/help/search*?query=
981+ Disallow: /business/news*&tags
982+ Disallow: /business/news*?tags
983+ Disallow: /business/search*&q=
984+ Disallow: /business/search*?q=
985+ Disallow: /checkpoint/
986+ Disallow: /contact_importer/
987+ Disallow: /dialog/
988+ Disallow: /fbml/ajax/dialog/
989+ Disallow: /feeds/
990+ Disallow: /file_download.php
991+ Disallow: /job_application/
992+ Disallow: /l.php
993+ Disallow: /login.php*&next=
994+ Disallow: /login.php/?next=
995+ Disallow: /login.php?next=
996+ Disallow: /login/*&next=
997+ Disallow: /login/?next=
998+ Disallow: /login/device-based/regular/login/*&next=
999+ Disallow: /login/device-based/regular/login/?next=
1000+ Disallow: /meta-ai-answers/
1001+ Disallow: /moments_app/
1002+ Disallow: /p.php
1003+ Disallow: /photos.php
1004+ Disallow: /plugins/
1005+ Disallow: /share.php
1006+ Disallow: /share/
1007+ Disallow: /sharer.php
1008+ Disallow: /sharer/
1009+ Disallow: /tr/
1010+ Disallow: /tr?
1011+ Disallow: /ufi/reaction/profile/browser/
1012+ Disallow: /x/oauth/
1013+ Allow: /ajax/bootloader-endpoint/
1014+ Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
1015+ Allow: /careers/
1016+ Allow: /safetycheck/
1017+
1018+ User-agent: Brightbot
1019+ Disallow: /*/plugins/*
1020+ Disallow: /?*next=
1021+ Disallow: /a/bz?
1022+ Disallow: /ajax/
1023+ Disallow: /album.php
1024+ Disallow: /business/*&categories
1025+ Disallow: /business/*?categories
1026+ Disallow: /business/help/search*&query=
1027+ Disallow: /business/help/search*?query=
1028+ Disallow: /business/news*&tags
1029+ Disallow: /business/news*?tags
1030+ Disallow: /business/search*&q=
1031+ Disallow: /business/search*?q=
1032+ Disallow: /checkpoint/
1033+ Disallow: /contact_importer/
1034+ Disallow: /dialog/
1035+ Disallow: /fbml/ajax/dialog/
1036+ Disallow: /feeds/
1037+ Disallow: /file_download.php
1038+ Disallow: /job_application/
1039+ Disallow: /l.php
1040+ Disallow: /login.php*&next=
1041+ Disallow: /login.php/?next=
1042+ Disallow: /login.php?next=
1043+ Disallow: /login/*&next=
1044+ Disallow: /login/?next=
1045+ Disallow: /login/device-based/regular/login/*&next=
1046+ Disallow: /login/device-based/regular/login/?next=
1047+ Disallow: /meta-ai-answers/
1048+ Disallow: /moments_app/
1049+ Disallow: /p.php
1050+ Disallow: /photos.php
1051+ Disallow: /plugins/
1052+ Disallow: /share.php
1053+ Disallow: /share/
1054+ Disallow: /sharer.php
1055+ Disallow: /sharer/
1056+ Disallow: /tr/
1057+ Disallow: /tr?
1058+ Disallow: /ufi/reaction/profile/browser/
1059+ Disallow: /x/oauth/
1060+ Allow: /ajax/bootloader-endpoint/
1061+ Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
1062+ Allow: /careers/
1063+ Allow: /safetycheck/
1064+
1065+ User-agent: ClaudeBot
1066+ Disallow: /*/plugins/*
1067+ Disallow: /?*next=
1068+ Disallow: /a/bz?
1069+ Disallow: /ajax/
1070+ Disallow: /album.php
1071+ Disallow: /business/*&categories
1072+ Disallow: /business/*?categories
1073+ Disallow: /business/help/search*&query=
1074+ Disallow: /business/help/search*?query=
1075+ Disallow: /business/news*&tags
1076+ Disallow: /business/news*?tags
1077+ Disallow: /business/search*&q=
1078+ Disallow: /business/search*?q=
1079+ Disallow: /checkpoint/
1080+ Disallow: /contact_importer/
1081+ Disallow: /dialog/
1082+ Disallow: /fbml/ajax/dialog/
1083+ Disallow: /feeds/
1084+ Disallow: /file_download.php
1085+ Disallow: /job_application/
1086+ Disallow: /l.php
1087+ Disallow: /login.php*&next=
1088+ Disallow: /login.php/?next=
1089+ Disallow: /login.php?next=
1090+ Disallow: /login/*&next=
1091+ Disallow: /login/?next=
1092+ Disallow: /login/device-based/regular/login/*&next=
1093+ Disallow: /login/device-based/regular/login/?next=
1094+ Disallow: /meta-ai-answers/
1095+ Disallow: /moments_app/
1096+ Disallow: /p.php
1097+ Disallow: /photos.php
1098+ Disallow: /plugins/
1099+ Disallow: /share.php
1100+ Disallow: /share/
1101+ Disallow: /sharer.php
1102+ Disallow: /sharer/
1103+ Disallow: /tr/
1104+ Disallow: /tr?
1105+ Disallow: /ufi/reaction/profile/browser/
1106+ Disallow: /x/oauth/
1107+ Allow: /ajax/bootloader-endpoint/
1108+ Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
1109+ Allow: /careers/
1110+ Allow: /safetycheck/
1111+
1112+ User-agent: Google-Extended
1113+ Disallow: /*/plugins/*
1114+ Disallow: /?*next=
1115+ Disallow: /a/bz?
1116+ Disallow: /ajax/
1117+ Disallow: /album.php
1118+ Disallow: /business/*&categories
1119+ Disallow: /business/*?categories
1120+ Disallow: /business/help/search*&query=
1121+ Disallow: /business/help/search*?query=
1122+ Disallow: /business/news*&tags
1123+ Disallow: /business/news*?tags
1124+ Disallow: /business/search*&q=
1125+ Disallow: /business/search*?q=
1126+ Disallow: /checkpoint/
1127+ Disallow: /contact_importer/
1128+ Disallow: /dialog/
1129+ Disallow: /fbml/ajax/dialog/
1130+ Disallow: /feeds/
1131+ Disallow: /file_download.php
1132+ Disallow: /job_application/
1133+ Disallow: /l.php
1134+ Disallow: /login.php*&next=
1135+ Disallow: /login.php/?next=
1136+ Disallow: /login.php?next=
1137+ Disallow: /login/*&next=
1138+ Disallow: /login/?next=
1139+ Disallow: /login/device-based/regular/login/*&next=
1140+ Disallow: /login/device-based/regular/login/?next=
1141+ Disallow: /meta-ai-answers/
1142+ Disallow: /moments_app/
1143+ Disallow: /p.php
1144+ Disallow: /photos.php
1145+ Disallow: /plugins/
1146+ Disallow: /share.php
1147+ Disallow: /share/
1148+ Disallow: /sharer.php
1149+ Disallow: /sharer/
1150+ Disallow: /tr/
1151+ Disallow: /tr?
1152+ Disallow: /ufi/reaction/profile/browser/
1153+ Disallow: /x/oauth/
1154+ Allow: /ajax/bootloader-endpoint/
1155+ Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
1156+ Allow: /careers/
1157+ Allow: /safetycheck/
1158+
1159+ User-agent: GPTBot
1160+ Disallow: /*/plugins/*
1161+ Disallow: /?*next=
1162+ Disallow: /a/bz?
1163+ Disallow: /ajax/
1164+ Disallow: /album.php
1165+ Disallow: /business/*&categories
1166+ Disallow: /business/*?categories
1167+ Disallow: /business/help/search*&query=
1168+ Disallow: /business/help/search*?query=
1169+ Disallow: /business/news*&tags
1170+ Disallow: /business/news*?tags
1171+ Disallow: /business/search*&q=
1172+ Disallow: /business/search*?q=
1173+ Disallow: /checkpoint/
1174+ Disallow: /contact_importer/
1175+ Disallow: /dialog/
1176+ Disallow: /fbml/ajax/dialog/
1177+ Disallow: /feeds/
1178+ Disallow: /file_download.php
1179+ Disallow: /job_application/
1180+ Disallow: /l.php
1181+ Disallow: /login.php*&next=
1182+ Disallow: /login.php/?next=
1183+ Disallow: /login.php?next=
1184+ Disallow: /login/*&next=
1185+ Disallow: /login/?next=
1186+ Disallow: /login/device-based/regular/login/*&next=
1187+ Disallow: /login/device-based/regular/login/?next=
1188+ Disallow: /meta-ai-answers/
1189+ Disallow: /moments_app/
1190+ Disallow: /p.php
1191+ Disallow: /photos.php
1192+ Disallow: /plugins/
1193+ Disallow: /share.php
1194+ Disallow: /share/
1195+ Disallow: /sharer.php
1196+ Disallow: /sharer/
1197+ Disallow: /tr/
1198+ Disallow: /tr?
1199+ Disallow: /ufi/reaction/profile/browser/
1200+ Disallow: /x/oauth/
1201+ Allow: /ajax/bootloader-endpoint/
1202+ Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
1203+ Allow: /careers/
1204+ Allow: /safetycheck/
1205+
1206+ User-agent: PerplexityBot
1207+ Disallow: /*/plugins/*
1208+ Disallow: /?*next=
1209+ Disallow: /a/bz?
1210+ Disallow: /ajax/
1211+ Disallow: /album.php
1212+ Disallow: /business/*&categories
1213+ Disallow: /business/*?categories
1214+ Disallow: /business/help/search*&query=
1215+ Disallow: /business/help/search*?query=
1216+ Disallow: /business/news*&tags
1217+ Disallow: /business/news*?tags
1218+ Disallow: /business/search*&q=
1219+ Disallow: /business/search*?q=
1220+ Disallow: /checkpoint/
1221+ Disallow: /contact_importer/
1222+ Disallow: /dialog/
1223+ Disallow: /fbml/ajax/dialog/
1224+ Disallow: /feeds/
1225+ Disallow: /file_download.php
1226+ Disallow: /job_application/
1227+ Disallow: /l.php
1228+ Disallow: /login.php*&next=
1229+ Disallow: /login.php/?next=
1230+ Disallow: /login.php?next=
1231+ Disallow: /login/*&next=
1232+ Disallow: /login/?next=
1233+ Disallow: /login/device-based/regular/login/*&next=
1234+ Disallow: /login/device-based/regular/login/?next=
1235+ Disallow: /meta-ai-answers/
1236+ Disallow: /moments_app/
1237+ Disallow: /p.php
1238+ Disallow: /photos.php
1239+ Disallow: /plugins/
1240+ Disallow: /share.php
1241+ Disallow: /share/
1242+ Disallow: /sharer.php
1243+ Disallow: /sharer/
1244+ Disallow: /tr/
1245+ Disallow: /tr?
1246+ Disallow: /ufi/reaction/profile/browser/
1247+ Disallow: /x/oauth/
1248+ Allow: /ajax/bootloader-endpoint/
1249+ Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
1250+ Allow: /careers/
1251+ Allow: /safetycheck/
1252+
9451253  User-agent: *
9461254  Disallow: /
9471255