Common Crawl / robots.txt
← back to commoncrawl.org · change detected 1mo ago (2026-06-24T03:58:34Z) · modified +48 −0
What changed
- Formatting/comment-only change (no semantic difference)
Diff old (2026-06-20T14:56:27Z) → new (2026-06-24T03:58:34Z)
| @@ -4,4 +4,52 @@ | ||
| 4 | 4 | Allow: / |
| 5 | 5 | Disallow: /search?* |
| 6 | 6 | |
| 7 | + # | |
| 8 | + # ^~. | |
| 9 | + # :5P5! | |
| 10 | + # .5PPGJ: | |
| 11 | + # ?PPPG5! | |
| 12 | + # ~PPPPGGY^ | |
| 13 | + # .YP55PGGGJ: | |
| 14 | + # JP5PPPGGBP?^ | |
| 15 | + # 7P5PGPPGGGGGY7: .:~!!~: | |
| 16 | + # 755PGGGGGGGBBB5!:. .^!YPGBBB#Y | |
| 17 | + # ^555PGGGGGBBBGPGPY?^... .~JPBBBBBBB#B^ | |
| 18 | + # ~5PPPPGGGGGGPPPGGGGGPP5YYYJJJ?!~~. .:~7YGBBBBBBBB##B~ | |
| 19 | + # :!YPGPPPGPPPP555PPPPGGGGBBBBBBBGGG5JJJ!:: .:. ....:^!7J5GB##BBBBGBBBB#G^ | |
| 20 | + # .?PPPGGGGGGGGGP5555P5PGGBBGGPGP5GGGGGGGPP5YPPY?JPPPGGGB####BBBBBBBBBBB#G7^. | |
| 21 | + # .?PPPPGGGGBBBBBBGPPGG5PGGGPP55PPGBBGGGPPGGBBBGB############BBBGGBB#BBB#5: | |
| 22 | + # ~GPGGBBBGGBBBBB#BBBBGPPGGP555PGGGGGGBBBBBGGBBBBB###########BBGBB###BB#J | |
| 23 | + # ~77JPGGGGGGGBBB########GPPGGPPPPG##BBB##BBBBBBBBBBB###########BBBBBBBB#B! | |
| 24 | + # .JGPPPGGGGGGGBBB#######BGPPPP5PGB##########BBBBBBBBBB##BBBBB###BBBBBBBBP^ | |
| 25 | + # !P55PPPPGGG5YPGB###BBBBG55PPPPGB###&&######BBBBBBBBB#######BBBBBB#BBB#5. | |
| 26 | + # 75Y555555GG7:^JYJPG55PPP5PPPPGB###&&&#####BBBGGGGBBB##############BBB#~ | |
| 27 | + # ~5YY555PPP5PGY?J^.^5PP555PPPPGB#####BB#BBBBGGGGGGGGBBBB###############5 | |
| 28 | + # .J555555PGGPPGBBGP55PP555PGGGGBBBB##BB##BPGGGPGGGGGGBBBBBB############B^ | |
| 29 | + # ~55555555P5PPGGBBBBBG55PGBBBBBBBBB##B#G5?^YBJ??YGBBBBBBBBB###########P~ | |
| 30 | + # .YP555555555555PBBBGPPBBBBBBBBBGGB##BBBG5JY5YYPGB##BB#############&&P. | |
| 31 | + # :YP555555555555GBBPPGBBBBBBBBBBGB####BBBGGPPGBBBBBBB################! | |
| 32 | + # !555PPPP555555PPPPGP5PPGGGGBB##B#####BGGPPGGBBBBBB#################B: | |
| 33 | + # ^555PPPPPPPPGBBBGB#GPPGPPPGGG##B###BBGBBBBBB######################B? | |
| 34 | + # :Y5P5PPPPPGB#########GPPPGBB###BBBBBBB############################B~ | |
| 35 | + # .J5PPPGGPGB##########BPGG&##BBBBB##BBBB##########################BB! | |
| 36 | + # .?55PPGGGBB#######BB#BPPGBBBBBB###################################GJ | |
| 37 | + # ^5PPGGGBBBBB#####BBBGGPGBGGB######BBBB##B#######################BGY^ | |
| 38 | + # 7GGGGGGBBBBBB#BBGBBB#BBBBBBB#######BB#BB#######################BBGGJ: | |
| 39 | + # .YBBBBBBBBBGGGGGPGBBGGPGBBBB#####BBBBB#########################BBBBBP7. | |
| 40 | + # :5BBBBBBBBGGGGG5GB#G57?J5PPGGGGGBBGB###########################BBBB#BY. | |
| 41 | + # ~G###BB#BBBGGG5PBBG?!7???????JP#GPG###B##################B###########P^ | |
| 42 | + # ^5B#B######BBGP5BB5!77??777!7JPGYPBB###BB####B###BB#######B###&&&#####G?. | |
| 43 | + # .?B#BB#######BBPYPG?!77777!!!?JP5PBB######BB######BBBBBBB##BB#&&&&&&#####P: | |
| 44 | + # :?GB##########BG55P!!!!!!!!!!?YY5BBBB#######B##########BB####&&&&&&&&&####P~ | |
| 45 | + # .!PB######&&###BG55?!!~~~~~~~!?55PBBB#########B############&&&&&&&&&&&&&&&&#GJ7!:. | |
| 46 | + # .!5B#B############GPJ!77~^^^~!7?PG5PBB#B############&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&#P7 | |
| 47 | + # .!5B################B5!7?7!~~!7?JP#GPBB#######&&&&&&##&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& | |
| 48 | + # .?GB################&B7!???!!77?JP&#BB########&&&&&&&&&#&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& | |
| 49 | + # 7GBB################&&BJ?J77J5PGG########&&&&&&&&&&&&&&&#&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& | |
| 50 | + # 7GGB###############&&&&&#G55GBB######B#&&&&&&&&&&&&&&&&&&&#&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& | |
| 51 | + #~PGB##################&#&&&##BBBB####&#&&&&&&&&&&&&&&&&&&&&##&&&&&&&&&&&&&&&&&&&&&&&&&&&&& | |
| 52 | + #!GGBBB##########&######&###&&&&##&&&&&#&&&&&&&&&&&&&&&&&&&&&#&&&&&&&&&&&&&&&&&&&&&&&&&&&&& | |
| 53 | + #7GBBB##########&&&&&&#&&###&&&&&&&&&&##&&&&&&&&&&&&&&&&&&&&&##&&&&&&&&&&&&&&&&&&&&&&&&&&&& | |
| 54 | + | |
| 7 | 55 | Sitemap: https://commoncrawl.org/sitemap.xml |