NeuralCrawl

IGN / robots.txt snapshot

← back to ign.com · fetched 2026-06-27T10:57:05Z (3h ago) · HTTP 200 · 11816 bytes · sha256 4e1229d0653091c1 · raw

final URL: manual:file

1# Ziff Davis content is made available for your non-commercial use subject to our
2# Terms of Use here: https://www.ziffdavis.com/terms-of-use
3# Use of any robot, crawler, or other tool to scrape, harvest, extract, or retrieve any content on
4# this website using automated means is prohibited without written permission from Ziff Davis.
5# Prohibited uses include but are not limited to:
6# (1) text and data mining under Art. 4 of the EU Directive on Copyright in the Digital Single
7# Market;
8# (2) development or operation of artificial intelligence or machine learning software or
9# databases, including by training, fine-tuning, embedding, and retrieval-augmented generation;
10# (3) creating data sets containing our content or sharing it with others; and
11# (4) any commercial purposes.
12# Contact [email protected] for assistance.
13
14User-agent: *
15Disallow: /_views/
16Disallow: /apiproxy
17Disallow: */empty/
18Disallow: /beta
19Disallow: /blogs/
20Disallow: /debug
21Disallow: /pokedex/search
22Disallow: /search
23Disallow: /static/pv_candidate.html
24Disallow: /stf/
25Disallow: /sign-in
26Disallow: /sugarfif.html
27Disallow: /wikis/*/edit
28Disallow: /wikis/*/compare/*
29
30#block crawlers
31User-agent: Amzn-SearchBot
32User-agent: ApifyBot
33User-agent: ApifyWebsiteContentCrawler
34User-agent: AzureAI-SearchBot
35User-agent: Brightbot
36User-agent: ChatGPT-User
37User-agent: ExaBot
38User-agent: FirecrawlAgent
39User-agent: GPTBot
40User-agent: iAskBot
41User-agent: OAI-SearchBot
42User-agent: opencode
43User-agent: TerraCotta
44User-agent: Trae
45Disallow: /
46
47#Block AI content scrapers
48User-agent: "008"
49User-agent: AddSearchBot
50User-agent: AI2Bot
51User-agent: Ai2Bot-DeepResearchEval
52User-agent: Ai2Bot-Dolma
53User-agent: Amazonbot
54User-agent: AmazonBuyForMe
55User-agent: Anchor Browser
56User-agent: anthropic-ai
57User-agent: Applebot
58User-agent: Applebot-extended
59User-agent: atlassian-bot
60User-agent: AutoRAG
61User-agent: AwarioSmartBot
62User-agent: Big Sur AI
63User-agent: bigsur.ai
64User-agent: Brandwatch
65User-agent: Bravebot
66User-agent: Bytespider
67User-agent: CCBot
68User-agent: Channel3Bot
69User-agent: ChatGLM-Spider
70User-agent: Claude-SearchBot
71User-agent: Claude-User
72User-agent: Claude-Web
73User-agent: ClaudeBot
74User-agent: cohere-ai
75User-agent: cohere-training-data-crawler
76User-agent: Cotoyogi
77User-agent: Crawl4AI
78User-agent: Datenbank Crawler
79User-agent: DDM*
80User-agent: DDM-DCipher*
81User-agent: DDM-DCipher/1.0.7
82User-agent: DeepSeekBot
83User-agent: Devin
84User-agent: Diffbot
85User-agent: Direqt Anomura
86User-agent: DuckAssistBot
87User-agent: Echobot Bot
88User-agent: FacebookBot
89User-agent: Factset_spyderbot
90User-agent: HTTrack
91User-agent: iAsk
92User-agent: iaskspider
93User-agent: ICC Crawler
94User-agent: ICC-Crawler
95User-agent: ImageSiftBot
96User-agent: imageSpider
97User-agent: Kangaroo Bot
98User-agent: KlaviyoAIBot
99User-agent: Kunato
100User-agent: laion-huggingface-processor
101User-agent: LCC
102User-agent: LINER Bot
103User-agent: LinerBot
104User-agent: LinkupBot
105User-agent: Manus-User
106User-agent: meta-externalagent
107User-agent: meta-externalfetcher
108User-agent: meta-webindexer
109User-agent: mistral.ai
110User-agent: MistralAI-User
111User-agent: netEstate Imprint Crawler
112User-agent: NovaAct
113User-agent: Novellum AI Crawl
114User-agent: Nutch
115User-agent: Offline Explorer
116User-agent: omgili
117User-agent: PanguBot
118User-agent: PeopleInc-DCipher-Scraper/1.1.0
119User-agent: Perplexity-User
120User-agent: PerplexityBot
121User-agent: PetalBot
122User-agent: Poggio-Citations
123User-agent: QualifiedBot
124User-agent: SBIntuitionsBot
125User-agent: Scrapy
126User-agent: SeekrBot
127User-agent: SemrushBot-OCOB
128User-agent: SemrushBotSwa
129User-agent: ShapBot
130User-agent: Spider
131User-agent: TavilyBot
132User-agent: Timpibot
133User-agent: TwinAgent
134User-agent: Velen Crawler
135User-agent: VelenPublicWebCrawler
136User-agent: Webzio-Extended
137User-agent: Wrtn
138User-agent: YouBot
139User-agent: ZanistaBot
140Allow: /articles/experimenting-with-galaxy-ai-camera-features-like-night-mode-generative-edit-and-more
141Allow: /articles/how-ai-assisted-gaming-on-the-galaxy-s25-means-extended-play-time-and-optimized-performance
142Allow: /articles/007-first-light-review
143Allow: /articles/1080p-at-30fps-valve-steam-machines-verified-program
144Allow: /articles/alien-earth-review
145Allow: /articles/avatar-fire-and-ash-is-now-available-to-watch-at-home
146Allow: /articles/battlefield-6-global-release-times-confirmed
147Allow: /articles/battlefield-6-review-multiplayer
148Allow: /articles/best-assassins-creed-games
149Allow: /articles/best-games-like-elder-scrolls-5-skyrim
150Allow: /articles/best-gaming-headset
151Allow: /articles/best-gaming-keyboard
152Allow: /articles/best-gaming-laptop
153Allow: /articles/best-gaming-monitor
154Allow: /articles/best-graphics-card
155Allow: /articles/best-handheld-gaming-pc
156Allow: /articles/best-horror-games
157Allow: /articles/best-lego-games
158Allow: /articles/best-pc-controller
159Allow: /articles/best-playstation-portal-accessories
160Allow: /articles/best-ps5-accessories
161Allow: /articles/best-samsung-galaxy-s26-cases
162Allow: /articles/best-tv-for-gaming
163Allow: /articles/best-xbox-series-x-games
164Allow: /articles/blox-fruits-codes
165Allow: /articles/blue-lock-rivals-codes
166Allow: /articles/borderlands-4-review
167Allow: /articles/call-of-duty-black-ops-7-campaign-review
168Allow: /articles/conjuring-movies-in-order
169Allow: /articles/crimson-desert-global-release-time-confirmed-as-preload-goes-live
170Allow: /articles/crimson-desert-review
171Allow: /articles/disney-debuts-new-avengers-doomsday-trailer-at-cinemacon-2026
172Allow: /articles/dress-to-impress-codes
173Allow: /articles/ghost-of-yotei-review
174Allow: /articles/gta-6-release-date-about-18-months-behind-original-target-take-two-boss-reveals-suggesting-spring-2025-was-once-the-goal-internally
175Allow: /articles/hollow-knight-silksong-global-release-times-confirmed
176Allow: /articles/ign-awards-2025-winners
177Allow: /articles/jurassic-world-rebirth-review-scarlett-johansson-mahershala-ali
178Allow: /articles/metroid-prime-4-beyond-review
179Allow: /articles/new-xbox-ceo-comments-on-game-pass
180Allow: /articles/nintendo-fans-demand-tariff-refunds-returned-to-customers
181Allow: /articles/nintendo-fans-surprised-to-find-switch-2-edition-game-cartridges-also-work-on-switch-1
182Allow: /articles/nintendo-is-absolutely-furious-former-employee-says-mario-zelda-developer-in-uncharted-territory-following-major-leaks
183Allow: /articles/nintendo-switch-2-price-hike-where-to-buy
184Allow: /articles/one-piece-series-movies-in-order
185Allow: /articles/phantom-blade-zero-is-a-delightfully-punishing-action-rpg
186Allow: /articles/plants-vs-brainrots-codes
187Allow: /articles/pokemon-fan-claims-world-first-completion-of-resident-evil-requiem-final-puzzle-but-claim-theyre-unsure-exactly-how-they-did-it
188Allow: /articles/resident-evil-requiem-review
189Allow: /articles/silent-hill-f-review
190Allow: /articles/sony-announces-price-rises-for-ps5-ps5-pro-and-playstation-portal-blames-continued-pressures-in-the-global-economic-landscape
191Allow: /articles/sony-confirms-playstation-plus-monthly-games-for-april-2026
192Allow: /articles/steam-machine-hands-on-first-impressions
193Allow: /articles/stranger-things-creators-promise-say-theyll-reveal-elevens-fate-in-20-years
194Allow: /articles/superman-2025-review-james-gunn-david-corenswet-rachel-brosnahan-nicholas-hoult
195Allow: /articles/the-10-best-pokemon-video-games
196Allow: /articles/the-best-100-video-games-of-all-time
197Allow: /articles/the-best-gaming-chairs
198Allow: /articles/the-best-ps5-games
199Allow: /articles/the-fate-anime-series-in-order
200Allow: /articles/the-forge-codes
201Allow: /articles/the-last-of-us-2-safe-combinations-and-codes
202Allow: /articles/video-game-release-dates-ps4-ps5-xbox-one-series-x-nintendo-switch
203Allow: /articles/what-to-expect-playstation-state-of-play-june-2026
204Allow: /articles/x-men-movies-in-order
205Allow: /playlist/rchnemesis/lists/top-100-indie-games
206Allow: /wikis/007-first-light/Essential_Tips_and_Tricks
207Allow: /wikis/arc-raiders/Avian_Alarm_Project_Guide_and_How_to_Complete
208Allow: /wikis/avatar-frontiers-of-pandora/All_Sarentu_Totem_Locations
209Allow: /wikis/baldurs-gate-3/The_Blood_of_Lathander
210Allow: /wikis/blue-prince/How_to_Solve_the_Chess_Piece_Puzzle
211Allow: /wikis/borderlands-4/Borderlands_4_SHiFT_Codes
212Allow: /wikis/call-of-duty-black-ops-7/Walkthrough
213Allow: /wikis/clair-obscur-expedition-33/All_Endings_and_How_to_Get_Them
214Allow: /wikis/clair-obscur-expedition-33/Walkthrough
215Allow: /wikis/crimson-desert/How_to_Get_the_Frostcursed_Armor_Set_-_All_Locations
216Allow: /wikis/crimson-desert/Smoking_Lands_Desert_Mouth_Maze_Puzzle_Solution
217Allow: /wikis/crimson-desert/Walkthrough
218Allow: /wikis/cyberpunk-2077/Panam_Romance_Guide
219Allow: /wikis/elden-ring/Millicent_Questline
220Allow: /wikis/elden-ring/Ranni_The_Witch_Questline
221Allow: /wikis/fallout-4/Companion_Perks
222Allow: /wikis/final-fantasy-7-rebirth/FF7_Rebirth_Walkthrough
223Allow: /wikis/forza-horizon-6/Barn_Finds
224Allow: /wikis/ghost-of-yotei/Trophy_List_and_Guide
225Allow: /wikis/grand-theft-auto-san-andreas/Cheat_Codes_and_Secrets
226Allow: /wikis/grow-a-garden/All_Bizzy_Bear_Honey_Crafter_Recipe_Quests_and_Rewards
227Allow: /wikis/grow-a-garden/All_Fruit_Plants_and_How_to_Get_Them
228Allow: /wikis/grow-a-garden/All_Vegetables_and_How_to_Get_Them
229Allow: /wikis/grow-a-garden/How_to_Make_a_Waffle
230Allow: /wikis/gta-5/All_GTA_5_Cell_Phone_Cheat_codes
231Allow: /wikis/gta-5/All_GTA_5_Cheat_Codes_and_Secrets_For_PC_and_Console
232Allow: /wikis/gta-5/GTA_Online_Weekly_Updates
233Allow: /wikis/hogwarts-legacy/All_Hogwarts_Secrets_Solutions
234Allow: /wikis/hogwarts-legacy/All_Merlin_Trials_Solutions
235Allow: /wikis/hogwarts-legacy/Demiguise_Statues_Locations
236Allow: /wikis/hollow-knight-silksong/Pale_Oil_Locations
237Allow: /wikis/hollow-knight-silksong/Widow_Boss_Guide
238Allow: /wikis/hollow-knight/walkthrough
239Allow: /wikis/kingdom-hearts-hd-15-remix/Kingdom_Hearts_Final_Mix_-_Walkthrough
240Allow: /wikis/la-noire/Walkthrough
241Allow: /wikis/mario-kart-world/All_Characters,_NPC_Drivers,_and_Outfits_List
242Allow: /wikis/neverness-to-everness/All_NTE_Codes_for_June_2026
243Allow: /wikis/paralives/Paralives_Cheats_List_and_Console_Commands
244Allow: /wikis/pokemon-firered-leafgreen-version/Best_Starter_to_Choose
245Allow: /wikis/pokemon-firered-leafgreen-version/Rock_Tunnel
246Allow: /wikis/pokemon-firered-leafgreen-version/Route_22
247Allow: /wikis/pokemon-legends-z-a/All_Mega_Pokemon
248Allow: /wikis/re2-remake/All_Safe_and_Lock_Combinations,_Codes,_and_Solutions
249Allow: /wikis/red-dead-redemption-2/Gold_Bar_Locations_and_Where_to_Sell_Them
250Allow: /wikis/red-dead-redemption-2/Horses_Guide_-_Locations,_Stats,_Bonding,_Taming,_and_Breeds
251Allow: /wikis/red-dead-redemption-2/Legendary_Animals_and_Fish_Locations_and_Map
252Allow: /wikis/resident-evil-requiem/File_Locations
253Allow: /wikis/resident-evil-requiem/Walkthrough
254Allow: /wikis/silent-hill-2-remake/Endings
255Allow: /wikis/silent-hill-f/Dark_Shrine_Finale_Walkthrough
256Allow: /wikis/silent-hill-f/Field_Puzzle_Guide
257Allow: /wikis/silent-hill-f/Light_Puzzle_Guide
258Allow: /wikis/silent-hill-f/Walkthrough
259Allow: /wikis/slime-rng-roblox/Slime_RNG_Codes
260Allow: /wikis/stardew-valley/Item_Codes_for_Spawning_Cheat
261Allow: /wikis/subnautica-2/Walkthrough_Hub
262Allow: /wikis/the-elder-scrolls-5-skyrim/PC_Console_Command_Cheats
263Allow: /wikis/the-forge/How_to_Reroll_Race
264Allow: /wikis/the-legend-of-zelda-breath-of-the-wild/Shrines
265Allow: /wikis/the-legend-of-zelda-tears-of-the-kingdom/All_Shrine_Locations_and_Solutions
266Allow: /wikis/vv-ultimatum-roblox/VV_ULTIMATUM_Codes
267Disallow: /
268
269User-agent: DuckDuckBot
270Allow: /
271
272Sitemap: https://www.ign.com/rss/sitemap-articles.xml
273Sitemap: https://www.ign.com/rss/sitemap-authors.xml
274Sitemap: https://www.ign.com/rss/sitemap-objects.xml
275Sitemap: https://www.ign.com/rss/sitemap-videos.xml
276Sitemap: https://www.ign.com/rss/sitemap-wikis.xml
277Sitemap: https://www.ign.com/rss/news/sitemap
278Sitemap: https://www.ign.com/rss/videos/sitemap-index-recently-updated
279Sitemap: https://www.ign.com/rss/wikis/sitemap-maps