NeuralCrawl

IGN / robots.txt snapshot

← back to ign.com · fetched 2026-10-12T17:28:08Z (25d ago) · HTTP 200 · 14179 bytes · sha256 b7e94709be0c0efd · raw

final URL: https://www.ign.com/robots.txt

1# Ziff Davis content is made available for your non-commercial use subject to our
2# Terms of Use here: https://www.ziffdavis.com/terms-of-use
3# Use of any robot, crawler, or other tool to scrape, harvest, extract, or retrieve any content on
4# this website using automated means is prohibited without written permission from Ziff Davis.
5# Prohibited uses include but are not limited to:
6# (1) text and data mining under Art. 4 of the EU Directive on Copyright in the Digital Single
7# Market;
8# (2) development or operation of artificial intelligence or machine learning software or
9# databases, including by training, fine-tuning, embedding, and retrieval-augmented generation;
10# (3) creating data sets containing our content or sharing it with others; and
11# (4) any commercial purposes.
12# Contact [email protected] for assistance.
13
14User-agent: *
15Disallow: /_views/
16Disallow: /apiproxy
17Disallow: */empty/
18Disallow: /beta
19Disallow: /blogs/
20Disallow: /debug
21Disallow: /pokedex/search
22Disallow: /search
23Disallow: /static/pv_candidate.html
24Disallow: /stf/
25Disallow: /sign-in
26Disallow: /sugarfif.html
27Disallow: /wikis/*/edit
28Disallow: /wikis/*/compare/*
29Allow: /sponsored/
30
31#block crawlers
32User-agent: Amzn-SearchBot
33User-agent: ApifyBot
34User-agent: ApifyWebsiteContentCrawler
35User-agent: AzureAI-SearchBot
36User-agent: Brightbot
37User-agent: ChatGPT-User
38User-agent: ExaBot
39User-agent: FirecrawlAgent
40User-agent: GPTBot
41User-agent: iAskBot
42User-agent: OAI-SearchBot
43User-agent: opencode
44User-agent: TerraCotta
45User-agent: Trae
46Allow: /sponsored/
47Disallow: /
48
49#Block AI content scrapers
50User-agent: "008"
51User-agent: AddSearchBot
52User-agent: AgentDataBot
53User-agent: AI2Bot
54User-agent: Ai2Bot-DeepResearchEval
55User-agent: Ai2Bot-Dolma
56User-agent: AIWebIndex
57User-agent: AIWebIndex-Agent
58User-agent: amazon-kendra
59User-agent: amazon-QBusiness
60User-agent: Amazonbot
61User-agent: AmazonBuyForMe
62User-agent: Amzn-SearchBot
63User-agent: Amzn-User
64User-agent: Anchor Browser
65User-agent: AndiAgent
66User-agent: anthropic-ai
67User-agent: ApifyBot
68User-agent: ApifyWebsiteContentCrawler
69User-agent: Applebot
70User-agent: Applebot-extended
71User-agent: atlassian-bot
72User-agent: AutoRAG
73User-agent: AwarioSmartBot
74User-agent: AzureAI-SearchBot
75User-agent: BeansLLM-CorpusBot
76User-agent: bedrockbot
77User-agent: Big Sur AI
78User-agent: bigsur.ai
79User-agent: BixelBot
80User-agent: Brandwatch
81User-agent: Bravebot
82User-agent: Brightbot
83User-agent: Bytespider
84User-agent: CCBot
85User-agent: Channel3Bot
86User-agent: ChatGLM-Spider
87User-agent: ChatGPT-User
88User-agent: Claude-Code
89User-agent: Claude-SearchBot
90User-agent: Claude-User
91User-agent: Claude-Web
92User-agent: ClaudeBot
93User-agent: Cloudflare-AutoRAG
94User-agent: CloudflareBrowserRenderingCrawler
95User-agent: Code
96User-agent: cohere-ai
97User-agent: cohere-training-data-crawler
98User-agent: CohereBot
99User-agent: Cotoyogi
100User-agent: CragCrawler
101User-agent: Crawl4AI
102User-agent: Crawlspace
103User-agent: Cursor
104User-agent: Datenbank Crawler
105User-agent: Datenbank-Crawler
106User-agent: DDM-DCipher*
107User-agent: DDM-DCipher/1.0.7
108User-agent: DDM*
109User-agent: DeepSeekBot
110User-agent: DesearchBot
111User-agent: Devin
112User-agent: Diffbot
113User-agent: Direqt Anomura
114User-agent: Dola-SearchBot
115User-agent: Doubaobot
116User-agent: DuckAssistBot
117User-agent: Echobot Bot
118User-agent: ERNIEBot
119User-agent: ExaBot
120User-agent: ExaSearchBot
121User-agent: FacebookBot
122User-agent: Factset_spyderbot
123User-agent: FirecrawlAgent
124User-agent: GeistHaus-PageFetcher
125User-agent: GPTBot
126User-agent: grok-agent
127User-agent: GrokBot
128User-agent: HenkBot
129User-agent: HTTrack
130User-agent: Hunyuan
131User-agent: iAsk
132User-agent: iAskBot
133User-agent: iaskspider
134User-agent: ICC Crawler
135User-agent: ICC-Crawler
136User-agent: ImageBot
137User-agent: ImageSiftBot
138User-agent: imageSpider
139User-agent: IsonSearchBot
140User-agent: kagi-fetcher
141User-agent: Kangaroo-Bot
142User-agent: Keenable-User
143User-agent: KeenableBot
144User-agent: Kimi-Agent
145User-agent: Kimi-SearchBot
146User-agent: Kimi-User
147User-agent: KimiBot
148User-agent: KimiCrawler
149User-agent: KlaviyoAIBot
150User-agent: Kunato
151User-agent: KunatoCrawler
152User-agent: laion-huggingface-processor
153User-agent: LCC
154User-agent: LINER Bot
155User-agent: LinkupBot
156User-agent: Manus-User
157User-agent: meta-externalagent
158User-agent: meta-externalfetcher
159User-agent: meta-webindexer
160User-agent: micro-crawl
161User-agent: mistral.ai
162User-agent: MistralAI-Index
163User-agent: MistralAI-User
164User-agent: MistralBot
165User-agent: MoonshotBot
166User-agent: MoonSpider
167User-agent: Mozilla-Tabstack
168User-agent: netEstate Imprint Crawler
169User-agent: NovaAct
170User-agent: Novellum AI Crawl
171User-agent: Nutch
172User-agent: Novellum
173User-agent: OAI-SearchBot
174User-agent: Offline Explorer
175User-agent: omgili
176User-agent: opencode
177User-agent: PanguBot
178User-agent: PeopleInc-DCipher-Scraper/1.1.0
179User-agent: Perplexity-User
180User-agent: PerplexityBot
181User-agent: PetalBot
182User-agent: PhindBot
183User-agent: Poggio-Citations
184User-agent: qodercli
185User-agent: QualifiedBot
186User-agent: Querit-SearchBot
187User-agent: QueritBot
188User-agent: QwenBot
189User-agent: recraft-image-dataset
190User-agent: Reflectionbot
191User-agent: SalamandraVLM
192User-agent: SBIntuitionsBot
193User-agent: ScryBot
194User-agent: Scrapy
195User-agent: SeekrBot
196User-agent: SemrushBot-OCOB
197User-agent: SemrushBotSwa
198User-agent: Shap-User
199User-agent: ShapBot
200User-agent: SofyaBot
201User-agent: SomethinggBot
202User-agent: Spider
203User-agent: TalarionFrontierCrawler
204User-agent: TalarionSearchBot
205User-agent: TavilyBot
206User-agent: TerraCotta
207User-agent: Timpibot
208User-agent: TongyiBot
209User-agent: Trae
210User-agent: TwinAgent
211User-agent: UseAI
212User-agent: Velen Crawler
213User-agent: VelenPublicWebCrawler
214User-agent: Webzio-Extended
215User-agent: Wrtn
216User-agent: WRTNBot
217User-agent: xAI-Grok
218User-agent: xAI-SearchBot
219User-agent: YandexAdditionalBot
220User-agent: YepBot
221User-agent: YiBot
222User-agent: YiyanBot
223User-agent: YouBot
224User-agent: ZanistaBot
225Allow: /articles/experimenting-with-galaxy-ai-camera-features-like-night-mode-generative-edit-and-more
226Allow: /articles/how-ai-assisted-gaming-on-the-galaxy-s25-means-extended-play-time-and-optimized-performance
227Allow: /articles/007-first-light-review
228Allow: /articles/1080p-at-30fps-valve-steam-machines-verified-program
229Allow: /articles/alien-earth-review
230Allow: /articles/avatar-fire-and-ash-is-now-available-to-watch-at-home
231Allow: /articles/battlefield-6-global-release-times-confirmed
232Allow: /articles/battlefield-6-review-multiplayer
233Allow: /articles/best-assassins-creed-games
234Allow: /articles/best-games-like-elder-scrolls-5-skyrim
235Allow: /articles/best-gaming-headset
236Allow: /articles/best-gaming-keyboard
237Allow: /articles/best-gaming-laptop
238Allow: /articles/best-gaming-monitor
239Allow: /articles/best-graphics-card
240Allow: /articles/best-handheld-gaming-pc
241Allow: /articles/best-horror-games
242Allow: /articles/best-lego-games
243Allow: /articles/best-pc-controller
244Allow: /articles/best-playstation-portal-accessories
245Allow: /articles/best-ps5-accessories
246Allow: /articles/best-samsung-galaxy-s26-cases
247Allow: /articles/best-tv-for-gaming
248Allow: /articles/best-xbox-series-x-games
249Allow: /articles/blox-fruits-codes
250Allow: /articles/blue-lock-rivals-codes
251Allow: /articles/borderlands-4-review
252Allow: /articles/call-of-duty-black-ops-7-campaign-review
253Allow: /articles/conjuring-movies-in-order
254Allow: /articles/crimson-desert-global-release-time-confirmed-as-preload-goes-live
255Allow: /articles/crimson-desert-review
256Allow: /articles/disney-debuts-new-avengers-doomsday-trailer-at-cinemacon-2026
257Allow: /articles/dress-to-impress-codes
258Allow: /articles/ghost-of-yotei-review
259Allow: /articles/gta-6-release-date-about-18-months-behind-original-target-take-two-boss-reveals-suggesting-spring-2025-was-once-the-goal-internally
260Allow: /articles/hollow-knight-silksong-global-release-times-confirmed
261Allow: /articles/how-the-samsung-galaxy-z-fold8-has-been-optimised-for-large-screen-entertainment-multitasking-and-all-day-battery-life
262Allow: /articles/ign-awards-2025-winners
263Allow: /articles/jurassic-world-rebirth-review-scarlett-johansson-mahershala-ali
264Allow: /articles/metroid-prime-4-beyond-review
265Allow: /articles/new-xbox-ceo-comments-on-game-pass
266Allow: /articles/nintendo-fans-demand-tariff-refunds-returned-to-customers
267Allow: /articles/nintendo-fans-surprised-to-find-switch-2-edition-game-cartridges-also-work-on-switch-1
268Allow: /articles/nintendo-is-absolutely-furious-former-employee-says-mario-zelda-developer-in-uncharted-territory-following-major-leaks
269Allow: /articles/nintendo-switch-2-price-hike-where-to-buy
270Allow: /articles/one-piece-series-movies-in-order
271Allow: /articles/phantom-blade-zero-is-a-delightfully-punishing-action-rpg
272Allow: /articles/plants-vs-brainrots-codes
273Allow: /articles/pokemon-fan-claims-world-first-completion-of-resident-evil-requiem-final-puzzle-but-claim-theyre-unsure-exactly-how-they-did-it
274Allow: /articles/resident-evil-requiem-review
275Allow: /articles/silent-hill-f-review
276Allow: /articles/sony-announces-price-rises-for-ps5-ps5-pro-and-playstation-portal-blames-continued-pressures-in-the-global-economic-landscape
277Allow: /articles/sony-confirms-playstation-plus-monthly-games-for-april-2026
278Allow: /articles/steam-machine-hands-on-first-impressions
279Allow: /articles/stranger-things-creators-promise-say-theyll-reveal-elevens-fate-in-20-years
280Allow: /articles/superman-2025-review-james-gunn-david-corenswet-rachel-brosnahan-nicholas-hoult
281Allow: /articles/the-10-best-pokemon-video-games
282Allow: /articles/the-best-100-video-games-of-all-time
283Allow: /articles/the-best-gaming-chairs
284Allow: /articles/the-best-ps5-games
285Allow: /articles/the-fate-anime-series-in-order
286Allow: /articles/the-forge-codes
287Allow: /articles/the-last-of-us-2-safe-combinations-and-codes
288Allow: /articles/video-game-release-dates-ps4-ps5-xbox-one-series-x-nintendo-switch
289Allow: /articles/what-to-expect-playstation-state-of-play-june-2026
290Allow: /articles/x-men-movies-in-order
291Allow: /articles/z-fold8-vs-previous-samsung-foldables-how-battery-performance-efficiency-and-display-experience-have-evolved
292Allow: /playlist/rchnemesis/lists/top-100-indie-games
293Allow: /wikis/007-first-light/Essential_Tips_and_Tricks
294Allow: /wikis/arc-raiders/Avian_Alarm_Project_Guide_and_How_to_Complete
295Allow: /wikis/avatar-frontiers-of-pandora/All_Sarentu_Totem_Locations
296Allow: /wikis/baldurs-gate-3/The_Blood_of_Lathander
297Allow: /wikis/blue-prince/How_to_Solve_the_Chess_Piece_Puzzle
298Allow: /wikis/borderlands-4/Borderlands_4_SHiFT_Codes
299Allow: /wikis/call-of-duty-black-ops-7/Walkthrough
300Allow: /wikis/clair-obscur-expedition-33/All_Endings_and_How_to_Get_Them
301Allow: /wikis/clair-obscur-expedition-33/Walkthrough
302Allow: /wikis/crimson-desert/How_to_Get_the_Frostcursed_Armor_Set_-_All_Locations
303Allow: /wikis/crimson-desert/Smoking_Lands_Desert_Mouth_Maze_Puzzle_Solution
304Allow: /wikis/crimson-desert/Walkthrough
305Allow: /wikis/cyberpunk-2077/Panam_Romance_Guide
306Allow: /wikis/elden-ring/Millicent_Questline
307Allow: /wikis/elden-ring/Ranni_The_Witch_Questline
308Allow: /wikis/fallout-4/Companion_Perks
309Allow: /wikis/final-fantasy-7-rebirth/FF7_Rebirth_Walkthrough
310Allow: /wikis/forza-horizon-6/Barn_Finds
311Allow: /wikis/ghost-of-yotei/Trophy_List_and_Guide
312Allow: /wikis/grand-theft-auto-san-andreas/Cheat_Codes_and_Secrets
313Allow: /wikis/grow-a-garden/All_Bizzy_Bear_Honey_Crafter_Recipe_Quests_and_Rewards
314Allow: /wikis/grow-a-garden/All_Fruit_Plants_and_How_to_Get_Them
315Allow: /wikis/grow-a-garden/All_Vegetables_and_How_to_Get_Them
316Allow: /wikis/grow-a-garden/How_to_Make_a_Waffle
317Allow: /wikis/gta-5/All_GTA_5_Cell_Phone_Cheat_codes
318Allow: /wikis/gta-5/All_GTA_5_Cheat_Codes_and_Secrets_For_PC_and_Console
319Allow: /wikis/gta-5/GTA_Online_Weekly_Updates
320Allow: /wikis/hogwarts-legacy/All_Hogwarts_Secrets_Solutions
321Allow: /wikis/hogwarts-legacy/All_Merlin_Trials_Solutions
322Allow: /wikis/hogwarts-legacy/Demiguise_Statues_Locations
323Allow: /wikis/hollow-knight-silksong/Pale_Oil_Locations
324Allow: /wikis/hollow-knight-silksong/Widow_Boss_Guide
325Allow: /wikis/hollow-knight/walkthrough
326Allow: /wikis/kingdom-hearts-hd-15-remix/Kingdom_Hearts_Final_Mix_-_Walkthrough
327Allow: /wikis/la-noire/Walkthrough
328Allow: /wikis/mario-kart-world/All_Characters,_NPC_Drivers,_and_Outfits_List
329Allow: /wikis/neverness-to-everness/All_NTE_Codes_for_June_2026
330Allow: /wikis/paralives/Paralives_Cheats_List_and_Console_Commands
331Allow: /wikis/pokemon-firered-leafgreen-version/Best_Starter_to_Choose
332Allow: /wikis/pokemon-firered-leafgreen-version/Rock_Tunnel
333Allow: /wikis/pokemon-firered-leafgreen-version/Route_22
334Allow: /wikis/pokemon-legends-z-a/All_Mega_Pokemon
335Allow: /wikis/re2-remake/All_Safe_and_Lock_Combinations,_Codes,_and_Solutions
336Allow: /wikis/red-dead-redemption-2/Gold_Bar_Locations_and_Where_to_Sell_Them
337Allow: /wikis/red-dead-redemption-2/Horses_Guide_-_Locations,_Stats,_Bonding,_Taming,_and_Breeds
338Allow: /wikis/red-dead-redemption-2/Legendary_Animals_and_Fish_Locations_and_Map
339Allow: /wikis/resident-evil-requiem/File_Locations
340Allow: /wikis/resident-evil-requiem/Walkthrough
341Allow: /wikis/silent-hill-2-remake/Endings
342Allow: /wikis/silent-hill-f/Dark_Shrine_Finale_Walkthrough
343Allow: /wikis/silent-hill-f/Field_Puzzle_Guide
344Allow: /wikis/silent-hill-f/Light_Puzzle_Guide
345Allow: /wikis/silent-hill-f/Walkthrough
346Allow: /wikis/slime-rng-roblox/Slime_RNG_Codes
347Allow: /wikis/stardew-valley/Item_Codes_for_Spawning_Cheat
348Allow: /wikis/subnautica-2/Walkthrough_Hub
349Allow: /wikis/the-elder-scrolls-5-skyrim/PC_Console_Command_Cheats
350Allow: /wikis/the-forge/How_to_Reroll_Race
351Allow: /wikis/the-legend-of-zelda-breath-of-the-wild/Shrines
352Allow: /wikis/the-legend-of-zelda-tears-of-the-kingdom/All_Shrine_Locations_and_Solutions
353Allow: /wikis/vv-ultimatum-roblox/VV_ULTIMATUM_Codes
354Allow: /sponsored/
355Disallow: /
356
357User-agent: DuckDuckBot
358Allow: /sponsored/
359Allow: /
360
361Sitemap: https://www.ign.com/rss/sitemap-articles.xml
362Sitemap: https://www.ign.com/rss/sitemap-authors.xml
363Sitemap: https://www.ign.com/rss/sitemap-objects.xml
364Sitemap: https://www.ign.com/rss/sitemap-videos.xml
365Sitemap: https://www.ign.com/rss/sitemap-wikis.xml
366Sitemap: https://www.ign.com/rss/news/sitemap
367Sitemap: https://www.ign.com/rss/videos/sitemap-index-recently-updated
368Sitemap: https://www.ign.com/rss/wikis/sitemap-maps