NeuralCrawl

IGN / robots.txt snapshot

← back to ign.com · fetched 2026-10-03T20:27:55Z (1mo ago) · HTTP 200 · 13860 bytes · sha256 18265aacfacf6c13 · raw

final URL: https://www.ign.com/robots.txt

1# Ziff Davis content is made available for your non-commercial use subject to our
2# Terms of Use here: https://www.ziffdavis.com/terms-of-use
3# Use of any robot, crawler, or other tool to scrape, harvest, extract, or retrieve any content on
4# this website using automated means is prohibited without written permission from Ziff Davis.
5# Prohibited uses include but are not limited to:
6# (1) text and data mining under Art. 4 of the EU Directive on Copyright in the Digital Single
7# Market;
8# (2) development or operation of artificial intelligence or machine learning software or
9# databases, including by training, fine-tuning, embedding, and retrieval-augmented generation;
10# (3) creating data sets containing our content or sharing it with others; and
11# (4) any commercial purposes.
12# Contact [email protected] for assistance.
13
14User-agent: *
15Disallow: /_views/
16Disallow: /apiproxy
17Disallow: */empty/
18Disallow: /beta
19Disallow: /blogs/
20Disallow: /debug
21Disallow: /pokedex/search
22Disallow: /search
23Disallow: /static/pv_candidate.html
24Disallow: /stf/
25Disallow: /sign-in
26Disallow: /sugarfif.html
27Disallow: /wikis/*/edit
28Disallow: /wikis/*/compare/*
29
30#block crawlers
31User-agent: Amzn-SearchBot
32User-agent: ApifyBot
33User-agent: ApifyWebsiteContentCrawler
34User-agent: AzureAI-SearchBot
35User-agent: Brightbot
36User-agent: ChatGPT-User
37User-agent: ExaBot
38User-agent: FirecrawlAgent
39User-agent: GPTBot
40User-agent: iAskBot
41User-agent: OAI-SearchBot
42User-agent: opencode
43User-agent: TerraCotta
44User-agent: Trae
45Disallow: /
46
47#Block AI content scrapers
48User-agent: "008"
49User-agent: AddSearchBot
50User-agent: AgentDataBot
51User-agent: AI2Bot
52User-agent: Ai2Bot-DeepResearchEval
53User-agent: Ai2Bot-Dolma
54User-agent: AIWebIndex
55User-agent: AIWebIndex-Agent
56User-agent: amazon-kendra
57User-agent: amazon-QBusiness
58User-agent: Amazonbot
59User-agent: AmazonBuyForMe
60User-agent: Amzn-SearchBot
61User-agent: Amzn-User
62User-agent: Anchor Browser
63User-agent: AndiAgent
64User-agent: anthropic-ai
65User-agent: ApifyBot
66User-agent: ApifyWebsiteContentCrawler
67User-agent: Applebot
68User-agent: Applebot-extended
69User-agent: atlassian-bot
70User-agent: AutoRAG
71User-agent: AwarioSmartBot
72User-agent: AzureAI-SearchBot
73User-agent: BeansLLM-CorpusBot
74User-agent: bedrockbot
75User-agent: Big Sur AI
76User-agent: bigsur.ai
77User-agent: BixelBot
78User-agent: Brandwatch
79User-agent: Bravebot
80User-agent: Brightbot
81User-agent: Bytespider
82User-agent: CCBot
83User-agent: Channel3Bot
84User-agent: ChatGLM-Spider
85User-agent: ChatGPT-User
86User-agent: Claude-Code
87User-agent: Claude-SearchBot
88User-agent: Claude-User
89User-agent: Claude-Web
90User-agent: ClaudeBot
91User-agent: Cloudflare-AutoRAG
92User-agent: CloudflareBrowserRenderingCrawler
93User-agent: Code
94User-agent: cohere-ai
95User-agent: cohere-training-data-crawler
96User-agent: CohereBot
97User-agent: Cotoyogi
98User-agent: CragCrawler
99User-agent: Crawl4AI
100User-agent: Crawlspace
101User-agent: Cursor
102User-agent: Datenbank Crawler
103User-agent: Datenbank-Crawler
104User-agent: DDM-DCipher*
105User-agent: DDM-DCipher/1.0.7
106User-agent: DDM*
107User-agent: DeepSeekBot
108User-agent: DesearchBot
109User-agent: Devin
110User-agent: Diffbot
111User-agent: Direqt Anomura
112User-agent: Dola-SearchBot
113User-agent: Doubaobot
114User-agent: DuckAssistBot
115User-agent: Echobot Bot
116User-agent: ERNIEBot
117User-agent: ExaBot
118User-agent: ExaSearchBot
119User-agent: FacebookBot
120User-agent: Factset_spyderbot
121User-agent: FirecrawlAgent
122User-agent: GeistHaus-PageFetcher
123User-agent: GPTBot
124User-agent: grok-agent
125User-agent: GrokBot
126User-agent: HenkBot
127User-agent: HTTrack
128User-agent: Hunyuan
129User-agent: iAsk
130User-agent: iAskBot
131User-agent: iaskspider
132User-agent: ICC Crawler
133User-agent: ICC-Crawler
134User-agent: ImageBot
135User-agent: ImageSiftBot
136User-agent: imageSpider
137User-agent: IsonSearchBot
138User-agent: kagi-fetcher
139User-agent: Kangaroo-Bot
140User-agent: Keenable-User
141User-agent: KeenableBot
142User-agent: Kimi-Agent
143User-agent: Kimi-SearchBot
144User-agent: Kimi-User
145User-agent: KimiBot
146User-agent: KimiCrawler
147User-agent: KlaviyoAIBot
148User-agent: Kunato
149User-agent: KunatoCrawler
150User-agent: laion-huggingface-processor
151User-agent: LCC
152User-agent: LINER Bot
153User-agent: LinkupBot
154User-agent: Manus-User
155User-agent: meta-externalagent
156User-agent: meta-externalfetcher
157User-agent: meta-webindexer
158User-agent: micro-crawl
159User-agent: mistral.ai
160User-agent: MistralAI-Index
161User-agent: MistralAI-User
162User-agent: MistralBot
163User-agent: MoonshotBot
164User-agent: MoonSpider
165User-agent: Mozilla-Tabstack
166User-agent: netEstate Imprint Crawler
167User-agent: NovaAct
168User-agent: Novellum AI Crawl
169User-agent: Nutch
170User-agent: Novellum
171User-agent: OAI-SearchBot
172User-agent: Offline Explorer
173User-agent: omgili
174User-agent: opencode
175User-agent: PanguBot
176User-agent: PeopleInc-DCipher-Scraper/1.1.0
177User-agent: Perplexity-User
178User-agent: PerplexityBot
179User-agent: PetalBot
180User-agent: PhindBot
181User-agent: Poggio-Citations
182User-agent: qodercli
183User-agent: QualifiedBot
184User-agent: Querit-SearchBot
185User-agent: QueritBot
186User-agent: QwenBot
187User-agent: recraft-image-dataset
188User-agent: Reflectionbot
189User-agent: SalamandraVLM
190User-agent: SBIntuitionsBot
191User-agent: ScryBot
192User-agent: Scrapy
193User-agent: SeekrBot
194User-agent: SemrushBot-OCOB
195User-agent: SemrushBotSwa
196User-agent: Shap-User
197User-agent: ShapBot
198User-agent: SofyaBot
199User-agent: SomethinggBot
200User-agent: Spider
201User-agent: TalarionFrontierCrawler
202User-agent: TalarionSearchBot
203User-agent: TavilyBot
204User-agent: TerraCotta
205User-agent: Timpibot
206User-agent: TongyiBot
207User-agent: Trae
208User-agent: TwinAgent
209User-agent: UseAI
210User-agent: Velen Crawler
211User-agent: VelenPublicWebCrawler
212User-agent: Webzio-Extended
213User-agent: Wrtn
214User-agent: WRTNBot
215User-agent: xAI-Grok
216User-agent: xAI-SearchBot
217User-agent: YandexAdditionalBot
218User-agent: YepBot
219User-agent: YiBot
220User-agent: YiyanBot
221User-agent: YouBot
222User-agent: ZanistaBot
223Allow: /articles/experimenting-with-galaxy-ai-camera-features-like-night-mode-generative-edit-and-more
224Allow: /articles/how-ai-assisted-gaming-on-the-galaxy-s25-means-extended-play-time-and-optimized-performance
225Allow: /articles/007-first-light-review
226Allow: /articles/1080p-at-30fps-valve-steam-machines-verified-program
227Allow: /articles/alien-earth-review
228Allow: /articles/avatar-fire-and-ash-is-now-available-to-watch-at-home
229Allow: /articles/battlefield-6-global-release-times-confirmed
230Allow: /articles/battlefield-6-review-multiplayer
231Allow: /articles/best-assassins-creed-games
232Allow: /articles/best-games-like-elder-scrolls-5-skyrim
233Allow: /articles/best-gaming-headset
234Allow: /articles/best-gaming-keyboard
235Allow: /articles/best-gaming-laptop
236Allow: /articles/best-gaming-monitor
237Allow: /articles/best-graphics-card
238Allow: /articles/best-handheld-gaming-pc
239Allow: /articles/best-horror-games
240Allow: /articles/best-lego-games
241Allow: /articles/best-pc-controller
242Allow: /articles/best-playstation-portal-accessories
243Allow: /articles/best-ps5-accessories
244Allow: /articles/best-samsung-galaxy-s26-cases
245Allow: /articles/best-tv-for-gaming
246Allow: /articles/best-xbox-series-x-games
247Allow: /articles/blox-fruits-codes
248Allow: /articles/blue-lock-rivals-codes
249Allow: /articles/borderlands-4-review
250Allow: /articles/call-of-duty-black-ops-7-campaign-review
251Allow: /articles/conjuring-movies-in-order
252Allow: /articles/crimson-desert-global-release-time-confirmed-as-preload-goes-live
253Allow: /articles/crimson-desert-review
254Allow: /articles/disney-debuts-new-avengers-doomsday-trailer-at-cinemacon-2026
255Allow: /articles/dress-to-impress-codes
256Allow: /articles/ghost-of-yotei-review
257Allow: /articles/gta-6-release-date-about-18-months-behind-original-target-take-two-boss-reveals-suggesting-spring-2025-was-once-the-goal-internally
258Allow: /articles/hollow-knight-silksong-global-release-times-confirmed
259Allow: /articles/ign-awards-2025-winners
260Allow: /articles/jurassic-world-rebirth-review-scarlett-johansson-mahershala-ali
261Allow: /articles/metroid-prime-4-beyond-review
262Allow: /articles/new-xbox-ceo-comments-on-game-pass
263Allow: /articles/nintendo-fans-demand-tariff-refunds-returned-to-customers
264Allow: /articles/nintendo-fans-surprised-to-find-switch-2-edition-game-cartridges-also-work-on-switch-1
265Allow: /articles/nintendo-is-absolutely-furious-former-employee-says-mario-zelda-developer-in-uncharted-territory-following-major-leaks
266Allow: /articles/nintendo-switch-2-price-hike-where-to-buy
267Allow: /articles/one-piece-series-movies-in-order
268Allow: /articles/phantom-blade-zero-is-a-delightfully-punishing-action-rpg
269Allow: /articles/plants-vs-brainrots-codes
270Allow: /articles/pokemon-fan-claims-world-first-completion-of-resident-evil-requiem-final-puzzle-but-claim-theyre-unsure-exactly-how-they-did-it
271Allow: /articles/resident-evil-requiem-review
272Allow: /articles/silent-hill-f-review
273Allow: /articles/sony-announces-price-rises-for-ps5-ps5-pro-and-playstation-portal-blames-continued-pressures-in-the-global-economic-landscape
274Allow: /articles/sony-confirms-playstation-plus-monthly-games-for-april-2026
275Allow: /articles/steam-machine-hands-on-first-impressions
276Allow: /articles/stranger-things-creators-promise-say-theyll-reveal-elevens-fate-in-20-years
277Allow: /articles/superman-2025-review-james-gunn-david-corenswet-rachel-brosnahan-nicholas-hoult
278Allow: /articles/the-10-best-pokemon-video-games
279Allow: /articles/the-best-100-video-games-of-all-time
280Allow: /articles/the-best-gaming-chairs
281Allow: /articles/the-best-ps5-games
282Allow: /articles/the-fate-anime-series-in-order
283Allow: /articles/the-forge-codes
284Allow: /articles/the-last-of-us-2-safe-combinations-and-codes
285Allow: /articles/video-game-release-dates-ps4-ps5-xbox-one-series-x-nintendo-switch
286Allow: /articles/what-to-expect-playstation-state-of-play-june-2026
287Allow: /articles/x-men-movies-in-order
288Allow: /playlist/rchnemesis/lists/top-100-indie-games
289Allow: /wikis/007-first-light/Essential_Tips_and_Tricks
290Allow: /wikis/arc-raiders/Avian_Alarm_Project_Guide_and_How_to_Complete
291Allow: /wikis/avatar-frontiers-of-pandora/All_Sarentu_Totem_Locations
292Allow: /wikis/baldurs-gate-3/The_Blood_of_Lathander
293Allow: /wikis/blue-prince/How_to_Solve_the_Chess_Piece_Puzzle
294Allow: /wikis/borderlands-4/Borderlands_4_SHiFT_Codes
295Allow: /wikis/call-of-duty-black-ops-7/Walkthrough
296Allow: /wikis/clair-obscur-expedition-33/All_Endings_and_How_to_Get_Them
297Allow: /wikis/clair-obscur-expedition-33/Walkthrough
298Allow: /wikis/crimson-desert/How_to_Get_the_Frostcursed_Armor_Set_-_All_Locations
299Allow: /wikis/crimson-desert/Smoking_Lands_Desert_Mouth_Maze_Puzzle_Solution
300Allow: /wikis/crimson-desert/Walkthrough
301Allow: /wikis/cyberpunk-2077/Panam_Romance_Guide
302Allow: /wikis/elden-ring/Millicent_Questline
303Allow: /wikis/elden-ring/Ranni_The_Witch_Questline
304Allow: /wikis/fallout-4/Companion_Perks
305Allow: /wikis/final-fantasy-7-rebirth/FF7_Rebirth_Walkthrough
306Allow: /wikis/forza-horizon-6/Barn_Finds
307Allow: /wikis/ghost-of-yotei/Trophy_List_and_Guide
308Allow: /wikis/grand-theft-auto-san-andreas/Cheat_Codes_and_Secrets
309Allow: /wikis/grow-a-garden/All_Bizzy_Bear_Honey_Crafter_Recipe_Quests_and_Rewards
310Allow: /wikis/grow-a-garden/All_Fruit_Plants_and_How_to_Get_Them
311Allow: /wikis/grow-a-garden/All_Vegetables_and_How_to_Get_Them
312Allow: /wikis/grow-a-garden/How_to_Make_a_Waffle
313Allow: /wikis/gta-5/All_GTA_5_Cell_Phone_Cheat_codes
314Allow: /wikis/gta-5/All_GTA_5_Cheat_Codes_and_Secrets_For_PC_and_Console
315Allow: /wikis/gta-5/GTA_Online_Weekly_Updates
316Allow: /wikis/hogwarts-legacy/All_Hogwarts_Secrets_Solutions
317Allow: /wikis/hogwarts-legacy/All_Merlin_Trials_Solutions
318Allow: /wikis/hogwarts-legacy/Demiguise_Statues_Locations
319Allow: /wikis/hollow-knight-silksong/Pale_Oil_Locations
320Allow: /wikis/hollow-knight-silksong/Widow_Boss_Guide
321Allow: /wikis/hollow-knight/walkthrough
322Allow: /wikis/kingdom-hearts-hd-15-remix/Kingdom_Hearts_Final_Mix_-_Walkthrough
323Allow: /wikis/la-noire/Walkthrough
324Allow: /wikis/mario-kart-world/All_Characters,_NPC_Drivers,_and_Outfits_List
325Allow: /wikis/neverness-to-everness/All_NTE_Codes_for_June_2026
326Allow: /wikis/paralives/Paralives_Cheats_List_and_Console_Commands
327Allow: /wikis/pokemon-firered-leafgreen-version/Best_Starter_to_Choose
328Allow: /wikis/pokemon-firered-leafgreen-version/Rock_Tunnel
329Allow: /wikis/pokemon-firered-leafgreen-version/Route_22
330Allow: /wikis/pokemon-legends-z-a/All_Mega_Pokemon
331Allow: /wikis/re2-remake/All_Safe_and_Lock_Combinations,_Codes,_and_Solutions
332Allow: /wikis/red-dead-redemption-2/Gold_Bar_Locations_and_Where_to_Sell_Them
333Allow: /wikis/red-dead-redemption-2/Horses_Guide_-_Locations,_Stats,_Bonding,_Taming,_and_Breeds
334Allow: /wikis/red-dead-redemption-2/Legendary_Animals_and_Fish_Locations_and_Map
335Allow: /wikis/resident-evil-requiem/File_Locations
336Allow: /wikis/resident-evil-requiem/Walkthrough
337Allow: /wikis/silent-hill-2-remake/Endings
338Allow: /wikis/silent-hill-f/Dark_Shrine_Finale_Walkthrough
339Allow: /wikis/silent-hill-f/Field_Puzzle_Guide
340Allow: /wikis/silent-hill-f/Light_Puzzle_Guide
341Allow: /wikis/silent-hill-f/Walkthrough
342Allow: /wikis/slime-rng-roblox/Slime_RNG_Codes
343Allow: /wikis/stardew-valley/Item_Codes_for_Spawning_Cheat
344Allow: /wikis/subnautica-2/Walkthrough_Hub
345Allow: /wikis/the-elder-scrolls-5-skyrim/PC_Console_Command_Cheats
346Allow: /wikis/the-forge/How_to_Reroll_Race
347Allow: /wikis/the-legend-of-zelda-breath-of-the-wild/Shrines
348Allow: /wikis/the-legend-of-zelda-tears-of-the-kingdom/All_Shrine_Locations_and_Solutions
349Allow: /wikis/vv-ultimatum-roblox/VV_ULTIMATUM_Codes
350Allow: /sponsored/
351Disallow: /
352
353User-agent: DuckDuckBot
354Allow: /
355
356Sitemap: https://www.ign.com/rss/sitemap-articles.xml
357Sitemap: https://www.ign.com/rss/sitemap-authors.xml
358Sitemap: https://www.ign.com/rss/sitemap-objects.xml
359Sitemap: https://www.ign.com/rss/sitemap-videos.xml
360Sitemap: https://www.ign.com/rss/sitemap-wikis.xml
361Sitemap: https://www.ign.com/rss/news/sitemap
362Sitemap: https://www.ign.com/rss/videos/sitemap-index-recently-updated
363Sitemap: https://www.ign.com/rss/wikis/sitemap-maps