NeuralCrawl

AT&T / robots.txt

← back to att.com · change detected 22d ago (2026-10-16T00:44:51Z) · modified +46 −48

What changed

Diff old (2026-10-13T23:51:59Z) → new (2026-10-16T00:44:51Z)

@@ -1,21 +1,25 @@
1- # Bad Robots!
2-
31  User-agent: DotBot
42  Disallow: /
53 
64  User-agent: dotbot
75  Disallow: /
86 
9-
107  User-agent: *
11- Content-Signal: search=yes, ai-input=yes, ai-train=yes
8+ Content-Signal: search=yes, ai-input=yes, ai-train=no
129  Allow: /
1310 
1411  # Public AI-agent discovery resources
1512  Allow: /.well-known/agent-card.json
1613  Allow: /.well-known/mcp/server-card.json
1714  Allow: /.well-known/agent-skills/index.json
15+ Allow: /.well-known/ai-catalog.json
16+ Allow: /.well-known/api-catalog.txt
1817  Allow: /ai-instructions/shopping-assistant.md
18+ Allow: /ai.txt
19+ Allow: /llms.txt
20+ Allow: /llms-full.txt
21+ Allow: /trust.txt
22+ Allow: /openapi.json
1923 
2024  Disallow: /wireless/switch-and-save-b/
2125  Disallow: /redressG/
@@ -28,26 +32,26 @@
2832 
2933  # edits 07.29.25 - start
3034 
31- # Allow exact homepage
32- Allow: /es-us/$
35+ # Allow exact homepage
36+ Allow: /es-us/$
3337 
34- # Allow exact folder roots only (no subfolders/files)
38+ # Allow exact folder roots only (no subfolders/files)
3539 
36- Allow: /es-us/local/$
37- Allow: /es-us/local/internet/$
38- Allow: /es-us/local/fiber/$
39- Allow: /es-us/local/cell-phones/$
40- Allow: /es-us/local/bundles/$
41- Allow: /es-us/stores/$
40+ Allow: /es-us/local/$
41+ Allow: /es-us/local/internet/$
42+ Allow: /es-us/local/fiber/$
43+ Allow: /es-us/local/cell-phones/$
44+ Allow: /es-us/local/bundles/$
45+ Allow: /es-us/stores/$
4246 
43- # Block all subfolders and files inside these folders
47+ # Block all subfolders and files inside these folders
4448 
45- Disallow: /es-us/local/*
46- Disallow: /es-us/local/internet/*
47- Disallow: /es-us/local/fiber/*
48- Disallow: /es-us/local/cell-phones/*
49- Disallow: /es-us/local/bundles/*
50- Disallow: /es-us/stores/*
49+ Disallow: /es-us/local/*
50+ Disallow: /es-us/local/internet/*
51+ Disallow: /es-us/local/fiber/*
52+ Disallow: /es-us/local/cell-phones/*
53+ Disallow: /es-us/local/bundles/*
54+ Disallow: /es-us/stores/*
5155 
5256  # edits 07.29.25 - end
5357 
@@ -67,14 +71,13 @@
6771 
6872  Disallow: /search/
6973  Disallow: /cfd/
70- Disallow: *bvstate=pg*
71- Disallow: */rest/clicktrack?*
72- Disallow: */rest/getlist?*
74+ Disallow: /*bvstate=pg*
75+ Disallow: /*/rest/clicktrack?*
76+ Disallow: /*/rest/getlist?*
7377  Disallow: /buy/postpaid/line-selection
7478  Disallow: /ssaf/ssafc/
7579  Disallow: /att-dgt-checkin-e/api/*
76- Disallow: */sdattservicesinc/*
77- Disallow: https://pre-www.att.com/home-phone/lifeline/
80+ Disallow: /*/sdattservicesinc/*
7881 
7982  # edits 01.25.25 - end
8083 
@@ -145,7 +148,7 @@
145148 
146149  Disallow: /smallbusiness/404.jsp?
147150  Disallow: /smallbusiness/b?
148- Disallow: /smallbusiness/search
151+ Disallow: /smallbusiness/search
149152  Disallow: /smallbusiness/wireless/buyFlowContainer.jsp
150153  Disallow: /smallbusiness/i9nvalidParamInput.jsp
151154  Disallow: /es-us/OLAM_PROD_CMS/*
@@ -191,25 +194,24 @@
191194 
192195  Disallow: /es-us/smallbusiness/404.jsp?
193196  Disallow: /es-us/smallbusiness/b?
194- Disallow: /es-us/smallbusiness/search
197+ Disallow: /es-us/smallbusiness/search
195198  Disallow: /es-us/smallbusiness/wireless/buyFlowContainer.jsp
196199  Disallow: /es-us/smallbusiness/i9nvalidParamInput.jsp
197200 
198201  User-agent: Googlebot
199202 
200203  Disallow: /gtm/
201- Disallow: https://pre-www.att.com/home-phone/lifeline/
202204 
203205  # edits 05.01.24 - start
204206 
205207  Disallow: /search/
206208  Disallow: /cfd/
207- Disallow: *bvstate=pg*
208- Disallow: */rest/clicktrack?*
209- Disallow: */rest/getlist?*
209+ Disallow: /*bvstate=pg*
210+ Disallow: /*/rest/clicktrack?*
211+ Disallow: /*/rest/getlist?*
210212  Disallow: /buy/postpaid/line-selection
211213  Disallow: /ssaf/ssafc/
212- Disallow: */sdattservicesinc/*
214+ Disallow: /*/sdattservicesinc/*
213215 
214216  # edits 01.25.25 - end
215217 
@@ -278,7 +280,7 @@
278280 
279281  Disallow: /smallbusiness/404.jsp?
280282  Disallow: /smallbusiness/b?
281- Disallow: /smallbusiness/search
283+ Disallow: /smallbusiness/search
282284  Disallow: /smallbusiness/wireless/buyFlowContainer.jsp
283285  Disallow: /smallbusiness/i9nvalidParamInput.jsp
284286  Disallow: /es-us/OLAM_PROD_CMS/*
@@ -326,19 +328,15 @@
326328 
327329  Disallow: /es-us/smallbusiness/404.jsp?
328330  Disallow: /es-us/smallbusiness/b?
329- Disallow: /es-us/smallbusiness/search
331+ Disallow: /es-us/smallbusiness/search
330332  Disallow: /es-us/smallbusiness/wireless/buyFlowContainer.jsp
331333  Disallow: /es-us/smallbusiness/i9nvalidParamInput.jsp
332334 
333-
334- # PROD-34469
335+ # PROD-34469
335336 
336337  Disallow: /u-verse/availability/pending-order.jsp
337338  Disallow: /u-verse/availability/check-customer-type.jsp
338-
339-
340- # PROD-34470
341-
339+ # PROD-34470
342340  Disallow: /shop/unified/availability.html?destination_page=BYOB
343341  Disallow: /shop/unified/availability/results.html
344342  Disallow: /shop/unified/availability/calltoorder.html
@@ -348,16 +346,16 @@
348346  Disallow: /myorders/*
349347  Disallow: /acctmgmt/*
350348 
351-
352- # Previous Update 8/14/2023
353- # Last Update 10.13.2026
349+ # Previous Update 8/14/2023
350+ # Last Update 10.15.2026
354351 
355352  # ====================================================================
356- # Global Pointers (Placed at the bottom of the file)
353+ # Global pointers
357354  # ====================================================================
358355 
359356  Sitemap: https://www.att.com/sitemap.xml
360- agent-card: https://www.att.com/.well-known/agent-card.json
361- mcp-server: https://www.att.com/.well-known/mcp/server-card.json
362- agent-skills: https://www.att.com/.well-known/agent-skills/index.json
363- ai-instructions: https://www.att.com/ai-instructions/shopping-assistant.md
357+ # Agent discovery resources are published through trust.txt and ai.txt.
358+ # https://www.att.com/.well-known/agent-card.json
359+ # https://www.att.com/.well-known/mcp/server-card.json
360+ # https://www.att.com/.well-known/agent-skills/index.json
361+ # https://www.att.com/ai-instructions/shopping-assistant.md