1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26/*
27 * Driver to retire/unretire L2/L3 cachelines on panther
28 */
29#include <sys/types.h>
30#include <sys/types32.h>
31#include <sys/time.h>
32#include <sys/errno.h>
33#include <sys/cmn_err.h>
34#include <sys/param.h>
35#include <sys/modctl.h>
36#include <sys/conf.h>
37#include <sys/open.h>
38#include <sys/stat.h>
39#include <sys/ddi.h>
40#include <sys/sunddi.h>
41#include <sys/file.h>
42#include <sys/cpuvar.h>
43#include <sys/x_call.h>
44#include <sys/cheetahregs.h>
45#include <sys/mem_cache.h>
46#include <sys/mem_cache_ioctl.h>
47
48extern int	retire_l2(uint64_t, uint64_t);
49extern int	retire_l2_alternate(uint64_t, uint64_t);
50extern int	unretire_l2(uint64_t, uint64_t);
51extern int	unretire_l2_alternate(uint64_t, uint64_t);
52extern int	retire_l3(uint64_t, uint64_t);
53extern int	retire_l3_alternate(uint64_t, uint64_t);
54extern int	unretire_l3(uint64_t, uint64_t);
55extern int	unretire_l3_alternate(uint64_t, uint64_t);
56
57extern void	retire_l2_start(uint64_t, uint64_t);
58extern void	retire_l2_end(uint64_t, uint64_t);
59extern void	unretire_l2_start(uint64_t, uint64_t);
60extern void	unretire_l2_end(uint64_t, uint64_t);
61extern void	retire_l3_start(uint64_t, uint64_t);
62extern void	retire_l3_end(uint64_t, uint64_t);
63extern void	unretire_l3_start(uint64_t, uint64_t);
64extern void	unretire_l3_end(uint64_t, uint64_t);
65
66extern void	get_ecache_dtags_tl1(uint64_t, ch_cpu_logout_t *);
67extern void	get_l2_tag_tl1(uint64_t, uint64_t);
68extern void	get_l3_tag_tl1(uint64_t, uint64_t);
69extern const int _ncpu;
70
71/* Macro for putting 64-bit onto stack as two 32-bit ints */
72#define	PRTF_64_TO_32(x)	(uint32_t)((x)>>32), (uint32_t)(x)
73
74
75uint_t l2_flush_retries_done = 0;
76int mem_cache_debug = 0x0;
77uint64_t pattern = 0;
78uint32_t retire_failures = 0;
79#ifdef DEBUG
80int	inject_anonymous_tag_error = 0;
81int32_t last_error_injected_way = 0;
82uint8_t last_error_injected_bit = 0;
83int32_t last_l3tag_error_injected_way;
84uint8_t last_l3tag_error_injected_bit;
85int32_t last_l2tag_error_injected_way;
86uint8_t last_l2tag_error_injected_bit;
87#endif
88
89/* dev_ops and cb_ops entry point function declarations */
90static int	mem_cache_attach(dev_info_t *, ddi_attach_cmd_t);
91static int	mem_cache_detach(dev_info_t *, ddi_detach_cmd_t);
92static int	mem_cache_getinfo(dev_info_t *, ddi_info_cmd_t, void *,
93				void **);
94static int	mem_cache_open(dev_t *, int, int, cred_t *);
95static int	mem_cache_close(dev_t, int, int, cred_t *);
96static int	mem_cache_ioctl_ops(int, int, cache_info_t *);
97static int	mem_cache_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
98
99struct cb_ops mem_cache_cb_ops = {
100	mem_cache_open,
101	mem_cache_close,
102	nodev,
103	nodev,
104	nodev,			/* dump */
105	nodev,
106	nodev,
107	mem_cache_ioctl,
108	nodev,			/* devmap */
109	nodev,
110	ddi_segmap,		/* segmap */
111	nochpoll,
112	ddi_prop_op,
113	NULL,			/* for STREAMS drivers */
114	D_NEW | D_MP		/* driver compatibility flag */
115};
116
117static struct dev_ops mem_cache_dev_ops = {
118	DEVO_REV,		/* driver build version */
119	0,			/* device reference count */
120	mem_cache_getinfo,
121	nulldev,
122	nulldev,		/* probe */
123	mem_cache_attach,
124	mem_cache_detach,
125	nulldev,		/* reset */
126	&mem_cache_cb_ops,
127	(struct bus_ops *)NULL,
128	nulldev,			/* power */
129	ddi_quiesce_not_needed,		/* quiesce */
130};
131
132/*
133 * Soft state
134 */
135struct mem_cache_softc {
136	dev_info_t	*dip;
137	kmutex_t	mutex;
138};
139#define	getsoftc(inst)	((struct mem_cache_softc *)ddi_get_soft_state(statep,\
140			(inst)))
141
142/* module configuration stuff */
143static void *statep;
144extern struct mod_ops mod_driverops;
145
146static struct modldrv modldrv = {
147	&mod_driverops,
148	"mem_cache_driver (08/01/30) ",
149	&mem_cache_dev_ops
150};
151
152static struct modlinkage modlinkage = {
153	MODREV_1,
154	&modldrv,
155	0
156};
157
158extern const int _ncpu; /* Pull the kernel's global _ncpu definition */
159
160int
161_init(void)
162{
163	int e;
164
165	if (e = ddi_soft_state_init(&statep, sizeof (struct mem_cache_softc),
166	    MAX_MEM_CACHE_INSTANCES)) {
167		return (e);
168	}
169
170	if ((e = mod_install(&modlinkage)) != 0)
171		ddi_soft_state_fini(&statep);
172
173	return (e);
174}
175
176int
177_fini(void)
178{
179	int e;
180
181	if ((e = mod_remove(&modlinkage)) != 0)
182		return (e);
183
184	ddi_soft_state_fini(&statep);
185
186	return (DDI_SUCCESS);
187}
188
189int
190_info(struct modinfo *modinfop)
191{
192	return (mod_info(&modlinkage, modinfop));
193}
194
195/*ARGSUSED*/
196static int
197mem_cache_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
198{
199	int	inst;
200	int	retval = DDI_SUCCESS;
201	struct mem_cache_softc *softc;
202
203	inst = getminor((dev_t)arg);
204
205	switch (cmd) {
206	case DDI_INFO_DEVT2DEVINFO:
207		if ((softc = getsoftc(inst)) == NULL) {
208			*result = (void *)NULL;
209			retval = DDI_FAILURE;
210		} else
211			*result = (void *)softc->dip;
212		break;
213
214	case DDI_INFO_DEVT2INSTANCE:
215		*result = (void *)((uintptr_t)inst);
216		break;
217
218	default:
219		retval = DDI_FAILURE;
220	}
221
222	return (retval);
223}
224
225static int
226mem_cache_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
227{
228	int inst;
229	struct mem_cache_softc *softc = NULL;
230	char name[80];
231
232	switch (cmd) {
233	case DDI_ATTACH:
234		inst = ddi_get_instance(dip);
235		if (inst >= MAX_MEM_CACHE_INSTANCES) {
236			cmn_err(CE_WARN, "attach failed, too many instances\n");
237			return (DDI_FAILURE);
238		}
239		(void) sprintf(name, MEM_CACHE_DRIVER_NAME"%d", inst);
240		if (ddi_create_priv_minor_node(dip, name,
241		    S_IFCHR,
242		    inst,
243		    DDI_PSEUDO,
244		    0, NULL, "all", 0640) ==
245		    DDI_FAILURE) {
246			ddi_remove_minor_node(dip, NULL);
247			return (DDI_FAILURE);
248		}
249
250		/* Allocate a soft state structure for this instance */
251		if (ddi_soft_state_zalloc(statep, inst) != DDI_SUCCESS) {
252			cmn_err(CE_WARN, " ddi_soft_state_zalloc() failed "
253			    "for inst %d\n", inst);
254			goto attach_failed;
255		}
256
257		/* Setup soft state */
258		softc = getsoftc(inst);
259		softc->dip = dip;
260		mutex_init(&softc->mutex, NULL, MUTEX_DRIVER, NULL);
261
262		/* Create main environmental node */
263		ddi_report_dev(dip);
264
265		return (DDI_SUCCESS);
266
267	case DDI_RESUME:
268		return (DDI_SUCCESS);
269
270	default:
271		return (DDI_FAILURE);
272	}
273
274attach_failed:
275
276	/* Free soft state, if allocated. remove minor node if added earlier */
277	if (softc)
278		ddi_soft_state_free(statep, inst);
279
280	ddi_remove_minor_node(dip, NULL);
281
282	return (DDI_FAILURE);
283}
284
285static int
286mem_cache_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
287{
288	int inst;
289	struct mem_cache_softc *softc;
290
291	switch (cmd) {
292	case DDI_DETACH:
293		inst = ddi_get_instance(dip);
294		if ((softc = getsoftc(inst)) == NULL)
295			return (ENXIO);
296
297		/* Free the soft state and remove minor node added earlier */
298		mutex_destroy(&softc->mutex);
299		ddi_soft_state_free(statep, inst);
300		ddi_remove_minor_node(dip, NULL);
301		return (DDI_SUCCESS);
302
303	case DDI_SUSPEND:
304		return (DDI_SUCCESS);
305
306	default:
307		return (DDI_FAILURE);
308	}
309}
310
311/*ARGSUSED*/
312static int
313mem_cache_open(dev_t *devp, int flag, int otyp, cred_t *credp)
314{
315	int	inst = getminor(*devp);
316
317	return (getsoftc(inst) == NULL ? ENXIO : 0);
318}
319
320/*ARGSUSED*/
321static int
322mem_cache_close(dev_t dev, int flag, int otyp, cred_t *credp)
323{
324	int	inst = getminor(dev);
325
326	return (getsoftc(inst) == NULL ? ENXIO : 0);
327}
328
329static char *tstate_to_desc[] = {
330	"Invalid",			/* 0 */
331	"Shared",			/* 1 */
332	"Exclusive",			/* 2 */
333	"Owner",			/* 3 */
334	"Modified",			/* 4 */
335	"NA",				/* 5 */
336	"Owner/Shared",			/* 6 */
337	"Reserved(7)",			/* 7 */
338};
339
340static char *
341tag_state_to_desc(uint8_t tagstate)
342{
343	return (tstate_to_desc[tagstate & CH_ECSTATE_MASK]);
344}
345
346void
347print_l2_tag(uint64_t tag_addr, uint64_t l2_tag)
348{
349	uint64_t l2_subaddr;
350	uint8_t	l2_state;
351
352	l2_subaddr = PN_L2TAG_TO_PA(l2_tag);
353	l2_subaddr |= (tag_addr & PN_L2_INDEX_MASK);
354
355	l2_state = (l2_tag & CH_ECSTATE_MASK);
356	cmn_err(CE_CONT,
357	    "PA=0x%08x.%08x E$tag 0x%08x.%08x E$state %s\n",
358	    PRTF_64_TO_32(l2_subaddr),
359	    PRTF_64_TO_32(l2_tag),
360	    tag_state_to_desc(l2_state));
361}
362
363void
364print_l2cache_line(ch_cpu_logout_t *clop)
365{
366	uint64_t l2_subaddr;
367	int i, offset;
368	uint8_t	way, l2_state;
369	ch_ec_data_t *ecp;
370
371
372	for (way = 0; way < PN_CACHE_NWAYS; way++) {
373		ecp = &clop->clo_data.chd_l2_data[way];
374		l2_subaddr = PN_L2TAG_TO_PA(ecp->ec_tag);
375		l2_subaddr |= (ecp->ec_idx & PN_L2_INDEX_MASK);
376
377		l2_state = (ecp->ec_tag & CH_ECSTATE_MASK);
378		cmn_err(CE_CONT,
379		    "\nWAY = %d index = 0x%08x PA=0x%08x.%08x\n"
380		    "E$tag 0x%08x.%08x E$state %s",
381		    way, (uint32_t)ecp->ec_idx, PRTF_64_TO_32(l2_subaddr),
382		    PRTF_64_TO_32(ecp->ec_tag),
383		    tag_state_to_desc(l2_state));
384		/*
385		 * Dump out Ecache subblock data captured.
386		 * For Cheetah, we need to compute the ECC for each 16-byte
387		 * chunk and compare it with the captured chunk ECC to figure
388		 * out which chunk is bad.
389		 */
390		for (i = 0; i < (CH_ECACHE_SUBBLK_SIZE/16); i++) {
391			ec_data_elm_t *ecdptr;
392			uint64_t d_low, d_high;
393			uint32_t ecc;
394			int l2_data_idx = (i/2);
395
396			offset = i * 16;
397			ecdptr = &clop->clo_data.chd_l2_data[way].ec_data
398			    [l2_data_idx];
399			if ((i & 1) == 0) {
400				ecc = (ecdptr->ec_eccd >> 9) & 0x1ff;
401				d_high = ecdptr->ec_d8[0];
402				d_low  = ecdptr->ec_d8[1];
403			} else {
404				ecc = ecdptr->ec_eccd & 0x1ff;
405				d_high = ecdptr->ec_d8[2];
406				d_low  = ecdptr->ec_d8[3];
407			}
408
409			cmn_err(CE_CONT,
410			    "\nE$Data (0x%02x) 0x%08x.%08x 0x%08x.%08x"
411			    " ECC 0x%03x",
412			    offset, PRTF_64_TO_32(d_high),
413			    PRTF_64_TO_32(d_low), ecc);
414		}
415	}	/* end of for way loop */
416}
417
418void
419print_ecache_line(ch_cpu_logout_t *clop)
420{
421	uint64_t ec_subaddr;
422	int i, offset;
423	uint8_t	way, ec_state;
424	ch_ec_data_t *ecp;
425
426
427	for (way = 0; way < PN_CACHE_NWAYS; way++) {
428		ecp = &clop->clo_data.chd_ec_data[way];
429		ec_subaddr = PN_L3TAG_TO_PA(ecp->ec_tag);
430		ec_subaddr |= (ecp->ec_idx & PN_L3_TAG_RD_MASK);
431
432		ec_state = (ecp->ec_tag & CH_ECSTATE_MASK);
433		cmn_err(CE_CONT,
434		    "\nWAY = %d index = 0x%08x PA=0x%08x.%08x\n"
435		    "E$tag 0x%08x.%08x E$state %s",
436		    way, (uint32_t)ecp->ec_idx, PRTF_64_TO_32(ec_subaddr),
437		    PRTF_64_TO_32(ecp->ec_tag),
438		    tag_state_to_desc(ec_state));
439		/*
440		 * Dump out Ecache subblock data captured.
441		 * For Cheetah, we need to compute the ECC for each 16-byte
442		 * chunk and compare it with the captured chunk ECC to figure
443		 * out which chunk is bad.
444		 */
445		for (i = 0; i < (CH_ECACHE_SUBBLK_SIZE/16); i++) {
446			ec_data_elm_t *ecdptr;
447			uint64_t d_low, d_high;
448			uint32_t ecc;
449			int ec_data_idx = (i/2);
450
451			offset = i * 16;
452			ecdptr =
453			    &clop->clo_data.chd_ec_data[way].ec_data
454			    [ec_data_idx];
455			if ((i & 1) == 0) {
456				ecc = (ecdptr->ec_eccd >> 9) & 0x1ff;
457				d_high = ecdptr->ec_d8[0];
458				d_low  = ecdptr->ec_d8[1];
459			} else {
460				ecc = ecdptr->ec_eccd & 0x1ff;
461				d_high = ecdptr->ec_d8[2];
462				d_low  = ecdptr->ec_d8[3];
463			}
464
465			cmn_err(CE_CONT,
466			    "\nE$Data (0x%02x) 0x%08x.%08x 0x%08x.%08x"
467			    " ECC 0x%03x",
468			    offset, PRTF_64_TO_32(d_high),
469			    PRTF_64_TO_32(d_low), ecc);
470		}
471	}
472}
473
474static boolean_t
475tag_addr_collides(uint64_t tag_addr, cache_id_t type,
476    retire_func_t start_of_func, retire_func_t end_of_func)
477{
478	uint64_t start_paddr, end_paddr;
479	char *type_str;
480
481	start_paddr = va_to_pa((void *)start_of_func);
482	end_paddr = va_to_pa((void *)end_of_func);
483	switch (type) {
484		case L2_CACHE_TAG:
485		case L2_CACHE_DATA:
486			tag_addr &= PN_L2_INDEX_MASK;
487			start_paddr &= PN_L2_INDEX_MASK;
488			end_paddr &= PN_L2_INDEX_MASK;
489			type_str = "L2:";
490			break;
491		case L3_CACHE_TAG:
492		case L3_CACHE_DATA:
493			tag_addr &= PN_L3_TAG_RD_MASK;
494			start_paddr &= PN_L3_TAG_RD_MASK;
495			end_paddr &= PN_L3_TAG_RD_MASK;
496			type_str = "L3:";
497			break;
498		default:
499			/*
500			 * Should never reach here.
501			 */
502			ASSERT(0);
503			return (B_FALSE);
504	}
505	if ((tag_addr > (start_paddr - 0x100)) &&
506	    (tag_addr < (end_paddr + 0x100))) {
507		if (mem_cache_debug & 0x1)
508			cmn_err(CE_CONT,
509			    "%s collision detected tag_addr = 0x%08x"
510			    " start_paddr = 0x%08x end_paddr = 0x%08x\n",
511			    type_str, (uint32_t)tag_addr, (uint32_t)start_paddr,
512			    (uint32_t)end_paddr);
513		return (B_TRUE);
514	}
515	else
516		return (B_FALSE);
517}
518
519static uint64_t
520get_tag_addr(cache_info_t *cache_info)
521{
522	uint64_t tag_addr, scratch;
523
524	switch (cache_info->cache) {
525		case L2_CACHE_TAG:
526		case L2_CACHE_DATA:
527			tag_addr = (uint64_t)(cache_info->index <<
528			    PN_CACHE_LINE_SHIFT);
529			scratch = (uint64_t)(cache_info->way <<
530			    PN_L2_WAY_SHIFT);
531			tag_addr |= scratch;
532			tag_addr |= PN_L2_IDX_HW_ECC_EN;
533			break;
534		case L3_CACHE_TAG:
535		case L3_CACHE_DATA:
536			tag_addr = (uint64_t)(cache_info->index <<
537			    PN_CACHE_LINE_SHIFT);
538			scratch = (uint64_t)(cache_info->way <<
539			    PN_L3_WAY_SHIFT);
540			tag_addr |= scratch;
541			tag_addr |= PN_L3_IDX_HW_ECC_EN;
542			break;
543		default:
544			/*
545			 * Should never reach here.
546			 */
547			ASSERT(0);
548			return (uint64_t)(0);
549	}
550	return (tag_addr);
551}
552
553static int
554mem_cache_ioctl_ops(int cmd, int mode, cache_info_t *cache_info)
555{
556	int	ret_val = 0;
557	uint64_t afar, tag_addr;
558	ch_cpu_logout_t clop;
559	uint64_t Lxcache_tag_data[PN_CACHE_NWAYS];
560	int	i, retire_retry_count;
561	cpu_t	*cpu;
562	uint64_t tag_data;
563	uint8_t state;
564
565	if (cache_info->way >= PN_CACHE_NWAYS)
566		return (EINVAL);
567	switch (cache_info->cache) {
568		case L2_CACHE_TAG:
569		case L2_CACHE_DATA:
570			if (cache_info->index >=
571			    (PN_L2_SET_SIZE/PN_L2_LINESIZE))
572				return (EINVAL);
573			break;
574		case L3_CACHE_TAG:
575		case L3_CACHE_DATA:
576			if (cache_info->index >=
577			    (PN_L3_SET_SIZE/PN_L3_LINESIZE))
578				return (EINVAL);
579			break;
580		default:
581			return (ENOTSUP);
582	}
583	/*
584	 * Check if we have a valid cpu ID and that
585	 * CPU is ONLINE.
586	 */
587	mutex_enter(&cpu_lock);
588	cpu = cpu_get(cache_info->cpu_id);
589	if ((cpu == NULL) || (!cpu_is_online(cpu))) {
590		mutex_exit(&cpu_lock);
591		return (EINVAL);
592	}
593	mutex_exit(&cpu_lock);
594	pattern = 0;	/* default value of TAG PA when cacheline is retired. */
595	switch (cmd) {
596		case MEM_CACHE_RETIRE:
597			tag_addr = get_tag_addr(cache_info);
598			pattern |= PN_ECSTATE_NA;
599			retire_retry_count = 0;
600			affinity_set(cache_info->cpu_id);
601			switch (cache_info->cache) {
602				case L2_CACHE_DATA:
603				case L2_CACHE_TAG:
604					if ((cache_info->bit & MSB_BIT_MASK) ==
605					    MSB_BIT_MASK)
606						pattern |= PN_L2TAG_PA_MASK;
607retry_l2_retire:
608					if (tag_addr_collides(tag_addr,
609					    cache_info->cache,
610					    retire_l2_start, retire_l2_end))
611						ret_val =
612						    retire_l2_alternate(
613						    tag_addr, pattern);
614					else
615						ret_val = retire_l2(tag_addr,
616						    pattern);
617					if (ret_val == 1) {
618						/*
619						 * cacheline was in retired
620						 * STATE already.
621						 * so return success.
622						 */
623						ret_val = 0;
624					}
625					if (ret_val < 0) {
626						cmn_err(CE_WARN,
627		"retire_l2() failed. index = 0x%x way %d. Retrying...\n",
628						    cache_info->index,
629						    cache_info->way);
630						if (retire_retry_count >= 2) {
631							retire_failures++;
632							affinity_clear();
633							return (EIO);
634						}
635						retire_retry_count++;
636						goto retry_l2_retire;
637					}
638					if (ret_val == 2)
639						l2_flush_retries_done++;
640			/*
641			 * We bind ourself to a CPU and send cross trap to
642			 * ourself. On return from xt_one we can rely on the
643			 * data in tag_data being filled in. Normally one would
644			 * do a xt_sync to make sure that the CPU has completed
645			 * the cross trap call xt_one.
646			 */
647					xt_one(cache_info->cpu_id,
648					    (xcfunc_t *)(get_l2_tag_tl1),
649					    tag_addr, (uint64_t)(&tag_data));
650					state = tag_data & CH_ECSTATE_MASK;
651					if (state != PN_ECSTATE_NA) {
652						retire_failures++;
653						print_l2_tag(tag_addr,
654						    tag_data);
655						cmn_err(CE_WARN,
656		"L2 RETIRE:failed for index 0x%x way %d. Retrying...\n",
657						    cache_info->index,
658						    cache_info->way);
659						if (retire_retry_count >= 2) {
660							retire_failures++;
661							affinity_clear();
662							return (EIO);
663						}
664						retire_retry_count++;
665						goto retry_l2_retire;
666					}
667					break;
668				case L3_CACHE_TAG:
669				case L3_CACHE_DATA:
670					if ((cache_info->bit & MSB_BIT_MASK) ==
671					    MSB_BIT_MASK)
672						pattern |= PN_L3TAG_PA_MASK;
673					if (tag_addr_collides(tag_addr,
674					    cache_info->cache,
675					    retire_l3_start, retire_l3_end))
676						ret_val =
677						    retire_l3_alternate(
678						    tag_addr, pattern);
679					else
680						ret_val = retire_l3(tag_addr,
681						    pattern);
682					if (ret_val == 1) {
683						/*
684						 * cacheline was in retired
685						 * STATE already.
686						 * so return success.
687						 */
688						ret_val = 0;
689					}
690					if (ret_val < 0) {
691						cmn_err(CE_WARN,
692			"retire_l3() failed. ret_val = %d index = 0x%x\n",
693						    ret_val,
694						    cache_info->index);
695						retire_failures++;
696						affinity_clear();
697						return (EIO);
698					}
699			/*
700			 * We bind ourself to a CPU and send cross trap to
701			 * ourself. On return from xt_one we can rely on the
702			 * data in tag_data being filled in. Normally one would
703			 * do a xt_sync to make sure that the CPU has completed
704			 * the cross trap call xt_one.
705			 */
706					xt_one(cache_info->cpu_id,
707					    (xcfunc_t *)(get_l3_tag_tl1),
708					    tag_addr, (uint64_t)(&tag_data));
709					state = tag_data & CH_ECSTATE_MASK;
710					if (state != PN_ECSTATE_NA) {
711						cmn_err(CE_WARN,
712					"L3 RETIRE failed for index 0x%x\n",
713						    cache_info->index);
714						retire_failures++;
715						affinity_clear();
716						return (EIO);
717					}
718
719					break;
720			}
721			affinity_clear();
722			break;
723		case MEM_CACHE_UNRETIRE:
724			tag_addr = get_tag_addr(cache_info);
725			pattern = PN_ECSTATE_INV;
726			affinity_set(cache_info->cpu_id);
727			switch (cache_info->cache) {
728				case L2_CACHE_DATA:
729				case L2_CACHE_TAG:
730			/*
731			 * We bind ourself to a CPU and send cross trap to
732			 * ourself. On return from xt_one we can rely on the
733			 * data in tag_data being filled in. Normally one would
734			 * do a xt_sync to make sure that the CPU has completed
735			 * the cross trap call xt_one.
736			 */
737					xt_one(cache_info->cpu_id,
738					    (xcfunc_t *)(get_l2_tag_tl1),
739					    tag_addr, (uint64_t)(&tag_data));
740					state = tag_data & CH_ECSTATE_MASK;
741					if (state != PN_ECSTATE_NA) {
742						affinity_clear();
743						return (EINVAL);
744					}
745					if (tag_addr_collides(tag_addr,
746					    cache_info->cache,
747					    unretire_l2_start, unretire_l2_end))
748						ret_val =
749						    unretire_l2_alternate(
750						    tag_addr, pattern);
751					else
752						ret_val =
753						    unretire_l2(tag_addr,
754						    pattern);
755					if (ret_val != 0) {
756						cmn_err(CE_WARN,
757			"unretire_l2() failed. ret_val = %d index = 0x%x\n",
758						    ret_val,
759						    cache_info->index);
760						retire_failures++;
761						affinity_clear();
762						return (EIO);
763					}
764					break;
765				case L3_CACHE_TAG:
766				case L3_CACHE_DATA:
767			/*
768			 * We bind ourself to a CPU and send cross trap to
769			 * ourself. On return from xt_one we can rely on the
770			 * data in tag_data being filled in. Normally one would
771			 * do a xt_sync to make sure that the CPU has completed
772			 * the cross trap call xt_one.
773			 */
774					xt_one(cache_info->cpu_id,
775					    (xcfunc_t *)(get_l3_tag_tl1),
776					    tag_addr, (uint64_t)(&tag_data));
777					state = tag_data & CH_ECSTATE_MASK;
778					if (state != PN_ECSTATE_NA) {
779						affinity_clear();
780						return (EINVAL);
781					}
782					if (tag_addr_collides(tag_addr,
783					    cache_info->cache,
784					    unretire_l3_start, unretire_l3_end))
785						ret_val =
786						    unretire_l3_alternate(
787						    tag_addr, pattern);
788					else
789						ret_val =
790						    unretire_l3(tag_addr,
791						    pattern);
792					if (ret_val != 0) {
793						cmn_err(CE_WARN,
794			"unretire_l3() failed. ret_val = %d index = 0x%x\n",
795						    ret_val,
796						    cache_info->index);
797						affinity_clear();
798						return (EIO);
799					}
800					break;
801			}
802			affinity_clear();
803			break;
804		case MEM_CACHE_ISRETIRED:
805		case MEM_CACHE_STATE:
806			return (ENOTSUP);
807		case MEM_CACHE_READ_TAGS:
808#ifdef DEBUG
809		case MEM_CACHE_READ_ERROR_INJECTED_TAGS:
810#endif
811			/*
812			 * Read tag and data for all the ways at a given afar
813			 */
814			afar = (uint64_t)(cache_info->index
815			    << PN_CACHE_LINE_SHIFT);
816			mutex_enter(&cpu_lock);
817			affinity_set(cache_info->cpu_id);
818			pause_cpus(NULL, NULL);
819			mutex_exit(&cpu_lock);
820			/*
821			 * We bind ourself to a CPU and send cross trap to
822			 * ourself. On return from xt_one we can rely on the
823			 * data in clop being filled in. Normally one would
824			 * do a xt_sync to make sure that the CPU has completed
825			 * the cross trap call xt_one.
826			 */
827			xt_one(cache_info->cpu_id,
828			    (xcfunc_t *)(get_ecache_dtags_tl1),
829			    afar, (uint64_t)(&clop));
830			mutex_enter(&cpu_lock);
831			(void) start_cpus();
832			mutex_exit(&cpu_lock);
833			affinity_clear();
834			switch (cache_info->cache) {
835				case L2_CACHE_TAG:
836					for (i = 0; i < PN_CACHE_NWAYS; i++) {
837						Lxcache_tag_data[i] =
838						    clop.clo_data.chd_l2_data
839						    [i].ec_tag;
840					}
841#ifdef DEBUG
842					last_error_injected_bit =
843					    last_l2tag_error_injected_bit;
844					last_error_injected_way =
845					    last_l2tag_error_injected_way;
846#endif
847					break;
848				case L3_CACHE_TAG:
849					for (i = 0; i < PN_CACHE_NWAYS; i++) {
850						Lxcache_tag_data[i] =
851						    clop.clo_data.chd_ec_data
852						    [i].ec_tag;
853					}
854#ifdef DEBUG
855					last_error_injected_bit =
856					    last_l3tag_error_injected_bit;
857					last_error_injected_way =
858					    last_l3tag_error_injected_way;
859#endif
860					break;
861				default:
862					return (ENOTSUP);
863			}	/* end if switch(cache) */
864#ifdef DEBUG
865			if ((cmd == MEM_CACHE_READ_ERROR_INJECTED_TAGS) &&
866			    (inject_anonymous_tag_error == 0) &&
867			    (last_error_injected_way >= 0) &&
868			    (last_error_injected_way <= 3)) {
869				pattern = ((uint64_t)1 <<
870				    last_error_injected_bit);
871				/*
872				 * If error bit is ECC we need to make sure
873				 * ECC on all all WAYS are corrupted.
874				 */
875				if ((last_error_injected_bit >= 6) &&
876				    (last_error_injected_bit <= 14)) {
877					for (i = 0; i < PN_CACHE_NWAYS; i++)
878						Lxcache_tag_data[i] ^=
879						    pattern;
880				} else
881					Lxcache_tag_data
882					    [last_error_injected_way] ^=
883					    pattern;
884			}
885#endif
886			if (ddi_copyout((caddr_t)Lxcache_tag_data,
887			    (caddr_t)cache_info->datap,
888			    sizeof (Lxcache_tag_data), mode)
889			    != DDI_SUCCESS) {
890				return (EFAULT);
891			}
892			break;	/* end of READ_TAGS */
893		default:
894			return (ENOTSUP);
895	}	/* end if switch(cmd) */
896	return (ret_val);
897}
898
899/*ARGSUSED*/
900static int
901mem_cache_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
902		int *rvalp)
903{
904	int	inst;
905	struct mem_cache_softc *softc;
906	cache_info_t	cache_info;
907	cache_info32_t	cache_info32;
908	int	ret_val;
909	int	is_panther;
910
911	inst = getminor(dev);
912	if ((softc = getsoftc(inst)) == NULL)
913		return (ENXIO);
914
915	mutex_enter(&softc->mutex);
916
917#ifdef _MULTI_DATAMODEL
918	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
919		if (ddi_copyin((cache_info32_t *)arg, &cache_info32,
920		    sizeof (cache_info32), mode) != DDI_SUCCESS) {
921			mutex_exit(&softc->mutex);
922			return (EFAULT);
923		}
924		cache_info.cache = cache_info32.cache;
925		cache_info.index = cache_info32.index;
926		cache_info.way = cache_info32.way;
927		cache_info.cpu_id = cache_info32.cpu_id;
928		cache_info.bit = cache_info32.bit;
929		cache_info.datap = (void *)((uint64_t)cache_info32.datap);
930	} else
931#endif
932	if (ddi_copyin((cache_info_t *)arg, &cache_info,
933	    sizeof (cache_info), mode) != DDI_SUCCESS) {
934		mutex_exit(&softc->mutex);
935		return (EFAULT);
936	}
937
938	if ((cache_info.cpu_id < 0) || (cache_info.cpu_id >= _ncpu)) {
939		mutex_exit(&softc->mutex);
940		return (EINVAL);
941	}
942	is_panther = IS_PANTHER(cpunodes[cache_info.cpu_id].implementation);
943	if (!is_panther) {
944		mutex_exit(&softc->mutex);
945		return (ENOTSUP);
946	}
947	switch (cmd) {
948		case MEM_CACHE_RETIRE:
949		case MEM_CACHE_UNRETIRE:
950			if ((mode & FWRITE) == 0) {
951				ret_val = EBADF;
952				break;
953			}
954		/*FALLTHROUGH*/
955		case MEM_CACHE_ISRETIRED:
956		case MEM_CACHE_STATE:
957		case MEM_CACHE_READ_TAGS:
958#ifdef DEBUG
959		case MEM_CACHE_READ_ERROR_INJECTED_TAGS:
960#endif
961			ret_val =  mem_cache_ioctl_ops(cmd, mode, &cache_info);
962			break;
963		default:
964			ret_val = ENOTSUP;
965			break;
966	}
967	mutex_exit(&softc->mutex);
968	return (ret_val);
969}
970