/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright 2015 Joyent, Inc. */ /* * lgroup system calls */ #include #include #include #include #include #include #include #include #include /* for prom_printf() */ #include #include #include /* definitions for mi_validity */ #define VALID_ADDR 1 #define VALID_REQ 2 /* * run through the given number of addresses and requests and return the * corresponding memory information for each address */ static int meminfo(int addr_count, struct meminfo *mip) { size_t in_size, out_size, req_size, val_size; struct as *as; struct hat *hat; int i, j, out_idx, info_count; lgrp_t *lgrp; pfn_t pfn; ssize_t pgsz; int *req_array, *val_array; uint64_t *in_array, *out_array; uint64_t addr, paddr; uintptr_t vaddr; int ret = 0; struct meminfo minfo; #if defined(_SYSCALL32_IMPL) struct meminfo32 minfo32; #endif /* * Make sure that there is at least one address to translate and * limit how many virtual addresses the kernel can do per call */ if (addr_count < 1) return (set_errno(EINVAL)); else if (addr_count > MAX_MEMINFO_CNT) addr_count = MAX_MEMINFO_CNT; if (get_udatamodel() == DATAMODEL_NATIVE) { if (copyin(mip, &minfo, sizeof (struct meminfo))) return (set_errno(EFAULT)); } #if defined(_SYSCALL32_IMPL) else { bzero(&minfo, sizeof (minfo)); if (copyin(mip, &minfo32, sizeof (struct meminfo32))) return (set_errno(EFAULT)); minfo.mi_inaddr = (const uint64_t *)(uintptr_t) minfo32.mi_inaddr; minfo.mi_info_req = (const uint_t *)(uintptr_t) minfo32.mi_info_req; minfo.mi_info_count = minfo32.mi_info_count; minfo.mi_outdata = (uint64_t *)(uintptr_t) minfo32.mi_outdata; minfo.mi_validity = (uint_t *)(uintptr_t) minfo32.mi_validity; } #endif /* * all the input parameters have been copied in:- * addr_count - number of input addresses * minfo.mi_inaddr - array of input addresses * minfo.mi_info_req - array of types of information requested * minfo.mi_info_count - no. of pieces of info requested for each addr * minfo.mi_outdata - array into which the results are placed * minfo.mi_validity - array containing bitwise result codes; 0th bit * evaluates validity of corresponding input * address, 1st bit validity of response to first * member of info_req, etc. */ /* make sure mi_info_count is within limit */ info_count = minfo.mi_info_count; if (info_count < 1 || info_count > MAX_MEMINFO_REQ) return (set_errno(EINVAL)); /* * allocate buffer in_array for the input addresses and copy them in */ in_size = sizeof (uint64_t) * addr_count; in_array = kmem_alloc(in_size, KM_SLEEP); if (copyin(minfo.mi_inaddr, in_array, in_size)) { kmem_free(in_array, in_size); return (set_errno(EFAULT)); } /* * allocate buffer req_array for the input info_reqs and copy them in */ req_size = sizeof (uint_t) * info_count; req_array = kmem_alloc(req_size, KM_SLEEP); if (copyin(minfo.mi_info_req, req_array, req_size)) { kmem_free(req_array, req_size); kmem_free(in_array, in_size); return (set_errno(EFAULT)); } /* * Validate privs for each req. */ for (i = 0; i < info_count; i++) { switch (req_array[i] & MEMINFO_MASK) { case MEMINFO_VLGRP: case MEMINFO_VPAGESIZE: break; default: if (secpolicy_meminfo(CRED()) != 0) { kmem_free(req_array, req_size); kmem_free(in_array, in_size); return (set_errno(EPERM)); } break; } } /* * allocate buffer out_array which holds the results and will have * to be copied out later */ out_size = sizeof (uint64_t) * addr_count * info_count; out_array = kmem_alloc(out_size, KM_SLEEP); /* * allocate buffer val_array which holds the validity bits and will * have to be copied out later */ val_size = sizeof (uint_t) * addr_count; val_array = kmem_alloc(val_size, KM_SLEEP); if ((req_array[0] & MEMINFO_MASK) == MEMINFO_PLGRP) { /* find the corresponding lgroup for each physical address */ for (i = 0; i < addr_count; i++) { paddr = in_array[i]; pfn = btop(paddr); lgrp = lgrp_pfn_to_lgrp(pfn); if (lgrp) { out_array[i] = lgrp->lgrp_id; val_array[i] = VALID_ADDR | VALID_REQ; } else { out_array[i] = 0; val_array[i] = 0; } } } else { /* get the corresponding memory info for each virtual address */ as = curproc->p_as; AS_LOCK_ENTER(as, RW_READER); hat = as->a_hat; for (i = out_idx = 0; i < addr_count; i++, out_idx += info_count) { addr = in_array[i]; vaddr = (uintptr_t)(addr & ~PAGEOFFSET); if (!as_segat(as, (caddr_t)vaddr)) { val_array[i] = 0; continue; } val_array[i] = VALID_ADDR; pfn = hat_getpfnum(hat, (caddr_t)vaddr); if (pfn != PFN_INVALID) { paddr = (uint64_t)((pfn << PAGESHIFT) | (addr & PAGEOFFSET)); for (j = 0; j < info_count; j++) { switch (req_array[j] & MEMINFO_MASK) { case MEMINFO_VPHYSICAL: /* * return the physical address * corresponding to the input * virtual address */ out_array[out_idx + j] = paddr; val_array[i] |= VALID_REQ << j; break; case MEMINFO_VLGRP: /* * return the lgroup of physical * page corresponding to the * input virtual address */ lgrp = lgrp_pfn_to_lgrp(pfn); if (lgrp) { out_array[out_idx + j] = lgrp->lgrp_id; val_array[i] |= VALID_REQ << j; } break; case MEMINFO_VPAGESIZE: /* * return the size of physical * page corresponding to the * input virtual address */ pgsz = hat_getpagesize(hat, (caddr_t)vaddr); if (pgsz != -1) { out_array[out_idx + j] = pgsz; val_array[i] |= VALID_REQ << j; } break; case MEMINFO_VREPLCNT: /* * for future use:- * return the no. replicated * physical pages corresponding * to the input virtual address, * so it is always 0 at the * moment */ out_array[out_idx + j] = 0; val_array[i] |= VALID_REQ << j; break; case MEMINFO_VREPL: /* * for future use:- * return the nth physical * replica of the specified * virtual address */ break; case MEMINFO_VREPL_LGRP: /* * for future use:- * return the lgroup of nth * physical replica of the * specified virtual address */ break; case MEMINFO_PLGRP: /* * this is for physical address * only, shouldn't mix with * virtual address */ break; default: break; } } } } AS_LOCK_EXIT(as); } /* copy out the results and validity bits and free the buffers */ if ((copyout(out_array, minfo.mi_outdata, out_size) != 0) || (copyout(val_array, minfo.mi_validity, val_size) != 0)) ret = set_errno(EFAULT); kmem_free(in_array, in_size); kmem_free(out_array, out_size); kmem_free(req_array, req_size); kmem_free(val_array, val_size); return (ret); } /* * Initialize lgroup affinities for thread */ void lgrp_affinity_init(lgrp_affinity_t **bufaddr) { if (bufaddr) *bufaddr = NULL; } /* * Free lgroup affinities for thread and set to NULL * just in case thread gets recycled */ void lgrp_affinity_free(lgrp_affinity_t **bufaddr) { if (bufaddr && *bufaddr) { kmem_free(*bufaddr, nlgrpsmax * sizeof (lgrp_affinity_t)); *bufaddr = NULL; } } #define P_ANY -2 /* cookie specifying any ID */ /* * Find LWP with given ID in specified process and get its affinity for * specified lgroup */ lgrp_affinity_t lgrp_affinity_get_thread(proc_t *p, id_t lwpid, lgrp_id_t lgrp) { lgrp_affinity_t aff; int found; kthread_t *t; ASSERT(MUTEX_HELD(&p->p_lock)); aff = LGRP_AFF_NONE; found = 0; t = p->p_tlist; /* * The process may be executing in proc_exit() and its p->p_list may be * already NULL. */ if (t == NULL) return (set_errno(ESRCH)); do { if (t->t_tid == lwpid || lwpid == P_ANY) { thread_lock(t); /* * Check to see whether caller has permission to set * affinity for LWP */ if (t->t_cid == 0 || !hasprocperm(t->t_cred, CRED())) { thread_unlock(t); return (set_errno(EPERM)); } if (t->t_lgrp_affinity) aff = t->t_lgrp_affinity[lgrp]; thread_unlock(t); found = 1; break; } } while ((t = t->t_forw) != p->p_tlist); if (!found) aff = set_errno(ESRCH); return (aff); } /* * Get lgroup affinity for given LWP */ lgrp_affinity_t lgrp_affinity_get(lgrp_affinity_args_t *ap) { lgrp_affinity_t aff; lgrp_affinity_args_t args; id_t id; idtype_t idtype; lgrp_id_t lgrp; proc_t *p; kthread_t *t; /* * Copyin arguments */ if (copyin(ap, &args, sizeof (lgrp_affinity_args_t)) != 0) return (set_errno(EFAULT)); id = args.id; idtype = args.idtype; lgrp = args.lgrp; /* * Check for invalid lgroup */ if (lgrp < 0 || lgrp == LGRP_NONE) return (set_errno(EINVAL)); /* * Check for existing lgroup */ if (lgrp > lgrp_alloc_max) return (set_errno(ESRCH)); /* * Get lgroup affinity for given LWP or process */ switch (idtype) { case P_LWPID: /* * LWP in current process */ p = curproc; mutex_enter(&p->p_lock); if (id != P_MYID) /* different thread */ aff = lgrp_affinity_get_thread(p, id, lgrp); else { /* current thread */ aff = LGRP_AFF_NONE; t = curthread; thread_lock(t); if (t->t_lgrp_affinity) aff = t->t_lgrp_affinity[lgrp]; thread_unlock(t); } mutex_exit(&p->p_lock); break; case P_PID: /* * Process */ mutex_enter(&pidlock); if (id == P_MYID) p = curproc; else { p = prfind(id); if (p == NULL) { mutex_exit(&pidlock); return (set_errno(ESRCH)); } } mutex_enter(&p->p_lock); aff = lgrp_affinity_get_thread(p, P_ANY, lgrp); mutex_exit(&p->p_lock); mutex_exit(&pidlock); break; default: aff = set_errno(EINVAL); break; } return (aff); } /* * Find lgroup for which this thread has most affinity in specified partition * starting from home lgroup unless specified starting lgroup is preferred */ lpl_t * lgrp_affinity_best(kthread_t *t, struct cpupart *cpupart, lgrp_id_t start, boolean_t prefer_start) { lgrp_affinity_t *affs; lgrp_affinity_t best_aff; lpl_t *best_lpl; lgrp_id_t finish; lgrp_id_t home; lgrp_id_t lgrpid; lpl_t *lpl; ASSERT(t != NULL); ASSERT((MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0) || (MUTEX_HELD(&ttoproc(t)->p_lock) && THREAD_LOCK_HELD(t))); ASSERT(cpupart != NULL); if (t->t_lgrp_affinity == NULL) return (NULL); affs = t->t_lgrp_affinity; /* * Thread bound to CPU */ if (t->t_bind_cpu != PBIND_NONE) { cpu_t *cp; /* * Find which lpl has most affinity among leaf lpl directly * containing CPU and its ancestor lpls */ cp = cpu[t->t_bind_cpu]; best_lpl = lpl = cp->cpu_lpl; best_aff = affs[best_lpl->lpl_lgrpid]; while (lpl->lpl_parent != NULL) { lpl = lpl->lpl_parent; lgrpid = lpl->lpl_lgrpid; if (affs[lgrpid] > best_aff) { best_lpl = lpl; best_aff = affs[lgrpid]; } } return (best_lpl); } /* * Start searching from home lgroup unless given starting lgroup is * preferred or home lgroup isn't in given pset. Use root lgroup as * starting point if both home and starting lgroups aren't in given * pset. */ ASSERT(start >= 0 && start <= lgrp_alloc_max); home = t->t_lpl->lpl_lgrpid; if (!prefer_start && LGRP_CPUS_IN_PART(home, cpupart)) lgrpid = home; else if (start != LGRP_NONE && LGRP_CPUS_IN_PART(start, cpupart)) lgrpid = start; else lgrpid = LGRP_ROOTID; best_lpl = &cpupart->cp_lgrploads[lgrpid]; best_aff = affs[lgrpid]; finish = lgrpid; do { /* * Skip any lgroups that don't have CPU resources * in this processor set. */ if (!LGRP_CPUS_IN_PART(lgrpid, cpupart)) { if (++lgrpid > lgrp_alloc_max) lgrpid = 0; /* wrap the search */ continue; } /* * Find lgroup with most affinity */ lpl = &cpupart->cp_lgrploads[lgrpid]; if (affs[lgrpid] > best_aff) { best_aff = affs[lgrpid]; best_lpl = lpl; } if (++lgrpid > lgrp_alloc_max) lgrpid = 0; /* wrap the search */ } while (lgrpid != finish); /* * No lgroup (in this pset) with any affinity */ if (best_aff == LGRP_AFF_NONE) return (NULL); lgrpid = best_lpl->lpl_lgrpid; ASSERT(LGRP_CPUS_IN_PART(lgrpid, cpupart) && best_lpl->lpl_ncpu > 0); return (best_lpl); } /* * Set thread's affinity for given lgroup */ int lgrp_affinity_set_thread(kthread_t *t, lgrp_id_t lgrp, lgrp_affinity_t aff, lgrp_affinity_t **aff_buf) { lgrp_affinity_t *affs; lgrp_id_t best; lpl_t *best_lpl; lgrp_id_t home; int retval; ASSERT(t != NULL); ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock)); retval = 0; thread_lock(t); /* * Check to see whether caller has permission to set affinity for * thread */ if (t->t_cid == 0 || !hasprocperm(t->t_cred, CRED())) { thread_unlock(t); return (set_errno(EPERM)); } if (t->t_lgrp_affinity == NULL) { if (aff == LGRP_AFF_NONE) { thread_unlock(t); return (0); } ASSERT(aff_buf != NULL && *aff_buf != NULL); t->t_lgrp_affinity = *aff_buf; *aff_buf = NULL; } affs = t->t_lgrp_affinity; affs[lgrp] = aff; /* * Find lgroup for which thread has most affinity, * starting with lgroup for which affinity being set */ best_lpl = lgrp_affinity_best(t, t->t_cpupart, lgrp, B_TRUE); /* * Rehome if found lgroup with more affinity than home or lgroup for * which affinity is being set has same affinity as home */ home = t->t_lpl->lpl_lgrpid; if (best_lpl != NULL && best_lpl != t->t_lpl) { best = best_lpl->lpl_lgrpid; if (affs[best] > affs[home] || (affs[best] == affs[home] && best == lgrp)) lgrp_move_thread(t, best_lpl, 1); } thread_unlock(t); return (retval); } /* * Set process' affinity for specified lgroup */ int lgrp_affinity_set_proc(proc_t *p, lgrp_id_t lgrp, lgrp_affinity_t aff, lgrp_affinity_t **aff_buf_array) { lgrp_affinity_t *buf; int err = 0; int i; int retval; kthread_t *t; ASSERT(MUTEX_HELD(&pidlock) && MUTEX_HELD(&p->p_lock)); ASSERT(aff_buf_array != NULL); i = 0; t = p->p_tlist; if (t != NULL) { do { /* * Set lgroup affinity for thread */ buf = aff_buf_array[i]; retval = lgrp_affinity_set_thread(t, lgrp, aff, &buf); if (err == 0 && retval != 0) err = retval; /* * Advance pointer to next buffer */ if (buf == NULL) { ASSERT(i < p->p_lwpcnt); aff_buf_array[i] = NULL; i++; } } while ((t = t->t_forw) != p->p_tlist); } return (err); } /* * Set LWP's or process' affinity for specified lgroup * * When setting affinities, pidlock, process p_lock, and thread_lock() * need to be held in that order to protect target thread's pset, process, * process contents, and thread contents. thread_lock() does splhigh(), * so it ends up having similiar effect as kpreempt_disable(), so it will * protect calls to lgrp_move_thread() and lgrp_choose() from pset changes. */ int lgrp_affinity_set(lgrp_affinity_args_t *ap) { lgrp_affinity_t aff; lgrp_affinity_t *aff_buf; lgrp_affinity_args_t args; id_t id; idtype_t idtype; lgrp_id_t lgrp; int nthreads; proc_t *p; int retval; /* * Copyin arguments */ if (copyin(ap, &args, sizeof (lgrp_affinity_args_t)) != 0) return (set_errno(EFAULT)); idtype = args.idtype; id = args.id; lgrp = args.lgrp; aff = args.aff; /* * Check for invalid lgroup */ if (lgrp < 0 || lgrp == LGRP_NONE) return (set_errno(EINVAL)); /* * Check for existing lgroup */ if (lgrp > lgrp_alloc_max) return (set_errno(ESRCH)); /* * Check for legal affinity */ if (aff != LGRP_AFF_NONE && aff != LGRP_AFF_WEAK && aff != LGRP_AFF_STRONG) return (set_errno(EINVAL)); /* * Must be process or LWP ID */ if (idtype != P_LWPID && idtype != P_PID) return (set_errno(EINVAL)); retval = EINVAL; /* * Set given LWP's or process' affinity for specified lgroup */ switch (idtype) { case P_LWPID: /* * Allocate memory for thread's lgroup affinities * ahead of time w/o holding locks */ aff_buf = kmem_zalloc(nlgrpsmax * sizeof (lgrp_affinity_t), KM_SLEEP); p = curproc; /* * Set affinity for thread */ mutex_enter(&p->p_lock); if (id == P_MYID) { /* current thread */ retval = lgrp_affinity_set_thread(curthread, lgrp, aff, &aff_buf); } else if (p->p_tlist == NULL) { retval = set_errno(ESRCH); } else { /* other thread */ int found = 0; kthread_t *t; t = p->p_tlist; do { if (t->t_tid == id) { retval = lgrp_affinity_set_thread(t, lgrp, aff, &aff_buf); found = 1; break; } } while ((t = t->t_forw) != p->p_tlist); if (!found) retval = set_errno(ESRCH); } mutex_exit(&p->p_lock); /* * Free memory for lgroup affinities, * since thread didn't need it */ if (aff_buf) kmem_free(aff_buf, nlgrpsmax * sizeof (lgrp_affinity_t)); break; case P_PID: do { lgrp_affinity_t **aff_buf_array; int i; size_t size; /* * Get process */ mutex_enter(&pidlock); if (id == P_MYID) p = curproc; else p = prfind(id); if (p == NULL) { mutex_exit(&pidlock); return (set_errno(ESRCH)); } /* * Get number of threads in process * * NOTE: Only care about user processes, * so p_lwpcnt should be number of threads. */ mutex_enter(&p->p_lock); nthreads = p->p_lwpcnt; mutex_exit(&p->p_lock); mutex_exit(&pidlock); if (nthreads < 1) return (set_errno(ESRCH)); /* * Preallocate memory for lgroup affinities for * each thread in process now to avoid holding * any locks. Allocate an array to hold a buffer * for each thread. */ aff_buf_array = kmem_zalloc(nthreads * sizeof (lgrp_affinity_t *), KM_SLEEP); size = nlgrpsmax * sizeof (lgrp_affinity_t); for (i = 0; i < nthreads; i++) aff_buf_array[i] = kmem_zalloc(size, KM_SLEEP); mutex_enter(&pidlock); /* * Get process again since dropped locks to allocate * memory (except current process) */ if (id != P_MYID) p = prfind(id); /* * Process went away after we dropped locks and before * reacquiring them, so drop locks, free memory, and * return. */ if (p == NULL) { mutex_exit(&pidlock); for (i = 0; i < nthreads; i++) kmem_free(aff_buf_array[i], size); kmem_free(aff_buf_array, nthreads * sizeof (lgrp_affinity_t *)); return (set_errno(ESRCH)); } mutex_enter(&p->p_lock); /* * See whether number of threads is same * If not, drop locks, free memory, and try again */ if (nthreads != p->p_lwpcnt) { mutex_exit(&p->p_lock); mutex_exit(&pidlock); for (i = 0; i < nthreads; i++) kmem_free(aff_buf_array[i], size); kmem_free(aff_buf_array, nthreads * sizeof (lgrp_affinity_t *)); continue; } /* * Set lgroup affinity for threads in process */ retval = lgrp_affinity_set_proc(p, lgrp, aff, aff_buf_array); mutex_exit(&p->p_lock); mutex_exit(&pidlock); /* * Free any leftover memory, since some threads may * have already allocated memory and set lgroup * affinities before */ for (i = 0; i < nthreads; i++) if (aff_buf_array[i] != NULL) kmem_free(aff_buf_array[i], size); kmem_free(aff_buf_array, nthreads * sizeof (lgrp_affinity_t *)); break; } while (nthreads != p->p_lwpcnt); break; default: retval = set_errno(EINVAL); break; } return (retval); } /* * Return the latest generation number for the lgroup hierarchy * with the given view */ lgrp_gen_t lgrp_generation(lgrp_view_t view) { cpupart_t *cpupart; uint_t gen; kpreempt_disable(); /* * Determine generation number for given view */ if (view == LGRP_VIEW_OS) /* * Return generation number of lgroup hierarchy for OS view */ gen = lgrp_gen; else { /* * For caller's view, use generation numbers for lgroup * hierarchy and caller's pset * NOTE: Caller needs to check for change in pset ID */ cpupart = curthread->t_cpupart; ASSERT(cpupart); gen = lgrp_gen + cpupart->cp_gen; } kpreempt_enable(); return (gen); } lgrp_id_t lgrp_home_thread(kthread_t *t) { lgrp_id_t home; ASSERT(t != NULL); ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock)); thread_lock(t); /* * Check to see whether caller has permission to set affinity for * thread */ if (t->t_cid == 0 || !hasprocperm(t->t_cred, CRED())) { thread_unlock(t); return (set_errno(EPERM)); } home = lgrp_home_id(t); thread_unlock(t); return (home); } /* * Get home lgroup of given process or thread */ lgrp_id_t lgrp_home_get(idtype_t idtype, id_t id) { proc_t *p; lgrp_id_t retval; kthread_t *t; /* * Get home lgroup of given LWP or process */ switch (idtype) { case P_LWPID: p = curproc; /* * Set affinity for thread */ mutex_enter(&p->p_lock); if (id == P_MYID) { /* current thread */ retval = lgrp_home_thread(curthread); } else if (p->p_tlist == NULL) { retval = set_errno(ESRCH); } else { /* other thread */ int found = 0; t = p->p_tlist; do { if (t->t_tid == id) { retval = lgrp_home_thread(t); found = 1; break; } } while ((t = t->t_forw) != p->p_tlist); if (!found) retval = set_errno(ESRCH); } mutex_exit(&p->p_lock); break; case P_PID: /* * Get process */ mutex_enter(&pidlock); if (id == P_MYID) p = curproc; else p = prfind(id); if (p == NULL) { mutex_exit(&pidlock); return (set_errno(ESRCH)); } mutex_enter(&p->p_lock); t = p->p_tlist; if (t == NULL) retval = set_errno(ESRCH); else retval = lgrp_home_thread(t); mutex_exit(&p->p_lock); mutex_exit(&pidlock); break; default: retval = set_errno(EINVAL); break; } return (retval); } /* * Return latency between "from" and "to" lgroups * * This latency number can only be used for relative comparison * between lgroups on the running system, cannot be used across platforms, * and may not reflect the actual latency. It is platform and implementation * specific, so platform gets to decide its value. It would be nice if the * number was at least proportional to make comparisons more meaningful though. */ int lgrp_latency(lgrp_id_t from, lgrp_id_t to) { lgrp_t *from_lgrp; int i; int latency; int latency_max; lgrp_t *to_lgrp; ASSERT(MUTEX_HELD(&cpu_lock)); if (from < 0 || to < 0) return (set_errno(EINVAL)); if (from > lgrp_alloc_max || to > lgrp_alloc_max) return (set_errno(ESRCH)); from_lgrp = lgrp_table[from]; to_lgrp = lgrp_table[to]; if (!LGRP_EXISTS(from_lgrp) || !LGRP_EXISTS(to_lgrp)) { return (set_errno(ESRCH)); } /* * Get latency for same lgroup */ if (from == to) { latency = from_lgrp->lgrp_latency; return (latency); } /* * Get latency between leaf lgroups */ if (from_lgrp->lgrp_childcnt == 0 && to_lgrp->lgrp_childcnt == 0) return (lgrp_plat_latency(from_lgrp->lgrp_plathand, to_lgrp->lgrp_plathand)); /* * Determine max latency between resources in two lgroups */ latency_max = 0; for (i = 0; i <= lgrp_alloc_max; i++) { lgrp_t *from_rsrc; int j; lgrp_t *to_rsrc; from_rsrc = lgrp_table[i]; if (!LGRP_EXISTS(from_rsrc) || !klgrpset_ismember(from_lgrp->lgrp_set[LGRP_RSRC_CPU], i)) continue; for (j = 0; j <= lgrp_alloc_max; j++) { to_rsrc = lgrp_table[j]; if (!LGRP_EXISTS(to_rsrc) || klgrpset_ismember(to_lgrp->lgrp_set[LGRP_RSRC_MEM], j) == 0) continue; latency = lgrp_plat_latency(from_rsrc->lgrp_plathand, to_rsrc->lgrp_plathand); if (latency > latency_max) latency_max = latency; } } return (latency_max); } /* * Return lgroup interface version number * 0 - none * 1 - original * 2 - lgrp_latency_cookie() and lgrp_resources() added */ int lgrp_version(int version) { /* * Return LGRP_VER_NONE when requested version isn't supported */ if (version < LGRP_VER_NONE || version > LGRP_VER_CURRENT) return (LGRP_VER_NONE); /* * Return current version when LGRP_VER_NONE passed in */ if (version == LGRP_VER_NONE) return (LGRP_VER_CURRENT); /* * Otherwise, return supported version. */ return (version); } /* * Snapshot of lgroup hieararchy * * One snapshot is kept and is based on the kernel's native data model, so * a 32-bit snapshot is kept for the 32-bit kernel and a 64-bit one for the * 64-bit kernel. If a 32-bit user wants a snapshot from the 64-bit kernel, * the kernel generates a 32-bit snapshot from the data in its 64-bit snapshot. * * The format is defined by lgroup snapshot header and the layout of * the snapshot in memory is as follows: * 1) lgroup snapshot header * - specifies format of snapshot * - defined by lgrp_snapshot_header_t * 2) lgroup info array * - contains information about each lgroup * - one element for each lgroup * - each element is defined by lgrp_info_t * 3) lgroup CPU ID array * - contains list (array) of CPU IDs for each lgroup * - lgrp_info_t points into array and specifies how many CPUs belong to * given lgroup * 4) lgroup parents array * - contains lgroup bitmask of parents for each lgroup * - bitmask is an array of unsigned longs and its size depends on nlgrpsmax * 5) lgroup children array * - contains lgroup bitmask of children for each lgroup * - bitmask is an array of unsigned longs and its size depends on nlgrpsmax * 6) lgroup resources array * - contains lgroup bitmask of resources for each lgroup * - bitmask is an array of unsigned longs and its size depends on nlgrpsmax * 7) lgroup latency table * - contains latency from each lgroup to each of other lgroups * * NOTE: Must use nlgrpsmax for per lgroup data structures because lgroups * may be sparsely allocated. */ lgrp_snapshot_header_t *lgrp_snap = NULL; /* lgroup snapshot */ static kmutex_t lgrp_snap_lock; /* snapshot lock */ /* * Take a snapshot of lgroup hierarchy and return size of buffer * needed to hold snapshot */ static int lgrp_snapshot(void) { size_t bitmask_size; size_t bitmasks_size; size_t bufsize; int cpu_index; size_t cpuids_size; int i; int j; size_t info_size; size_t lats_size; ulong_t *lgrp_children; processorid_t *lgrp_cpuids; lgrp_info_t *lgrp_info; int **lgrp_lats; ulong_t *lgrp_parents; ulong_t *lgrp_rsets; ulong_t *lgrpset; int snap_ncpus; int snap_nlgrps; int snap_nlgrpsmax; size_t snap_hdr_size; #ifdef _SYSCALL32_IMPL model_t model = DATAMODEL_NATIVE; /* * Have up-to-date snapshot, so check to see whether caller is 32-bit * program and need to return size of 32-bit snapshot now. */ model = get_udatamodel(); if (model == DATAMODEL_ILP32 && lgrp_snap && lgrp_snap->ss_gen == lgrp_gen) { snap_nlgrpsmax = lgrp_snap->ss_nlgrps_max; /* * Calculate size of buffer needed for 32-bit snapshot, * rounding up size of each object to allow for alignment * of next object in buffer. */ snap_hdr_size = P2ROUNDUP(sizeof (lgrp_snapshot_header32_t), sizeof (caddr32_t)); info_size = P2ROUNDUP(snap_nlgrpsmax * sizeof (lgrp_info32_t), sizeof (processorid_t)); cpuids_size = P2ROUNDUP(lgrp_snap->ss_ncpus * sizeof (processorid_t), sizeof (ulong_t)); /* * lgroup bitmasks needed for parents, children, and resources * for each lgroup and pset lgroup set */ bitmask_size = BT_SIZEOFMAP(snap_nlgrpsmax); bitmasks_size = (((2 + LGRP_RSRC_COUNT) * snap_nlgrpsmax) + 1) * bitmask_size; /* * Size of latency table and buffer */ lats_size = snap_nlgrpsmax * sizeof (caddr32_t) + snap_nlgrpsmax * snap_nlgrpsmax * sizeof (int); bufsize = snap_hdr_size + info_size + cpuids_size + bitmasks_size + lats_size; return (bufsize); } #endif /* _SYSCALL32_IMPL */ /* * Check whether snapshot is up-to-date * Free it and take another one if not */ if (lgrp_snap) { if (lgrp_snap->ss_gen == lgrp_gen) return (lgrp_snap->ss_size); kmem_free(lgrp_snap, lgrp_snap->ss_size); lgrp_snap = NULL; } /* * Allocate memory for snapshot * w/o holding cpu_lock while waiting for memory */ while (lgrp_snap == NULL) { int old_generation; /* * Take snapshot of lgroup generation number * and configuration size dependent information * NOTE: Only count number of online CPUs, * since only online CPUs appear in lgroups. */ mutex_enter(&cpu_lock); old_generation = lgrp_gen; snap_ncpus = ncpus_online; snap_nlgrps = nlgrps; snap_nlgrpsmax = nlgrpsmax; mutex_exit(&cpu_lock); /* * Calculate size of buffer needed for snapshot, * rounding up size of each object to allow for alignment * of next object in buffer. */ snap_hdr_size = P2ROUNDUP(sizeof (lgrp_snapshot_header_t), sizeof (void *)); info_size = P2ROUNDUP(snap_nlgrpsmax * sizeof (lgrp_info_t), sizeof (processorid_t)); cpuids_size = P2ROUNDUP(snap_ncpus * sizeof (processorid_t), sizeof (ulong_t)); /* * lgroup bitmasks needed for pset lgroup set and parents, * children, and resource sets for each lgroup */ bitmask_size = BT_SIZEOFMAP(snap_nlgrpsmax); bitmasks_size = (((2 + LGRP_RSRC_COUNT) * snap_nlgrpsmax) + 1) * bitmask_size; /* * Size of latency table and buffer */ lats_size = snap_nlgrpsmax * sizeof (int *) + snap_nlgrpsmax * snap_nlgrpsmax * sizeof (int); bufsize = snap_hdr_size + info_size + cpuids_size + bitmasks_size + lats_size; /* * Allocate memory for buffer */ lgrp_snap = kmem_zalloc(bufsize, KM_NOSLEEP); if (lgrp_snap == NULL) return (set_errno(ENOMEM)); /* * Check whether generation number has changed */ mutex_enter(&cpu_lock); if (lgrp_gen == old_generation) break; /* hasn't change, so done. */ /* * Generation number changed, so free memory and try again. */ mutex_exit(&cpu_lock); kmem_free(lgrp_snap, bufsize); lgrp_snap = NULL; } /* * Fill in lgroup snapshot header * (including pointers to tables of lgroup info, CPU IDs, and parents * and children) */ lgrp_snap->ss_version = LGRP_VER_CURRENT; /* * XXX For now, liblgrp only needs to know whether the hierarchy * XXX only has one level or not */ if (snap_nlgrps == 1) lgrp_snap->ss_levels = 1; else lgrp_snap->ss_levels = 2; lgrp_snap->ss_root = LGRP_ROOTID; lgrp_snap->ss_nlgrps = lgrp_snap->ss_nlgrps_os = snap_nlgrps; lgrp_snap->ss_nlgrps_max = snap_nlgrpsmax; lgrp_snap->ss_ncpus = snap_ncpus; lgrp_snap->ss_gen = lgrp_gen; lgrp_snap->ss_view = LGRP_VIEW_OS; lgrp_snap->ss_pset = 0; /* NOTE: caller should set if needed */ lgrp_snap->ss_size = bufsize; lgrp_snap->ss_magic = (uintptr_t)lgrp_snap; lgrp_snap->ss_info = lgrp_info = (lgrp_info_t *)((uintptr_t)lgrp_snap + snap_hdr_size); lgrp_snap->ss_cpuids = lgrp_cpuids = (processorid_t *)((uintptr_t)lgrp_info + info_size); lgrp_snap->ss_lgrpset = lgrpset = (ulong_t *)((uintptr_t)lgrp_cpuids + cpuids_size); lgrp_snap->ss_parents = lgrp_parents = (ulong_t *)((uintptr_t)lgrpset + bitmask_size); lgrp_snap->ss_children = lgrp_children = (ulong_t *)((uintptr_t)lgrp_parents + (snap_nlgrpsmax * bitmask_size)); lgrp_snap->ss_rsets = lgrp_rsets = (ulong_t *)((uintptr_t)lgrp_children + (snap_nlgrpsmax * bitmask_size)); lgrp_snap->ss_latencies = lgrp_lats = (int **)((uintptr_t)lgrp_rsets + (LGRP_RSRC_COUNT * snap_nlgrpsmax * bitmask_size)); /* * Fill in lgroup information */ cpu_index = 0; for (i = 0; i < snap_nlgrpsmax; i++) { struct cpu *cp; int cpu_count; struct cpu *head; int k; lgrp_t *lgrp; lgrp = lgrp_table[i]; if (!LGRP_EXISTS(lgrp)) { bzero(&lgrp_info[i], sizeof (lgrp_info[i])); lgrp_info[i].info_lgrpid = LGRP_NONE; continue; } lgrp_info[i].info_lgrpid = i; lgrp_info[i].info_latency = lgrp->lgrp_latency; /* * Fill in parents, children, and lgroup resources */ lgrp_info[i].info_parents = (ulong_t *)((uintptr_t)lgrp_parents + (i * bitmask_size)); if (lgrp->lgrp_parent) BT_SET(lgrp_info[i].info_parents, lgrp->lgrp_parent->lgrp_id); lgrp_info[i].info_children = (ulong_t *)((uintptr_t)lgrp_children + (i * bitmask_size)); for (j = 0; j < snap_nlgrpsmax; j++) if (klgrpset_ismember(lgrp->lgrp_children, j)) BT_SET(lgrp_info[i].info_children, j); lgrp_info[i].info_rset = (ulong_t *)((uintptr_t)lgrp_rsets + (i * LGRP_RSRC_COUNT * bitmask_size)); for (j = 0; j < LGRP_RSRC_COUNT; j++) { ulong_t *rset; rset = (ulong_t *)((uintptr_t)lgrp_info[i].info_rset + (j * bitmask_size)); for (k = 0; k < snap_nlgrpsmax; k++) if (klgrpset_ismember(lgrp->lgrp_set[j], k)) BT_SET(rset, k); } /* * Fill in CPU IDs */ cpu_count = 0; lgrp_info[i].info_cpuids = NULL; cp = head = lgrp->lgrp_cpu; if (head != NULL) { lgrp_info[i].info_cpuids = &lgrp_cpuids[cpu_index]; do { lgrp_cpuids[cpu_index] = cp->cpu_id; cpu_index++; cpu_count++; cp = cp->cpu_next_lgrp; } while (cp != head); } ASSERT(cpu_count == lgrp->lgrp_cpucnt); lgrp_info[i].info_ncpus = cpu_count; /* * Fill in memory sizes for lgroups that directly contain * memory */ if (klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_MEM], i)) { lgrp_info[i].info_mem_free = lgrp_mem_size(i, LGRP_MEM_SIZE_FREE); lgrp_info[i].info_mem_install = lgrp_mem_size(i, LGRP_MEM_SIZE_INSTALL); } /* * Fill in latency table and buffer */ lgrp_lats[i] = (int *)((uintptr_t)lgrp_lats + snap_nlgrpsmax * sizeof (int *) + i * snap_nlgrpsmax * sizeof (int)); for (j = 0; j < snap_nlgrpsmax; j++) { lgrp_t *to; to = lgrp_table[j]; if (!LGRP_EXISTS(to)) continue; lgrp_lats[i][j] = lgrp_latency(lgrp->lgrp_id, to->lgrp_id); } } ASSERT(cpu_index == snap_ncpus); mutex_exit(&cpu_lock); #ifdef _SYSCALL32_IMPL /* * Check to see whether caller is 32-bit program and need to return * size of 32-bit snapshot now that snapshot has been taken/updated. * May not have been able to do this earlier if snapshot was out of * date or didn't exist yet. */ if (model == DATAMODEL_ILP32) { snap_nlgrpsmax = lgrp_snap->ss_nlgrps_max; /* * Calculate size of buffer needed for 32-bit snapshot, * rounding up size of each object to allow for alignment * of next object in buffer. */ snap_hdr_size = P2ROUNDUP(sizeof (lgrp_snapshot_header32_t), sizeof (caddr32_t)); info_size = P2ROUNDUP(snap_nlgrpsmax * sizeof (lgrp_info32_t), sizeof (processorid_t)); cpuids_size = P2ROUNDUP(lgrp_snap->ss_ncpus * sizeof (processorid_t), sizeof (ulong_t)); bitmask_size = BT_SIZEOFMAP(snap_nlgrpsmax); bitmasks_size = (((2 + LGRP_RSRC_COUNT) * snap_nlgrpsmax) + 1) * bitmask_size; /* * Size of latency table and buffer */ lats_size = (snap_nlgrpsmax * sizeof (caddr32_t)) + (snap_nlgrpsmax * snap_nlgrpsmax * sizeof (int)); bufsize = snap_hdr_size + info_size + cpuids_size + bitmasks_size + lats_size; return (bufsize); } #endif /* _SYSCALL32_IMPL */ return (lgrp_snap->ss_size); } /* * Copy snapshot into given user buffer, fix up any pointers in buffer to point * into user instead of kernel address space, and return size of buffer * needed to hold snapshot */ static int lgrp_snapshot_copy(char *buf, size_t bufsize) { size_t bitmask_size; int cpu_index; size_t cpuids_size; int i; size_t info_size; lgrp_info_t *lgrp_info; int retval; size_t snap_hdr_size; int snap_ncpus; int snap_nlgrpsmax; lgrp_snapshot_header_t *user_snap; lgrp_info_t *user_info; lgrp_info_t *user_info_buffer; processorid_t *user_cpuids; ulong_t *user_lgrpset; ulong_t *user_parents; ulong_t *user_children; int **user_lats; int **user_lats_buffer; ulong_t *user_rsets; if (lgrp_snap == NULL) return (0); if (buf == NULL || bufsize <= 0) return (lgrp_snap->ss_size); /* * User needs to try getting size of buffer again * because given buffer size is too small. * The lgroup hierarchy may have changed after they asked for the size * but before the snapshot was taken. */ if (bufsize < lgrp_snap->ss_size) return (set_errno(EAGAIN)); snap_ncpus = lgrp_snap->ss_ncpus; snap_nlgrpsmax = lgrp_snap->ss_nlgrps_max; /* * Fill in lgrpset now because caller may have change psets */ kpreempt_disable(); for (i = 0; i < snap_nlgrpsmax; i++) { if (klgrpset_ismember(curthread->t_cpupart->cp_lgrpset, i)) { BT_SET(lgrp_snap->ss_lgrpset, i); } } kpreempt_enable(); /* * Copy lgroup snapshot (snapshot header, lgroup info, and CPU IDs) * into user buffer all at once */ if (copyout(lgrp_snap, buf, lgrp_snap->ss_size) != 0) return (set_errno(EFAULT)); /* * Round up sizes of lgroup snapshot header and info for alignment */ snap_hdr_size = P2ROUNDUP(sizeof (lgrp_snapshot_header_t), sizeof (void *)); info_size = P2ROUNDUP(snap_nlgrpsmax * sizeof (lgrp_info_t), sizeof (processorid_t)); cpuids_size = P2ROUNDUP(snap_ncpus * sizeof (processorid_t), sizeof (ulong_t)); bitmask_size = BT_SIZEOFMAP(snap_nlgrpsmax); /* * Calculate pointers into user buffer for lgroup snapshot header, * info, and CPU IDs */ user_snap = (lgrp_snapshot_header_t *)buf; user_info = (lgrp_info_t *)((uintptr_t)user_snap + snap_hdr_size); user_cpuids = (processorid_t *)((uintptr_t)user_info + info_size); user_lgrpset = (ulong_t *)((uintptr_t)user_cpuids + cpuids_size); user_parents = (ulong_t *)((uintptr_t)user_lgrpset + bitmask_size); user_children = (ulong_t *)((uintptr_t)user_parents + (snap_nlgrpsmax * bitmask_size)); user_rsets = (ulong_t *)((uintptr_t)user_children + (snap_nlgrpsmax * bitmask_size)); user_lats = (int **)((uintptr_t)user_rsets + (LGRP_RSRC_COUNT * snap_nlgrpsmax * bitmask_size)); /* * Copyout magic number (ie. pointer to beginning of buffer) */ if (copyout(&buf, &user_snap->ss_magic, sizeof (buf)) != 0) return (set_errno(EFAULT)); /* * Fix up pointers in user buffer to point into user buffer * not kernel snapshot */ if (copyout(&user_info, &user_snap->ss_info, sizeof (user_info)) != 0) return (set_errno(EFAULT)); if (copyout(&user_cpuids, &user_snap->ss_cpuids, sizeof (user_cpuids)) != 0) return (set_errno(EFAULT)); if (copyout(&user_lgrpset, &user_snap->ss_lgrpset, sizeof (user_lgrpset)) != 0) return (set_errno(EFAULT)); if (copyout(&user_parents, &user_snap->ss_parents, sizeof (user_parents)) != 0) return (set_errno(EFAULT)); if (copyout(&user_children, &user_snap->ss_children, sizeof (user_children)) != 0) return (set_errno(EFAULT)); if (copyout(&user_rsets, &user_snap->ss_rsets, sizeof (user_rsets)) != 0) return (set_errno(EFAULT)); if (copyout(&user_lats, &user_snap->ss_latencies, sizeof (user_lats)) != 0) return (set_errno(EFAULT)); /* * Make copies of lgroup info and latency table, fix up pointers, * and then copy them into user buffer */ user_info_buffer = kmem_zalloc(info_size, KM_NOSLEEP); if (user_info_buffer == NULL) return (set_errno(ENOMEM)); user_lats_buffer = kmem_zalloc(snap_nlgrpsmax * sizeof (int *), KM_NOSLEEP); if (user_lats_buffer == NULL) { kmem_free(user_info_buffer, info_size); return (set_errno(ENOMEM)); } lgrp_info = (lgrp_info_t *)((uintptr_t)lgrp_snap + snap_hdr_size); bcopy(lgrp_info, user_info_buffer, info_size); cpu_index = 0; for (i = 0; i < snap_nlgrpsmax; i++) { ulong_t *snap_rset; /* * Skip non-existent lgroups */ if (user_info_buffer[i].info_lgrpid == LGRP_NONE) continue; /* * Update free memory size since it changes frequently * Only do so for lgroups directly containing memory * * NOTE: This must be done before changing the pointers to * point into user space since we need to dereference * lgroup resource set */ snap_rset = &lgrp_info[i].info_rset[LGRP_RSRC_MEM * BT_BITOUL(snap_nlgrpsmax)]; if (BT_TEST(snap_rset, i)) user_info_buffer[i].info_mem_free = lgrp_mem_size(i, LGRP_MEM_SIZE_FREE); /* * Fix up pointers to parents, children, resources, and * latencies */ user_info_buffer[i].info_parents = (ulong_t *)((uintptr_t)user_parents + (i * bitmask_size)); user_info_buffer[i].info_children = (ulong_t *)((uintptr_t)user_children + (i * bitmask_size)); user_info_buffer[i].info_rset = (ulong_t *)((uintptr_t)user_rsets + (i * LGRP_RSRC_COUNT * bitmask_size)); user_lats_buffer[i] = (int *)((uintptr_t)user_lats + (snap_nlgrpsmax * sizeof (int *)) + (i * snap_nlgrpsmax * sizeof (int))); /* * Fix up pointer to CPU IDs */ if (user_info_buffer[i].info_ncpus == 0) { user_info_buffer[i].info_cpuids = NULL; continue; } user_info_buffer[i].info_cpuids = &user_cpuids[cpu_index]; cpu_index += user_info_buffer[i].info_ncpus; } ASSERT(cpu_index == snap_ncpus); /* * Copy lgroup info and latency table with pointers fixed up to point * into user buffer out to user buffer now */ retval = lgrp_snap->ss_size; if (copyout(user_info_buffer, user_info, info_size) != 0) retval = set_errno(EFAULT); kmem_free(user_info_buffer, info_size); if (copyout(user_lats_buffer, user_lats, snap_nlgrpsmax * sizeof (int *)) != 0) retval = set_errno(EFAULT); kmem_free(user_lats_buffer, snap_nlgrpsmax * sizeof (int *)); return (retval); } #ifdef _SYSCALL32_IMPL /* * Make 32-bit copy of snapshot, fix up any pointers in buffer to point * into user instead of kernel address space, copy 32-bit snapshot into * given user buffer, and return size of buffer needed to hold snapshot */ static int lgrp_snapshot_copy32(caddr32_t buf, size32_t bufsize) { size32_t bitmask_size; size32_t bitmasks_size; size32_t children_size; int cpu_index; size32_t cpuids_size; int i; int j; size32_t info_size; size32_t lats_size; lgrp_info_t *lgrp_info; lgrp_snapshot_header32_t *lgrp_snap32; lgrp_info32_t *lgrp_info32; processorid_t *lgrp_cpuids32; caddr32_t *lgrp_lats32; int **lgrp_lats32_kernel; uint_t *lgrp_set32; uint_t *lgrp_parents32; uint_t *lgrp_children32; uint_t *lgrp_rsets32; size32_t parents_size; size32_t rsets_size; size32_t set_size; size32_t snap_hdr_size; int snap_ncpus; int snap_nlgrpsmax; size32_t snap_size; if (lgrp_snap == NULL) return (0); snap_ncpus = lgrp_snap->ss_ncpus; snap_nlgrpsmax = lgrp_snap->ss_nlgrps_max; /* * Calculate size of buffer needed for 32-bit snapshot, * rounding up size of each object to allow for alignment * of next object in buffer. */ snap_hdr_size = P2ROUNDUP(sizeof (lgrp_snapshot_header32_t), sizeof (caddr32_t)); info_size = P2ROUNDUP(snap_nlgrpsmax * sizeof (lgrp_info32_t), sizeof (processorid_t)); cpuids_size = P2ROUNDUP(snap_ncpus * sizeof (processorid_t), sizeof (ulong_t)); bitmask_size = BT_SIZEOFMAP32(snap_nlgrpsmax); set_size = bitmask_size; parents_size = snap_nlgrpsmax * bitmask_size; children_size = snap_nlgrpsmax * bitmask_size; rsets_size = P2ROUNDUP(LGRP_RSRC_COUNT * snap_nlgrpsmax * (int)bitmask_size, sizeof (caddr32_t)); bitmasks_size = set_size + parents_size + children_size + rsets_size; /* * Size of latency table and buffer */ lats_size = (snap_nlgrpsmax * sizeof (caddr32_t)) + (snap_nlgrpsmax * snap_nlgrpsmax * sizeof (int)); snap_size = snap_hdr_size + info_size + cpuids_size + bitmasks_size + lats_size; if (buf == 0 || bufsize <= 0) { return (snap_size); } /* * User needs to try getting size of buffer again * because given buffer size is too small. * The lgroup hierarchy may have changed after they asked for the size * but before the snapshot was taken. */ if (bufsize < snap_size) return (set_errno(EAGAIN)); /* * Make 32-bit copy of snapshot, fix up pointers to point into user * buffer not kernel, and then copy whole thing into user buffer */ lgrp_snap32 = kmem_zalloc(snap_size, KM_NOSLEEP); if (lgrp_snap32 == NULL) return (set_errno(ENOMEM)); /* * Calculate pointers into 32-bit copy of snapshot * for lgroup info, CPU IDs, pset lgroup bitmask, parents, children, * resources, and latency table and buffer */ lgrp_info32 = (lgrp_info32_t *)((uintptr_t)lgrp_snap32 + snap_hdr_size); lgrp_cpuids32 = (processorid_t *)((uintptr_t)lgrp_info32 + info_size); lgrp_set32 = (uint_t *)((uintptr_t)lgrp_cpuids32 + cpuids_size); lgrp_parents32 = (uint_t *)((uintptr_t)lgrp_set32 + set_size); lgrp_children32 = (uint_t *)((uintptr_t)lgrp_parents32 + parents_size); lgrp_rsets32 = (uint_t *)((uintptr_t)lgrp_children32 + children_size); lgrp_lats32 = (caddr32_t *)((uintptr_t)lgrp_rsets32 + rsets_size); /* * Make temporary lgroup latency table of pointers for kernel to use * to fill in rows of table with latencies from each lgroup */ lgrp_lats32_kernel = kmem_zalloc(snap_nlgrpsmax * sizeof (int *), KM_NOSLEEP); if (lgrp_lats32_kernel == NULL) { kmem_free(lgrp_snap32, snap_size); return (set_errno(ENOMEM)); } /* * Fill in 32-bit lgroup snapshot header * (with pointers into user's buffer for lgroup info, CPU IDs, * bit masks, and latencies) */ lgrp_snap32->ss_version = lgrp_snap->ss_version; lgrp_snap32->ss_levels = lgrp_snap->ss_levels; lgrp_snap32->ss_nlgrps = lgrp_snap32->ss_nlgrps_os = lgrp_snap->ss_nlgrps; lgrp_snap32->ss_nlgrps_max = snap_nlgrpsmax; lgrp_snap32->ss_root = lgrp_snap->ss_root; lgrp_snap32->ss_ncpus = lgrp_snap->ss_ncpus; lgrp_snap32->ss_gen = lgrp_snap->ss_gen; lgrp_snap32->ss_view = LGRP_VIEW_OS; lgrp_snap32->ss_size = snap_size; lgrp_snap32->ss_magic = buf; lgrp_snap32->ss_info = buf + snap_hdr_size; lgrp_snap32->ss_cpuids = lgrp_snap32->ss_info + info_size; lgrp_snap32->ss_lgrpset = lgrp_snap32->ss_cpuids + cpuids_size; lgrp_snap32->ss_parents = lgrp_snap32->ss_lgrpset + bitmask_size; lgrp_snap32->ss_children = lgrp_snap32->ss_parents + (snap_nlgrpsmax * bitmask_size); lgrp_snap32->ss_rsets = lgrp_snap32->ss_children + (snap_nlgrpsmax * bitmask_size); lgrp_snap32->ss_latencies = lgrp_snap32->ss_rsets + (LGRP_RSRC_COUNT * snap_nlgrpsmax * bitmask_size); /* * Fill in lgrpset now because caller may have change psets */ kpreempt_disable(); for (i = 0; i < snap_nlgrpsmax; i++) { if (klgrpset_ismember(curthread->t_cpupart->cp_lgrpset, i)) { BT_SET32(lgrp_set32, i); } } kpreempt_enable(); /* * Fill in 32-bit copy of lgroup info and fix up pointers * to point into user's buffer instead of kernel's */ cpu_index = 0; lgrp_info = lgrp_snap->ss_info; for (i = 0; i < snap_nlgrpsmax; i++) { uint_t *children; uint_t *lgrp_rset; uint_t *parents; ulong_t *snap_rset; /* * Skip non-existent lgroups */ if (lgrp_info[i].info_lgrpid == LGRP_NONE) { bzero(&lgrp_info32[i], sizeof (lgrp_info32[i])); lgrp_info32[i].info_lgrpid = LGRP_NONE; continue; } /* * Fill in parents, children, lgroup resource set, and * latencies from snapshot */ parents = (uint_t *)((uintptr_t)lgrp_parents32 + i * bitmask_size); children = (uint_t *)((uintptr_t)lgrp_children32 + i * bitmask_size); snap_rset = (ulong_t *)((uintptr_t)lgrp_snap->ss_rsets + (i * LGRP_RSRC_COUNT * BT_SIZEOFMAP(snap_nlgrpsmax))); lgrp_rset = (uint_t *)((uintptr_t)lgrp_rsets32 + (i * LGRP_RSRC_COUNT * bitmask_size)); lgrp_lats32_kernel[i] = (int *)((uintptr_t)lgrp_lats32 + snap_nlgrpsmax * sizeof (caddr32_t) + i * snap_nlgrpsmax * sizeof (int)); for (j = 0; j < snap_nlgrpsmax; j++) { int k; uint_t *rset; if (BT_TEST(&lgrp_snap->ss_parents[i], j)) BT_SET32(parents, j); if (BT_TEST(&lgrp_snap->ss_children[i], j)) BT_SET32(children, j); for (k = 0; k < LGRP_RSRC_COUNT; k++) { rset = (uint_t *)((uintptr_t)lgrp_rset + k * bitmask_size); if (BT_TEST(&snap_rset[k], j)) BT_SET32(rset, j); } lgrp_lats32_kernel[i][j] = lgrp_snap->ss_latencies[i][j]; } /* * Fix up pointer to latency buffer */ lgrp_lats32[i] = lgrp_snap32->ss_latencies + snap_nlgrpsmax * sizeof (caddr32_t) + i * snap_nlgrpsmax * sizeof (int); /* * Fix up pointers for parents, children, and resources */ lgrp_info32[i].info_parents = lgrp_snap32->ss_parents + (i * bitmask_size); lgrp_info32[i].info_children = lgrp_snap32->ss_children + (i * bitmask_size); lgrp_info32[i].info_rset = lgrp_snap32->ss_rsets + (i * LGRP_RSRC_COUNT * bitmask_size); /* * Fill in memory and CPU info * Only fill in memory for lgroups directly containing memory */ snap_rset = &lgrp_info[i].info_rset[LGRP_RSRC_MEM * BT_BITOUL(snap_nlgrpsmax)]; if (BT_TEST(snap_rset, i)) { lgrp_info32[i].info_mem_free = lgrp_mem_size(i, LGRP_MEM_SIZE_FREE); lgrp_info32[i].info_mem_install = lgrp_info[i].info_mem_install; } lgrp_info32[i].info_ncpus = lgrp_info[i].info_ncpus; lgrp_info32[i].info_lgrpid = lgrp_info[i].info_lgrpid; lgrp_info32[i].info_latency = lgrp_info[i].info_latency; if (lgrp_info32[i].info_ncpus == 0) { lgrp_info32[i].info_cpuids = 0; continue; } /* * Fix up pointer for CPU IDs */ lgrp_info32[i].info_cpuids = lgrp_snap32->ss_cpuids + (cpu_index * sizeof (processorid_t)); cpu_index += lgrp_info32[i].info_ncpus; } ASSERT(cpu_index == snap_ncpus); /* * Copy lgroup CPU IDs into 32-bit snapshot * before copying it out into user's buffer */ bcopy(lgrp_snap->ss_cpuids, lgrp_cpuids32, cpuids_size); /* * Copy 32-bit lgroup snapshot into user's buffer all at once */ if (copyout(lgrp_snap32, (void *)(uintptr_t)buf, snap_size) != 0) { kmem_free(lgrp_snap32, snap_size); kmem_free(lgrp_lats32_kernel, snap_nlgrpsmax * sizeof (int *)); return (set_errno(EFAULT)); } kmem_free(lgrp_snap32, snap_size); kmem_free(lgrp_lats32_kernel, snap_nlgrpsmax * sizeof (int *)); return (snap_size); } #endif /* _SYSCALL32_IMPL */ int lgrpsys(int subcode, long ia, void *ap) { size_t bufsize; int latency; switch (subcode) { case LGRP_SYS_AFFINITY_GET: return (lgrp_affinity_get((lgrp_affinity_args_t *)ap)); case LGRP_SYS_AFFINITY_SET: return (lgrp_affinity_set((lgrp_affinity_args_t *)ap)); case LGRP_SYS_GENERATION: return (lgrp_generation(ia)); case LGRP_SYS_HOME: return (lgrp_home_get((idtype_t)ia, (id_t)(uintptr_t)ap)); case LGRP_SYS_LATENCY: mutex_enter(&cpu_lock); latency = lgrp_latency(ia, (lgrp_id_t)(uintptr_t)ap); mutex_exit(&cpu_lock); return (latency); case LGRP_SYS_MEMINFO: return (meminfo(ia, (struct meminfo *)ap)); case LGRP_SYS_VERSION: return (lgrp_version(ia)); case LGRP_SYS_SNAPSHOT: mutex_enter(&lgrp_snap_lock); bufsize = lgrp_snapshot(); if (ap && ia > 0) { if (get_udatamodel() == DATAMODEL_NATIVE) bufsize = lgrp_snapshot_copy(ap, ia); #ifdef _SYSCALL32_IMPL else bufsize = lgrp_snapshot_copy32( (caddr32_t)(uintptr_t)ap, ia); #endif /* _SYSCALL32_IMPL */ } mutex_exit(&lgrp_snap_lock); return (bufsize); default: break; } return (set_errno(EINVAL)); }