17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
5a359d6b1Strevtom * Common Development and Distribution License (the "License").
6a359d6b1Strevtom * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
217c478bd9Sstevel@tonic-gate /*
22c6224917SMichael Corcoran * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
237c478bd9Sstevel@tonic-gate * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate */
257c478bd9Sstevel@tonic-gate
267c478bd9Sstevel@tonic-gate /*
277c478bd9Sstevel@tonic-gate * The following routines implement the hat layer's
287c478bd9Sstevel@tonic-gate * recording of the referenced and modified bits.
297c478bd9Sstevel@tonic-gate */
307c478bd9Sstevel@tonic-gate
317c478bd9Sstevel@tonic-gate #include <sys/types.h>
327c478bd9Sstevel@tonic-gate #include <sys/param.h>
337c478bd9Sstevel@tonic-gate #include <sys/systm.h>
347c478bd9Sstevel@tonic-gate #include <sys/debug.h>
357c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
367c478bd9Sstevel@tonic-gate
377c478bd9Sstevel@tonic-gate /*
387c478bd9Sstevel@tonic-gate * Note, usage of cmn_err requires you not hold any hat layer locks.
397c478bd9Sstevel@tonic-gate */
407c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
417c478bd9Sstevel@tonic-gate
427c478bd9Sstevel@tonic-gate #include <vm/as.h>
437c478bd9Sstevel@tonic-gate #include <vm/hat.h>
447c478bd9Sstevel@tonic-gate
457c478bd9Sstevel@tonic-gate kmutex_t hat_statlock; /* protects all hat statistics data */
467c478bd9Sstevel@tonic-gate struct hrmstat *hrm_memlist; /* tracks memory alloced for hrm_blist blocks */
477c478bd9Sstevel@tonic-gate struct hrmstat **hrm_hashtab; /* hash table for finding blocks quickly */
487c478bd9Sstevel@tonic-gate struct hrmstat *hrm_blist;
497c478bd9Sstevel@tonic-gate int hrm_blist_incr = HRM_BLIST_INCR;
507c478bd9Sstevel@tonic-gate int hrm_blist_lowater = HRM_BLIST_INCR/2;
517c478bd9Sstevel@tonic-gate int hrm_blist_num = 0;
527c478bd9Sstevel@tonic-gate int hrm_blist_total = 0;
537c478bd9Sstevel@tonic-gate int hrm_mlockinited = 0;
547c478bd9Sstevel@tonic-gate int hrm_allocfailmsg = 0; /* print a message when allocations fail */
557c478bd9Sstevel@tonic-gate int hrm_allocfail = 0;
567c478bd9Sstevel@tonic-gate
577c478bd9Sstevel@tonic-gate static struct hrmstat *hrm_balloc(void);
587c478bd9Sstevel@tonic-gate static void hrm_link(struct hrmstat *);
597c478bd9Sstevel@tonic-gate static void hrm_setbits(struct hrmstat *, caddr_t, uint_t);
607c478bd9Sstevel@tonic-gate static void hrm_hashout(struct hrmstat *);
617c478bd9Sstevel@tonic-gate static void hrm_getblk(int);
627c478bd9Sstevel@tonic-gate
637c478bd9Sstevel@tonic-gate #define hrm_hash(as, addr) \
647c478bd9Sstevel@tonic-gate (HRM_HASHMASK & \
657c478bd9Sstevel@tonic-gate (((uintptr_t)(addr) >> HRM_BASESHIFT) ^ ((uintptr_t)(as) >> 2)))
667c478bd9Sstevel@tonic-gate
677c478bd9Sstevel@tonic-gate #define hrm_match(hrm, as, addr) \
687c478bd9Sstevel@tonic-gate (((hrm)->hrm_as == (as) && \
697c478bd9Sstevel@tonic-gate ((hrm)->hrm_base == ((uintptr_t)(addr) & HRM_BASEMASK))) ? 1 : 0)
707c478bd9Sstevel@tonic-gate
717c478bd9Sstevel@tonic-gate /*
72c6224917SMichael Corcoran * Called when an address space maps in more pages while stats are being
73c6224917SMichael Corcoran * collected.
747c478bd9Sstevel@tonic-gate */
757c478bd9Sstevel@tonic-gate /* ARGSUSED */
767c478bd9Sstevel@tonic-gate void
hat_resvstat(size_t chunk,struct as * as,caddr_t addr)777c478bd9Sstevel@tonic-gate hat_resvstat(size_t chunk, struct as *as, caddr_t addr)
787c478bd9Sstevel@tonic-gate {
797c478bd9Sstevel@tonic-gate }
807c478bd9Sstevel@tonic-gate
817c478bd9Sstevel@tonic-gate /*
827c478bd9Sstevel@tonic-gate * Start the statistics gathering for an address space.
837c478bd9Sstevel@tonic-gate * Return -1 if we can't do it, otherwise return an opaque
847c478bd9Sstevel@tonic-gate * identifier to be used when querying for the gathered statistics.
857c478bd9Sstevel@tonic-gate * The identifier is an unused bit in a_vbits.
867c478bd9Sstevel@tonic-gate * Bit 0 is reserved for swsmon.
877c478bd9Sstevel@tonic-gate */
887c478bd9Sstevel@tonic-gate int
hat_startstat(struct as * as)897c478bd9Sstevel@tonic-gate hat_startstat(struct as *as)
907c478bd9Sstevel@tonic-gate {
917c478bd9Sstevel@tonic-gate uint_t nbits; /* number of bits */
927c478bd9Sstevel@tonic-gate uint_t bn; /* bit number */
937c478bd9Sstevel@tonic-gate uint_t id; /* new vbit, identifier */
947c478bd9Sstevel@tonic-gate uint_t vbits; /* used vbits of address space */
957c478bd9Sstevel@tonic-gate size_t chunk; /* mapped size for stats */
967c478bd9Sstevel@tonic-gate
977c478bd9Sstevel@tonic-gate /*
987c478bd9Sstevel@tonic-gate * If the refmod saving memory allocator runs out, print
997c478bd9Sstevel@tonic-gate * a warning message about how to fix it, see comment at
1007c478bd9Sstevel@tonic-gate * the beginning of hat_setstat.
1017c478bd9Sstevel@tonic-gate */
1027c478bd9Sstevel@tonic-gate if (hrm_allocfailmsg) {
1037c478bd9Sstevel@tonic-gate cmn_err(CE_WARN,
1047c478bd9Sstevel@tonic-gate "hrm_balloc failures occured, increase hrm_blist_incr");
1057c478bd9Sstevel@tonic-gate hrm_allocfailmsg = 0;
1067c478bd9Sstevel@tonic-gate }
1077c478bd9Sstevel@tonic-gate
1087c478bd9Sstevel@tonic-gate /*
1097c478bd9Sstevel@tonic-gate * Verify that a buffer of statistics blocks exists
1107c478bd9Sstevel@tonic-gate * and allocate more, if needed.
1117c478bd9Sstevel@tonic-gate */
1127c478bd9Sstevel@tonic-gate
1137c478bd9Sstevel@tonic-gate chunk = hat_get_mapped_size(as->a_hat);
1147c478bd9Sstevel@tonic-gate chunk = (btop(chunk)/HRM_PAGES);
1157c478bd9Sstevel@tonic-gate if (chunk < HRM_BLIST_INCR)
1167c478bd9Sstevel@tonic-gate chunk = 0;
1177c478bd9Sstevel@tonic-gate
1187c478bd9Sstevel@tonic-gate hrm_getblk((int)chunk);
1197c478bd9Sstevel@tonic-gate
1207c478bd9Sstevel@tonic-gate /*
1217c478bd9Sstevel@tonic-gate * Find a unused id in the given address space.
1227c478bd9Sstevel@tonic-gate */
1237c478bd9Sstevel@tonic-gate hat_enter(as->a_hat);
1247c478bd9Sstevel@tonic-gate vbits = as->a_vbits;
1257c478bd9Sstevel@tonic-gate nbits = sizeof (as->a_vbits) * NBBY;
1267c478bd9Sstevel@tonic-gate for (bn = 1, id = 2; bn < (nbits - 1); bn++, id <<= 1)
1277c478bd9Sstevel@tonic-gate if ((id & vbits) == 0)
1287c478bd9Sstevel@tonic-gate break;
1297c478bd9Sstevel@tonic-gate if (bn >= (nbits - 1)) {
1307c478bd9Sstevel@tonic-gate hat_exit(as->a_hat);
1317c478bd9Sstevel@tonic-gate return (-1);
1327c478bd9Sstevel@tonic-gate }
1337c478bd9Sstevel@tonic-gate as->a_vbits |= id;
1347c478bd9Sstevel@tonic-gate hat_exit(as->a_hat);
1357c478bd9Sstevel@tonic-gate (void) hat_stats_enable(as->a_hat);
1367c478bd9Sstevel@tonic-gate return (id);
1377c478bd9Sstevel@tonic-gate }
1387c478bd9Sstevel@tonic-gate
1397c478bd9Sstevel@tonic-gate /*
1407c478bd9Sstevel@tonic-gate * Record referenced and modified information for an address space.
1417c478bd9Sstevel@tonic-gate * Rmbits is a word containing the referenced bit in bit position 1
1427c478bd9Sstevel@tonic-gate * and the modified bit in bit position 0.
1437c478bd9Sstevel@tonic-gate *
1447c478bd9Sstevel@tonic-gate * For current informational uses, one can rerun any program using
1457c478bd9Sstevel@tonic-gate * this facility after modifying the hrm_blist_incr to be a larger
1467c478bd9Sstevel@tonic-gate * amount so that a larger buffer of blocks will be maintained.
1477c478bd9Sstevel@tonic-gate */
1487c478bd9Sstevel@tonic-gate void
hat_setstat(struct as * as,caddr_t addr,size_t len,uint_t rmbits)1497c478bd9Sstevel@tonic-gate hat_setstat(struct as *as, caddr_t addr, size_t len, uint_t rmbits)
1507c478bd9Sstevel@tonic-gate {
1517c478bd9Sstevel@tonic-gate struct hrmstat *hrm;
1527c478bd9Sstevel@tonic-gate uint_t vbits, newbits, nb;
1537c478bd9Sstevel@tonic-gate int h;
1547c478bd9Sstevel@tonic-gate
1557c478bd9Sstevel@tonic-gate ASSERT(len == PAGESIZE);
1567c478bd9Sstevel@tonic-gate ASSERT((rmbits & ~(P_MOD|P_REF)) == 0);
1577c478bd9Sstevel@tonic-gate
1587c478bd9Sstevel@tonic-gate if (rmbits == 0)
1597c478bd9Sstevel@tonic-gate return;
1607c478bd9Sstevel@tonic-gate
1617c478bd9Sstevel@tonic-gate mutex_enter(&hat_statlock);
1627c478bd9Sstevel@tonic-gate
1637c478bd9Sstevel@tonic-gate /*
1647c478bd9Sstevel@tonic-gate * Search the hash list for the as and addr we are looking for
1657c478bd9Sstevel@tonic-gate * and set the ref and mod bits in every block that matches.
1667c478bd9Sstevel@tonic-gate */
1677c478bd9Sstevel@tonic-gate vbits = 0;
1687c478bd9Sstevel@tonic-gate h = hrm_hash(as, addr);
1697c478bd9Sstevel@tonic-gate for (hrm = hrm_hashtab[h]; hrm; hrm = hrm->hrm_hnext) {
1707c478bd9Sstevel@tonic-gate if (hrm_match(hrm, as, addr)) {
1717c478bd9Sstevel@tonic-gate hrm_setbits(hrm, addr, rmbits);
1727c478bd9Sstevel@tonic-gate vbits |= hrm->hrm_id;
1737c478bd9Sstevel@tonic-gate }
1747c478bd9Sstevel@tonic-gate }
1757c478bd9Sstevel@tonic-gate
1767c478bd9Sstevel@tonic-gate /*
1777c478bd9Sstevel@tonic-gate * If we didn't find a block for all of the enabled
1787c478bd9Sstevel@tonic-gate * vpages bits, then allocate and initialize a block
1797c478bd9Sstevel@tonic-gate * for each bit that was not found.
1807c478bd9Sstevel@tonic-gate */
1817c478bd9Sstevel@tonic-gate if (vbits != as->a_vbits) {
1829d9461f9Strevtom newbits = (vbits ^ as->a_vbits) & as->a_vbits;
183*c6f039c7SToomas Soome nb = 0;
1847c478bd9Sstevel@tonic-gate while (newbits) {
1857c478bd9Sstevel@tonic-gate if (ffs(newbits))
1867c478bd9Sstevel@tonic-gate nb = 1 << (ffs(newbits)-1);
1877c478bd9Sstevel@tonic-gate hrm = (struct hrmstat *)hrm_balloc();
1887c478bd9Sstevel@tonic-gate if (hrm == NULL) {
1897c478bd9Sstevel@tonic-gate hrm_allocfailmsg = 1;
1907c478bd9Sstevel@tonic-gate hrm_allocfail++;
1917c478bd9Sstevel@tonic-gate mutex_exit(&hat_statlock);
1927c478bd9Sstevel@tonic-gate return;
1937c478bd9Sstevel@tonic-gate }
1947c478bd9Sstevel@tonic-gate hrm->hrm_as = as;
1957c478bd9Sstevel@tonic-gate hrm->hrm_base = (uintptr_t)addr & HRM_BASEMASK;
1967c478bd9Sstevel@tonic-gate hrm->hrm_id = nb;
1977c478bd9Sstevel@tonic-gate hrm_link(hrm);
1987c478bd9Sstevel@tonic-gate hrm_setbits(hrm, addr, rmbits);
1997c478bd9Sstevel@tonic-gate newbits &= ~nb;
2007c478bd9Sstevel@tonic-gate }
2017c478bd9Sstevel@tonic-gate }
2027c478bd9Sstevel@tonic-gate mutex_exit(&hat_statlock);
2037c478bd9Sstevel@tonic-gate }
2047c478bd9Sstevel@tonic-gate
2057c478bd9Sstevel@tonic-gate /*
2067c478bd9Sstevel@tonic-gate * Free the resources used to maintain the referenced and modified
2077c478bd9Sstevel@tonic-gate * statistics for the virtual page view of an address space
2087c478bd9Sstevel@tonic-gate * identified by id.
2097c478bd9Sstevel@tonic-gate */
2107c478bd9Sstevel@tonic-gate void
hat_freestat(struct as * as,int id)2117c478bd9Sstevel@tonic-gate hat_freestat(struct as *as, int id)
2127c478bd9Sstevel@tonic-gate {
2139d9461f9Strevtom struct hrmstat *hrm;
2149d9461f9Strevtom struct hrmstat *prev_ahrm;
2159d9461f9Strevtom struct hrmstat *hrm_tmplist;
2169d9461f9Strevtom struct hrmstat *hrm_next;
2177c478bd9Sstevel@tonic-gate
2187c478bd9Sstevel@tonic-gate hat_stats_disable(as->a_hat); /* tell the hat layer to stop */
2197c478bd9Sstevel@tonic-gate hat_enter(as->a_hat);
2207c478bd9Sstevel@tonic-gate if (id == 0)
2217c478bd9Sstevel@tonic-gate as->a_vbits = 0;
2227c478bd9Sstevel@tonic-gate else
2237c478bd9Sstevel@tonic-gate as->a_vbits &= ~id;
2247c478bd9Sstevel@tonic-gate
2257c478bd9Sstevel@tonic-gate if ((hrm = as->a_hrm) == NULL) {
2267c478bd9Sstevel@tonic-gate hat_exit(as->a_hat);
2277c478bd9Sstevel@tonic-gate return;
2287c478bd9Sstevel@tonic-gate }
2297c478bd9Sstevel@tonic-gate hat_exit(as->a_hat);
2307c478bd9Sstevel@tonic-gate
2317c478bd9Sstevel@tonic-gate mutex_enter(&hat_statlock);
2329d9461f9Strevtom
2337c478bd9Sstevel@tonic-gate for (prev_ahrm = NULL; hrm; hrm = hrm->hrm_anext) {
2347e12ceb3SToomas Soome if ((id == hrm->hrm_id) || (id == 0)) {
2357c478bd9Sstevel@tonic-gate
2367c478bd9Sstevel@tonic-gate hrm_hashout(hrm);
2377c478bd9Sstevel@tonic-gate hrm->hrm_hnext = hrm_blist;
2387c478bd9Sstevel@tonic-gate hrm_blist = hrm;
2397c478bd9Sstevel@tonic-gate hrm_blist_num++;
2407c478bd9Sstevel@tonic-gate
2417c478bd9Sstevel@tonic-gate if (prev_ahrm == NULL)
2427c478bd9Sstevel@tonic-gate as->a_hrm = hrm->hrm_anext;
2437c478bd9Sstevel@tonic-gate else
2447c478bd9Sstevel@tonic-gate prev_ahrm->hrm_anext = hrm->hrm_anext;
2457c478bd9Sstevel@tonic-gate
2467c478bd9Sstevel@tonic-gate } else
2477c478bd9Sstevel@tonic-gate prev_ahrm = hrm;
2487c478bd9Sstevel@tonic-gate }
2497c478bd9Sstevel@tonic-gate
2507c478bd9Sstevel@tonic-gate /*
2517c478bd9Sstevel@tonic-gate * If all statistics blocks are free,
2527c478bd9Sstevel@tonic-gate * return the memory to the system.
2537c478bd9Sstevel@tonic-gate */
2547c478bd9Sstevel@tonic-gate if (hrm_blist_num == hrm_blist_total) {
2557c478bd9Sstevel@tonic-gate /* zero the block list since we are giving back its memory */
2567c478bd9Sstevel@tonic-gate hrm_blist = NULL;
2577c478bd9Sstevel@tonic-gate hrm_blist_num = 0;
2587c478bd9Sstevel@tonic-gate hrm_blist_total = 0;
2599d9461f9Strevtom hrm_tmplist = hrm_memlist;
260a359d6b1Strevtom hrm_memlist = NULL;
261a359d6b1Strevtom } else {
2629d9461f9Strevtom hrm_tmplist = NULL;
2637c478bd9Sstevel@tonic-gate }
264a359d6b1Strevtom
2657c478bd9Sstevel@tonic-gate mutex_exit(&hat_statlock);
266a359d6b1Strevtom
2677c478bd9Sstevel@tonic-gate /*
2689d9461f9Strevtom * If there are any hrmstat structures to be freed, this must only
2699d9461f9Strevtom * be done after we've released hat_statlock.
2707c478bd9Sstevel@tonic-gate */
2719d9461f9Strevtom while (hrm_tmplist != NULL) {
2729d9461f9Strevtom hrm_next = hrm_tmplist->hrm_hnext;
2739d9461f9Strevtom kmem_free(hrm_tmplist, hrm_tmplist->hrm_base);
2749d9461f9Strevtom hrm_tmplist = hrm_next;
2759d9461f9Strevtom }
2767c478bd9Sstevel@tonic-gate }
2777c478bd9Sstevel@tonic-gate
2787c478bd9Sstevel@tonic-gate /*
2797c478bd9Sstevel@tonic-gate * Grab memory for statistics gathering of the hat layer.
2807c478bd9Sstevel@tonic-gate */
2817c478bd9Sstevel@tonic-gate static void
hrm_getblk(int chunk)2827c478bd9Sstevel@tonic-gate hrm_getblk(int chunk)
2837c478bd9Sstevel@tonic-gate {
2847c478bd9Sstevel@tonic-gate struct hrmstat *hrm, *l;
2857c478bd9Sstevel@tonic-gate int i;
2867c478bd9Sstevel@tonic-gate int hrm_incr;
2877c478bd9Sstevel@tonic-gate
2887c478bd9Sstevel@tonic-gate mutex_enter(&hat_statlock);
289c6224917SMichael Corcoran /*
290c6224917SMichael Corcoran * XXX The whole private freelist management here really should be
291c6224917SMichael Corcoran * overhauled.
292c6224917SMichael Corcoran *
293c6224917SMichael Corcoran * The freelist should have some knowledge of how much memory is
294c6224917SMichael Corcoran * needed by a process and thus when hat_resvstat get's called, we can
295c6224917SMichael Corcoran * increment the freelist needs for that process within this subsystem.
296c6224917SMichael Corcoran * Thus there will be reservations for all processes which are being
297c6224917SMichael Corcoran * watched which should be accurate, and consume less memory overall.
298c6224917SMichael Corcoran *
299c6224917SMichael Corcoran * For now, just make sure there's enough entries on the freelist to
300c6224917SMichael Corcoran * handle the current chunk.
301c6224917SMichael Corcoran */
3027c478bd9Sstevel@tonic-gate if ((hrm_blist == NULL) ||
3037c478bd9Sstevel@tonic-gate (hrm_blist_num <= hrm_blist_lowater) ||
304c6224917SMichael Corcoran (chunk && (hrm_blist_num < chunk + hrm_blist_incr))) {
3057c478bd9Sstevel@tonic-gate mutex_exit(&hat_statlock);
3067c478bd9Sstevel@tonic-gate
307c6224917SMichael Corcoran hrm_incr = chunk + hrm_blist_incr;
3087c478bd9Sstevel@tonic-gate hrm = kmem_zalloc(sizeof (struct hrmstat) * hrm_incr, KM_SLEEP);
3097c478bd9Sstevel@tonic-gate hrm->hrm_base = sizeof (struct hrmstat) * hrm_incr;
3107c478bd9Sstevel@tonic-gate
3117c478bd9Sstevel@tonic-gate /*
3127c478bd9Sstevel@tonic-gate * thread the allocated blocks onto a freelist
3137c478bd9Sstevel@tonic-gate * using the first block to hold information for
3147c478bd9Sstevel@tonic-gate * freeing them all later
3157c478bd9Sstevel@tonic-gate */
3167c478bd9Sstevel@tonic-gate mutex_enter(&hat_statlock);
3177c478bd9Sstevel@tonic-gate hrm->hrm_hnext = hrm_memlist;
3187c478bd9Sstevel@tonic-gate hrm_memlist = hrm;
3197c478bd9Sstevel@tonic-gate
3207c478bd9Sstevel@tonic-gate hrm_blist_total += (hrm_incr - 1);
3217c478bd9Sstevel@tonic-gate for (i = 1; i < hrm_incr; i++) {
3227c478bd9Sstevel@tonic-gate l = &hrm[i];
3237c478bd9Sstevel@tonic-gate l->hrm_hnext = hrm_blist;
3247c478bd9Sstevel@tonic-gate hrm_blist = l;
3257c478bd9Sstevel@tonic-gate hrm_blist_num++;
3267c478bd9Sstevel@tonic-gate }
3277c478bd9Sstevel@tonic-gate }
3287c478bd9Sstevel@tonic-gate mutex_exit(&hat_statlock);
3297c478bd9Sstevel@tonic-gate }
3307c478bd9Sstevel@tonic-gate
3317c478bd9Sstevel@tonic-gate static void
hrm_hashin(struct hrmstat * hrm)3327c478bd9Sstevel@tonic-gate hrm_hashin(struct hrmstat *hrm)
3337c478bd9Sstevel@tonic-gate {
3347c478bd9Sstevel@tonic-gate int h;
3357c478bd9Sstevel@tonic-gate
3367c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&hat_statlock));
3377c478bd9Sstevel@tonic-gate h = hrm_hash(hrm->hrm_as, hrm->hrm_base);
3387c478bd9Sstevel@tonic-gate
3397c478bd9Sstevel@tonic-gate hrm->hrm_hnext = hrm_hashtab[h];
3407c478bd9Sstevel@tonic-gate hrm_hashtab[h] = hrm;
3417c478bd9Sstevel@tonic-gate }
3427c478bd9Sstevel@tonic-gate
3437c478bd9Sstevel@tonic-gate static void
hrm_hashout(struct hrmstat * hrm)3447c478bd9Sstevel@tonic-gate hrm_hashout(struct hrmstat *hrm)
3457c478bd9Sstevel@tonic-gate {
3467c478bd9Sstevel@tonic-gate struct hrmstat *list, **prev_hrm;
3477c478bd9Sstevel@tonic-gate int h;
3487c478bd9Sstevel@tonic-gate
3497c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&hat_statlock));
3507c478bd9Sstevel@tonic-gate h = hrm_hash(hrm->hrm_as, hrm->hrm_base);
3517c478bd9Sstevel@tonic-gate list = hrm_hashtab[h];
3527c478bd9Sstevel@tonic-gate prev_hrm = &hrm_hashtab[h];
3537c478bd9Sstevel@tonic-gate
3547c478bd9Sstevel@tonic-gate while (list) {
3557c478bd9Sstevel@tonic-gate if (list == hrm) {
3567c478bd9Sstevel@tonic-gate *prev_hrm = list->hrm_hnext;
3577c478bd9Sstevel@tonic-gate return;
3587c478bd9Sstevel@tonic-gate }
3597c478bd9Sstevel@tonic-gate prev_hrm = &list->hrm_hnext;
3607c478bd9Sstevel@tonic-gate list = list->hrm_hnext;
3617c478bd9Sstevel@tonic-gate }
3627c478bd9Sstevel@tonic-gate }
3637c478bd9Sstevel@tonic-gate
3647c478bd9Sstevel@tonic-gate
3657c478bd9Sstevel@tonic-gate /*
3667c478bd9Sstevel@tonic-gate * Link a statistic block into an address space and also put it
3677c478bd9Sstevel@tonic-gate * on the hash list for future references.
3687c478bd9Sstevel@tonic-gate */
3697c478bd9Sstevel@tonic-gate static void
hrm_link(struct hrmstat * hrm)3707c478bd9Sstevel@tonic-gate hrm_link(struct hrmstat *hrm)
3717c478bd9Sstevel@tonic-gate {
3727c478bd9Sstevel@tonic-gate struct as *as = hrm->hrm_as;
3737c478bd9Sstevel@tonic-gate
3747c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&hat_statlock));
3757c478bd9Sstevel@tonic-gate hrm->hrm_anext = as->a_hrm;
3767c478bd9Sstevel@tonic-gate as->a_hrm = hrm;
3777c478bd9Sstevel@tonic-gate hrm_hashin(hrm);
3787c478bd9Sstevel@tonic-gate }
3797c478bd9Sstevel@tonic-gate
3807c478bd9Sstevel@tonic-gate /*
3817c478bd9Sstevel@tonic-gate * Allocate a block for statistics keeping.
3827c478bd9Sstevel@tonic-gate * Returns NULL if blocks are unavailable.
3837c478bd9Sstevel@tonic-gate */
3847c478bd9Sstevel@tonic-gate static struct hrmstat *
hrm_balloc(void)3857c478bd9Sstevel@tonic-gate hrm_balloc(void)
3867c478bd9Sstevel@tonic-gate {
3877c478bd9Sstevel@tonic-gate struct hrmstat *hrm;
3887c478bd9Sstevel@tonic-gate
3897c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&hat_statlock));
3907c478bd9Sstevel@tonic-gate
3917c478bd9Sstevel@tonic-gate hrm = hrm_blist;
3927c478bd9Sstevel@tonic-gate if (hrm != NULL) {
3937c478bd9Sstevel@tonic-gate hrm_blist = hrm->hrm_hnext;
3947c478bd9Sstevel@tonic-gate hrm_blist_num--;
3957c478bd9Sstevel@tonic-gate hrm->hrm_hnext = NULL;
3967c478bd9Sstevel@tonic-gate }
3977c478bd9Sstevel@tonic-gate return (hrm);
3987c478bd9Sstevel@tonic-gate }
3997c478bd9Sstevel@tonic-gate
4007c478bd9Sstevel@tonic-gate /*
4017c478bd9Sstevel@tonic-gate * Set the ref and mod bits for addr within statistics block hrm.
4027c478bd9Sstevel@tonic-gate */
4037c478bd9Sstevel@tonic-gate static void
hrm_setbits(struct hrmstat * hrm,caddr_t addr,uint_t bits)4047c478bd9Sstevel@tonic-gate hrm_setbits(struct hrmstat *hrm, caddr_t addr, uint_t bits)
4057c478bd9Sstevel@tonic-gate {
4067c478bd9Sstevel@tonic-gate uint_t po, bo, spb;
4077c478bd9Sstevel@tonic-gate uint_t nbits;
4087c478bd9Sstevel@tonic-gate
4097c478bd9Sstevel@tonic-gate po = ((uintptr_t)addr & HRM_BASEOFFSET) >> MMU_PAGESHIFT; /* pg off */
4107c478bd9Sstevel@tonic-gate bo = po / (NBBY / 2); /* which byte in bit array */
4117c478bd9Sstevel@tonic-gate spb = (3 - (po & 3)) * 2; /* shift position within byte */
4127c478bd9Sstevel@tonic-gate nbits = bits << spb; /* bit mask */
4137c478bd9Sstevel@tonic-gate hrm->hrm_bits[bo] |= nbits;
4147c478bd9Sstevel@tonic-gate }
4157c478bd9Sstevel@tonic-gate
4167c478bd9Sstevel@tonic-gate /*
4177c478bd9Sstevel@tonic-gate * Return collected statistics about an address space.
4187c478bd9Sstevel@tonic-gate * If clearflag is set, atomically read and zero the bits.
4197c478bd9Sstevel@tonic-gate *
4207c478bd9Sstevel@tonic-gate * Fill in the data array supplied with the referenced and
4217c478bd9Sstevel@tonic-gate * modified bits collected for address range [addr ... addr + len]
4227c478bd9Sstevel@tonic-gate * in address space, as, uniquely identified by id.
4237c478bd9Sstevel@tonic-gate * The destination is a byte array. We fill in three bits per byte:
4247c478bd9Sstevel@tonic-gate * referenced, modified, and hwmapped bits.
4257c478bd9Sstevel@tonic-gate * Kernel only interface, can't fault on destination data array.
4267c478bd9Sstevel@tonic-gate *
4277c478bd9Sstevel@tonic-gate */
4287c478bd9Sstevel@tonic-gate void
hat_getstat(struct as * as,caddr_t addr,size_t len,uint_t id,caddr_t datap,int clearflag)4297c478bd9Sstevel@tonic-gate hat_getstat(struct as *as, caddr_t addr, size_t len, uint_t id,
4307c478bd9Sstevel@tonic-gate caddr_t datap, int clearflag)
4317c478bd9Sstevel@tonic-gate {
4327c478bd9Sstevel@tonic-gate size_t np; /* number of pages */
4337c478bd9Sstevel@tonic-gate caddr_t a;
4347c478bd9Sstevel@tonic-gate char *dp;
4357c478bd9Sstevel@tonic-gate
4367c478bd9Sstevel@tonic-gate np = btop(len);
4377c478bd9Sstevel@tonic-gate bzero(datap, np);
4387c478bd9Sstevel@tonic-gate
439c6224917SMichael Corcoran /* allocate enough statistics blocks to cover the len passed in */
440c6224917SMichael Corcoran hrm_getblk(np / HRM_PAGES);
441c6224917SMichael Corcoran
4427c478bd9Sstevel@tonic-gate hat_sync(as->a_hat, addr, len, clearflag);
4437c478bd9Sstevel@tonic-gate
4447c478bd9Sstevel@tonic-gate /* allocate more statistics blocks if needed */
4457c478bd9Sstevel@tonic-gate hrm_getblk(0);
4467c478bd9Sstevel@tonic-gate
4477c478bd9Sstevel@tonic-gate mutex_enter(&hat_statlock);
4487c478bd9Sstevel@tonic-gate if (hrm_hashtab == NULL) {
4497c478bd9Sstevel@tonic-gate /* can happen when victim process exits */
4507c478bd9Sstevel@tonic-gate mutex_exit(&hat_statlock);
4517c478bd9Sstevel@tonic-gate return;
4527c478bd9Sstevel@tonic-gate }
4537c478bd9Sstevel@tonic-gate dp = datap;
4547c478bd9Sstevel@tonic-gate a = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
4557c478bd9Sstevel@tonic-gate while (a < addr + len) {
4567c478bd9Sstevel@tonic-gate struct hrmstat *hrm;
4577c478bd9Sstevel@tonic-gate size_t n; /* number of pages, temp */
4587c478bd9Sstevel@tonic-gate int h; /* hash index */
4597c478bd9Sstevel@tonic-gate uint_t po;
4607c478bd9Sstevel@tonic-gate
4617c478bd9Sstevel@tonic-gate h = hrm_hash(as, a);
4627c478bd9Sstevel@tonic-gate n = (HRM_PAGES -
463c6224917SMichael Corcoran (((uintptr_t)a & HRM_PAGEMASK) >> MMU_PAGESHIFT));
4647c478bd9Sstevel@tonic-gate if (n > np)
4657c478bd9Sstevel@tonic-gate n = np;
4667c478bd9Sstevel@tonic-gate po = ((uintptr_t)a & HRM_BASEOFFSET) >> MMU_PAGESHIFT;
4677c478bd9Sstevel@tonic-gate
4687c478bd9Sstevel@tonic-gate for (hrm = hrm_hashtab[h]; hrm; hrm = hrm->hrm_hnext) {
4697c478bd9Sstevel@tonic-gate if (hrm->hrm_as == as &&
4707c478bd9Sstevel@tonic-gate hrm->hrm_base == ((uintptr_t)a & HRM_BASEMASK) &&
4717c478bd9Sstevel@tonic-gate id == hrm->hrm_id) {
4727c478bd9Sstevel@tonic-gate int i, nr;
4737c478bd9Sstevel@tonic-gate uint_t bo, spb;
4747c478bd9Sstevel@tonic-gate
4757c478bd9Sstevel@tonic-gate /*
4767c478bd9Sstevel@tonic-gate * Extract leading unaligned bits.
4777c478bd9Sstevel@tonic-gate */
4787c478bd9Sstevel@tonic-gate i = 0;
4797c478bd9Sstevel@tonic-gate while (i < n && (po & 3)) {
4807c478bd9Sstevel@tonic-gate bo = po / (NBBY / 2);
4817c478bd9Sstevel@tonic-gate spb = (3 - (po & 3)) * 2;
4827c478bd9Sstevel@tonic-gate *dp++ |= (hrm->hrm_bits[bo] >> spb) & 3;
4837c478bd9Sstevel@tonic-gate if (clearflag)
4847c478bd9Sstevel@tonic-gate hrm->hrm_bits[bo] &= ~(3<<spb);
4857c478bd9Sstevel@tonic-gate po++;
4867c478bd9Sstevel@tonic-gate i++;
4877c478bd9Sstevel@tonic-gate }
4887c478bd9Sstevel@tonic-gate /*
4897c478bd9Sstevel@tonic-gate * Extract aligned bits.
4907c478bd9Sstevel@tonic-gate */
4917c478bd9Sstevel@tonic-gate nr = n/4*4;
4927c478bd9Sstevel@tonic-gate bo = po / (NBBY / 2);
4937c478bd9Sstevel@tonic-gate while (i < nr) {
4947c478bd9Sstevel@tonic-gate int bits = hrm->hrm_bits[bo];
4957c478bd9Sstevel@tonic-gate *dp++ |= (bits >> 6) & 3;
4967c478bd9Sstevel@tonic-gate *dp++ |= (bits >> 4) & 3;
4977c478bd9Sstevel@tonic-gate *dp++ |= (bits >> 2) & 3;
4987c478bd9Sstevel@tonic-gate *dp++ |= (bits >> 0) & 3;
4997c478bd9Sstevel@tonic-gate if (clearflag)
5007c478bd9Sstevel@tonic-gate hrm->hrm_bits[bo] = 0;
5017c478bd9Sstevel@tonic-gate bo++;
5027c478bd9Sstevel@tonic-gate po += 4;
5037c478bd9Sstevel@tonic-gate i += 4;
5047c478bd9Sstevel@tonic-gate }
5057c478bd9Sstevel@tonic-gate /*
5067c478bd9Sstevel@tonic-gate * Extract trailing unaligned bits.
5077c478bd9Sstevel@tonic-gate */
5087c478bd9Sstevel@tonic-gate while (i < n) {
5097c478bd9Sstevel@tonic-gate bo = po / (NBBY / 2);
5107c478bd9Sstevel@tonic-gate spb = (3 - (po & 3)) * 2;
5117c478bd9Sstevel@tonic-gate *dp++ |= (hrm->hrm_bits[bo] >> spb) & 3;
5127c478bd9Sstevel@tonic-gate if (clearflag)
5137c478bd9Sstevel@tonic-gate hrm->hrm_bits[bo] &= ~(3<<spb);
5147c478bd9Sstevel@tonic-gate po++;
5157c478bd9Sstevel@tonic-gate i++;
5167c478bd9Sstevel@tonic-gate }
5177c478bd9Sstevel@tonic-gate
5187c478bd9Sstevel@tonic-gate break;
5197c478bd9Sstevel@tonic-gate }
5207c478bd9Sstevel@tonic-gate }
5217c478bd9Sstevel@tonic-gate if (hrm == NULL)
5227c478bd9Sstevel@tonic-gate dp += n;
5237c478bd9Sstevel@tonic-gate np -= n;
5247c478bd9Sstevel@tonic-gate a += n * MMU_PAGESIZE;
5257c478bd9Sstevel@tonic-gate }
5267c478bd9Sstevel@tonic-gate mutex_exit(&hat_statlock);
5277c478bd9Sstevel@tonic-gate }
528