17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
51d530678Sraf  * Common Development and Distribution License (the "License").
61d530678Sraf  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
211d530678Sraf 
227c478bd9Sstevel@tonic-gate /*
23db3659e5SSurya Prakki  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
247c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
257c478bd9Sstevel@tonic-gate  */
267c478bd9Sstevel@tonic-gate 
277c478bd9Sstevel@tonic-gate #include <mtmalloc.h>
287c478bd9Sstevel@tonic-gate #include "mtmalloc_impl.h"
297c478bd9Sstevel@tonic-gate #include <unistd.h>
307c478bd9Sstevel@tonic-gate #include <synch.h>
317c478bd9Sstevel@tonic-gate #include <thread.h>
321d530678Sraf #include <pthread.h>
337c478bd9Sstevel@tonic-gate #include <stdio.h>
347c478bd9Sstevel@tonic-gate #include <limits.h>
357c478bd9Sstevel@tonic-gate #include <errno.h>
367c478bd9Sstevel@tonic-gate #include <string.h>
377c478bd9Sstevel@tonic-gate #include <strings.h>
387c478bd9Sstevel@tonic-gate #include <sys/param.h>
397c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
407c478bd9Sstevel@tonic-gate 
417c478bd9Sstevel@tonic-gate /*
427c478bd9Sstevel@tonic-gate  * To turn on the asserts just compile -DDEBUG
437c478bd9Sstevel@tonic-gate  */
447c478bd9Sstevel@tonic-gate 
457c478bd9Sstevel@tonic-gate #ifndef	DEBUG
467c478bd9Sstevel@tonic-gate #define	NDEBUG
477c478bd9Sstevel@tonic-gate #endif
487c478bd9Sstevel@tonic-gate 
497c478bd9Sstevel@tonic-gate #include <assert.h>
507c478bd9Sstevel@tonic-gate 
517c478bd9Sstevel@tonic-gate /*
527c478bd9Sstevel@tonic-gate  * The MT hot malloc implementation contained herein is designed to be
537c478bd9Sstevel@tonic-gate  * plug-compatible with the libc version of malloc. It is not intended
547c478bd9Sstevel@tonic-gate  * to replace that implementation until we decide that it is ok to break
557c478bd9Sstevel@tonic-gate  * customer apps (Solaris 3.0).
567c478bd9Sstevel@tonic-gate  *
577c478bd9Sstevel@tonic-gate  * For requests up to 2^^16, the allocator initializes itself into NCPUS
587c478bd9Sstevel@tonic-gate  * worth of chains of caches. When a memory request is made, the calling thread
597c478bd9Sstevel@tonic-gate  * is vectored into one of NCPUS worth of caches.  The LWP id gives us a cheap,
607c478bd9Sstevel@tonic-gate  * contention-reducing index to use, eventually, this should be replaced with
617c478bd9Sstevel@tonic-gate  * the actual CPU sequence number, when an interface to get it is available.
627c478bd9Sstevel@tonic-gate  *
637c478bd9Sstevel@tonic-gate  * Once the thread is vectored into one of the list of caches the real
647c478bd9Sstevel@tonic-gate  * allocation of the memory begins. The size is determined to figure out which
657c478bd9Sstevel@tonic-gate  * bucket the allocation should be satisfied from. The management of free
667c478bd9Sstevel@tonic-gate  * buckets is done via a bitmask. A free bucket is represented by a 1. The
677c478bd9Sstevel@tonic-gate  * first free bit represents the first free bucket. The position of the bit,
687c478bd9Sstevel@tonic-gate  * represents the position of the bucket in the arena.
697c478bd9Sstevel@tonic-gate  *
707c478bd9Sstevel@tonic-gate  * When the memory from the arena is handed out, the address of the cache
717c478bd9Sstevel@tonic-gate  * control structure is written in the word preceeding the returned memory.
727c478bd9Sstevel@tonic-gate  * This cache control address is used during free() to mark the buffer free
737c478bd9Sstevel@tonic-gate  * in the cache control structure.
747c478bd9Sstevel@tonic-gate  *
757c478bd9Sstevel@tonic-gate  * When all available memory in a cache has been depleted, a new chunk of memory
767c478bd9Sstevel@tonic-gate  * is allocated via sbrk(). The new cache is allocated from this chunk of memory
777c478bd9Sstevel@tonic-gate  * and initialized in the function create_cache(). New caches are installed at
787c478bd9Sstevel@tonic-gate  * the front of a singly linked list of the same size memory pools. This helps
797c478bd9Sstevel@tonic-gate  * to ensure that there will tend to be available memory in the beginning of the
807c478bd9Sstevel@tonic-gate  * list.
817c478bd9Sstevel@tonic-gate  *
827c478bd9Sstevel@tonic-gate  * Long linked lists hurt performance. To decrease this effect, there is a
837c478bd9Sstevel@tonic-gate  * tunable, requestsize, that bumps up the sbrk allocation size and thus
847c478bd9Sstevel@tonic-gate  * increases the number of available blocks within an arena.  We also keep
857c478bd9Sstevel@tonic-gate  * a "hint" for each cache list, which is the last cache in the list allocated
867c478bd9Sstevel@tonic-gate  * from.  This lowers the cost of searching if there are a lot of fully
877c478bd9Sstevel@tonic-gate  * allocated blocks at the front of the list.
887c478bd9Sstevel@tonic-gate  *
897c478bd9Sstevel@tonic-gate  * For requests greater than 2^^16 (oversize allocations), there are two pieces
907c478bd9Sstevel@tonic-gate  * of overhead. There is the OVERHEAD used to hold the cache addr
917c478bd9Sstevel@tonic-gate  * (&oversize_list), plus an oversize_t structure to further describe the block.
927c478bd9Sstevel@tonic-gate  *
937c478bd9Sstevel@tonic-gate  * The oversize list is kept as defragmented as possible by coalescing
947c478bd9Sstevel@tonic-gate  * freed oversized allocations with adjacent neighbors.
957c478bd9Sstevel@tonic-gate  *
967c478bd9Sstevel@tonic-gate  * Addresses handed out are stored in a hash table, and are aligned on
977c478bd9Sstevel@tonic-gate  * MTMALLOC_MIN_ALIGN-byte boundaries at both ends. Request sizes are rounded-up
987c478bd9Sstevel@tonic-gate  * where necessary in order to achieve this. This eases the implementation of
997c478bd9Sstevel@tonic-gate  * MTDEBUGPATTERN and MTINITPATTERN, particularly where coalescing occurs.
1007c478bd9Sstevel@tonic-gate  *
1017c478bd9Sstevel@tonic-gate  * A memalign allocation takes memalign header overhead.  There's two
1027c478bd9Sstevel@tonic-gate  * types of memalign headers distinguished by MTMALLOC_MEMALIGN_MAGIC
1037c478bd9Sstevel@tonic-gate  * and MTMALLOC_MEMALIGN_MIN_MAGIC.  When the size of memory taken to
1047c478bd9Sstevel@tonic-gate  * get to the aligned address from malloc'ed address is the minimum size
1057c478bd9Sstevel@tonic-gate  * OVERHEAD, we create a header taking only one OVERHEAD space with magic
1067c478bd9Sstevel@tonic-gate  * number MTMALLOC_MEMALIGN_MIN_MAGIC, and we know by subtracting OVERHEAD
1077c478bd9Sstevel@tonic-gate  * from memaligned address, we can get to the malloc'ed address. Otherwise,
1087c478bd9Sstevel@tonic-gate  * we create a memalign header taking two OVERHEAD space, one stores
1097c478bd9Sstevel@tonic-gate  * MTMALLOC_MEMALIGN_MAGIC magic number, the other one points back to the
1107c478bd9Sstevel@tonic-gate  * malloc'ed address.
1117c478bd9Sstevel@tonic-gate  */
1127c478bd9Sstevel@tonic-gate 
1137c478bd9Sstevel@tonic-gate #if defined(__i386) || defined(__amd64)
1147c478bd9Sstevel@tonic-gate #include <arpa/inet.h>	/* for htonl() */
1157c478bd9Sstevel@tonic-gate #endif
1167c478bd9Sstevel@tonic-gate 
1177c478bd9Sstevel@tonic-gate static void * morecore(size_t);
1187c478bd9Sstevel@tonic-gate static void create_cache(cache_t *, size_t bufsize, uint_t hunks);
1197c478bd9Sstevel@tonic-gate static void * malloc_internal(size_t, percpu_t *);
1207c478bd9Sstevel@tonic-gate static void * oversize(size_t);
1217c478bd9Sstevel@tonic-gate static oversize_t *find_oversize(size_t);
1227c478bd9Sstevel@tonic-gate static void add_oversize(oversize_t *);
1237c478bd9Sstevel@tonic-gate static void copy_pattern(uint32_t, void *, size_t);
1247c478bd9Sstevel@tonic-gate static void * verify_pattern(uint32_t, void *, size_t);
1257c478bd9Sstevel@tonic-gate static void reinit_cpu_list(void);
1267c478bd9Sstevel@tonic-gate static void reinit_cache(cache_t *);
1277c478bd9Sstevel@tonic-gate static void free_oversize(oversize_t *);
1287c478bd9Sstevel@tonic-gate static oversize_t *oversize_header_alloc(uintptr_t, size_t);
1297c478bd9Sstevel@tonic-gate 
1307c478bd9Sstevel@tonic-gate /*
1317c478bd9Sstevel@tonic-gate  * oversize hash table stuff
1327c478bd9Sstevel@tonic-gate  */
1337c478bd9Sstevel@tonic-gate #define	NUM_BUCKETS	67	/* must be prime */
1347c478bd9Sstevel@tonic-gate #define	HASH_OVERSIZE(caddr)	((uintptr_t)(caddr) % NUM_BUCKETS)
1357c478bd9Sstevel@tonic-gate oversize_t *ovsz_hashtab[NUM_BUCKETS];
1367c478bd9Sstevel@tonic-gate 
1377c478bd9Sstevel@tonic-gate #define	ALIGN(x, a)	((((uintptr_t)(x) + ((uintptr_t)(a) - 1)) \
1387c478bd9Sstevel@tonic-gate 			& ~((uintptr_t)(a) - 1)))
1397c478bd9Sstevel@tonic-gate 
1407c478bd9Sstevel@tonic-gate /* need this to deal with little endianess of x86 */
1417c478bd9Sstevel@tonic-gate #if defined(__i386) || defined(__amd64)
1427c478bd9Sstevel@tonic-gate #define	FLIP_EM(x)	htonl((x))
1437c478bd9Sstevel@tonic-gate #else
1447c478bd9Sstevel@tonic-gate #define	FLIP_EM(x)	(x)
1457c478bd9Sstevel@tonic-gate #endif
1467c478bd9Sstevel@tonic-gate 
1477c478bd9Sstevel@tonic-gate #define	INSERT_ONLY			0
1487c478bd9Sstevel@tonic-gate #define	COALESCE_LEFT			0x00000001
1497c478bd9Sstevel@tonic-gate #define	COALESCE_RIGHT			0x00000002
1507c478bd9Sstevel@tonic-gate #define	COALESCE_WITH_BOTH_SIDES	(COALESCE_LEFT | COALESCE_RIGHT)
1517c478bd9Sstevel@tonic-gate 
1527c478bd9Sstevel@tonic-gate #define	OVERHEAD	8	/* size needed to write cache addr */
1537c478bd9Sstevel@tonic-gate #define	HUNKSIZE	8192	/* just a multiplier */
1547c478bd9Sstevel@tonic-gate 
1557c478bd9Sstevel@tonic-gate #define	MAX_CACHED_SHIFT	16	/* 64K is the max cached size */
1567c478bd9Sstevel@tonic-gate #define	MAX_CACHED		(1 << MAX_CACHED_SHIFT)
1577c478bd9Sstevel@tonic-gate #define	MIN_CACHED_SHIFT	4	/* smaller requests rounded up */
1587c478bd9Sstevel@tonic-gate #define	MTMALLOC_MIN_ALIGN	8	/* min guaranteed alignment */
1597c478bd9Sstevel@tonic-gate 
16070911a0dSrm /* maximum size before overflow */
16170911a0dSrm #define	MAX_MTMALLOC	(SIZE_MAX - (SIZE_MAX % MTMALLOC_MIN_ALIGN) \
16270911a0dSrm 			- OVSZ_HEADER_SIZE)
16370911a0dSrm 
1647c478bd9Sstevel@tonic-gate #define	NUM_CACHES	(MAX_CACHED_SHIFT - MIN_CACHED_SHIFT + 1)
1657c478bd9Sstevel@tonic-gate #define	CACHELIST_SIZE	ALIGN(NUM_CACHES * sizeof (cache_head_t), \
1667c478bd9Sstevel@tonic-gate     CACHE_COHERENCY_UNIT)
1677c478bd9Sstevel@tonic-gate 
1687c478bd9Sstevel@tonic-gate #define	MINSIZE		9	/* for requestsize, tunable */
1697c478bd9Sstevel@tonic-gate #define	MAXSIZE		256	/* arbitrary, big enough, for requestsize */
1707c478bd9Sstevel@tonic-gate 
1717c478bd9Sstevel@tonic-gate #define	FREEPATTERN	0xdeadbeef /* debug fill pattern for free buf */
1727c478bd9Sstevel@tonic-gate #define	INITPATTERN	0xbaddcafe /* debug fill pattern for new buf */
1737c478bd9Sstevel@tonic-gate 
1747c478bd9Sstevel@tonic-gate #define	misaligned(p)	((unsigned)(p) & (sizeof (int) - 1))
1757c478bd9Sstevel@tonic-gate #define	IS_OVERSIZE(x, y)	(((x) < (y)) && (((x) > MAX_CACHED)? 1 : 0))
1767c478bd9Sstevel@tonic-gate 
1777c478bd9Sstevel@tonic-gate static long requestsize = MINSIZE; /* 9 pages per cache; tunable; 9 is min */
1787c478bd9Sstevel@tonic-gate 
1797c478bd9Sstevel@tonic-gate static uint_t cpu_mask;
1807c478bd9Sstevel@tonic-gate static curcpu_func curcpu;
1817c478bd9Sstevel@tonic-gate 
1827c478bd9Sstevel@tonic-gate static int32_t debugopt;
1837c478bd9Sstevel@tonic-gate static int32_t reinit;
1847c478bd9Sstevel@tonic-gate 
1857c478bd9Sstevel@tonic-gate static percpu_t *cpu_list;
1867c478bd9Sstevel@tonic-gate static oversize_t oversize_list;
1871d530678Sraf static mutex_t oversize_lock = DEFAULTMUTEX;
1887c478bd9Sstevel@tonic-gate 
1891d530678Sraf static int ncpus = 0;
1907c478bd9Sstevel@tonic-gate 
1917c478bd9Sstevel@tonic-gate #define	MTMALLOC_OVERSIZE_MAGIC		((uintptr_t)&oversize_list)
1927c478bd9Sstevel@tonic-gate #define	MTMALLOC_MEMALIGN_MAGIC		((uintptr_t)&oversize_list + 1)
1937c478bd9Sstevel@tonic-gate #define	MTMALLOC_MEMALIGN_MIN_MAGIC	((uintptr_t)&oversize_list + 2)
1947c478bd9Sstevel@tonic-gate 
1957c478bd9Sstevel@tonic-gate /*
1967c478bd9Sstevel@tonic-gate  * We require allocations handed out to be aligned on MTMALLOC_MIN_ALIGN-byte
1977c478bd9Sstevel@tonic-gate  * boundaries. We round up sizeof (oversize_t) (when necessary) to ensure that
1987c478bd9Sstevel@tonic-gate  * this is achieved.
1997c478bd9Sstevel@tonic-gate  */
2007c478bd9Sstevel@tonic-gate #define	OVSZ_SIZE		(ALIGN(sizeof (oversize_t), MTMALLOC_MIN_ALIGN))
2017c478bd9Sstevel@tonic-gate #define	OVSZ_HEADER_SIZE	(OVSZ_SIZE + OVERHEAD)
2027c478bd9Sstevel@tonic-gate 
2037c478bd9Sstevel@tonic-gate /*
2047c478bd9Sstevel@tonic-gate  * memalign header takes 2 OVERHEAD space.  One for memalign magic, and the
2057c478bd9Sstevel@tonic-gate  * other one points back to the start address of originally allocated space.
2067c478bd9Sstevel@tonic-gate  */
2077c478bd9Sstevel@tonic-gate #define	MEMALIGN_HEADER_SIZE	2 * OVERHEAD
2087c478bd9Sstevel@tonic-gate #define	MEMALIGN_HEADER_ALLOC(x, shift, malloc_addr)\
2097c478bd9Sstevel@tonic-gate 	if (shift == OVERHEAD)\
2107c478bd9Sstevel@tonic-gate 		*((uintptr_t *)((caddr_t)x - OVERHEAD)) = \
2117c478bd9Sstevel@tonic-gate 			MTMALLOC_MEMALIGN_MIN_MAGIC; \
2127c478bd9Sstevel@tonic-gate 	else {\
2137c478bd9Sstevel@tonic-gate 		*((uintptr_t *)((caddr_t)x - OVERHEAD)) = \
2147c478bd9Sstevel@tonic-gate 			MTMALLOC_MEMALIGN_MAGIC; \
2157c478bd9Sstevel@tonic-gate 		*((uintptr_t *)((caddr_t)x - 2 * OVERHEAD)) = \
2167c478bd9Sstevel@tonic-gate 			(uintptr_t)malloc_addr; \
2177c478bd9Sstevel@tonic-gate 	}
2187c478bd9Sstevel@tonic-gate 
2198f2954faSraf /*
2208f2954faSraf  * Add big to the oversize hash table at the head of the relevant bucket.
2218f2954faSraf  */
2228f2954faSraf static void
insert_hash(oversize_t * big)2238f2954faSraf insert_hash(oversize_t *big)
2248f2954faSraf {
2258f2954faSraf 	caddr_t ret = big->addr;
2268f2954faSraf 	int bucket = HASH_OVERSIZE(ret);
2278f2954faSraf 
2288f2954faSraf 	assert(MUTEX_HELD(&oversize_lock));
2298f2954faSraf 	big->hash_next = ovsz_hashtab[bucket];
2308f2954faSraf 	ovsz_hashtab[bucket] = big;
2318f2954faSraf }
2328f2954faSraf 
2337c478bd9Sstevel@tonic-gate void *
malloc(size_t bytes)2347c478bd9Sstevel@tonic-gate malloc(size_t bytes)
2357c478bd9Sstevel@tonic-gate {
2367c478bd9Sstevel@tonic-gate 	percpu_t *list_rotor;
2377c478bd9Sstevel@tonic-gate 	uint_t	list_index;
2387c478bd9Sstevel@tonic-gate 
2397c478bd9Sstevel@tonic-gate 	if (bytes > MAX_CACHED)
2407c478bd9Sstevel@tonic-gate 		return (oversize(bytes));
2417c478bd9Sstevel@tonic-gate 
2427c478bd9Sstevel@tonic-gate 	list_index = (curcpu() & cpu_mask);
2437c478bd9Sstevel@tonic-gate 
2447c478bd9Sstevel@tonic-gate 	list_rotor = &cpu_list[list_index];
2457c478bd9Sstevel@tonic-gate 
2467c478bd9Sstevel@tonic-gate 	return (malloc_internal(bytes, list_rotor));
2477c478bd9Sstevel@tonic-gate }
2487c478bd9Sstevel@tonic-gate 
2497c478bd9Sstevel@tonic-gate void *
realloc(void * ptr,size_t bytes)2507c478bd9Sstevel@tonic-gate realloc(void * ptr, size_t bytes)
2517c478bd9Sstevel@tonic-gate {
2527c478bd9Sstevel@tonic-gate 	void *new, *data_ptr;
2537c478bd9Sstevel@tonic-gate 	cache_t *cacheptr;
2547c478bd9Sstevel@tonic-gate 	caddr_t mem;
2557c478bd9Sstevel@tonic-gate 	size_t shift = 0;
2567c478bd9Sstevel@tonic-gate 
2577c478bd9Sstevel@tonic-gate 	if (ptr == NULL)
2587c478bd9Sstevel@tonic-gate 		return (malloc(bytes));
2597c478bd9Sstevel@tonic-gate 
2607c478bd9Sstevel@tonic-gate 	if (bytes == 0) {
2617c478bd9Sstevel@tonic-gate 		free(ptr);
2627c478bd9Sstevel@tonic-gate 		return (NULL);
2637c478bd9Sstevel@tonic-gate 	}
2647c478bd9Sstevel@tonic-gate 
2657c478bd9Sstevel@tonic-gate 	data_ptr = ptr;
2667c478bd9Sstevel@tonic-gate 	mem = (caddr_t)ptr - OVERHEAD;
2677c478bd9Sstevel@tonic-gate 
268db3659e5SSurya Prakki 	/*
269db3659e5SSurya Prakki 	 * Optimization possibility :
270db3659e5SSurya Prakki 	 *	p = malloc(64);
271db3659e5SSurya Prakki 	 *	q = realloc(p, 64);
272db3659e5SSurya Prakki 	 * q can be same as p.
273db3659e5SSurya Prakki 	 * Apply this optimization for the normal
274db3659e5SSurya Prakki 	 * sized caches for now.
275db3659e5SSurya Prakki 	 */
276db3659e5SSurya Prakki 	if (*(uintptr_t *)mem < MTMALLOC_OVERSIZE_MAGIC ||
277db3659e5SSurya Prakki 	    *(uintptr_t *)mem > MTMALLOC_MEMALIGN_MIN_MAGIC) {
278db3659e5SSurya Prakki 		cacheptr = (cache_t *)*(uintptr_t *)mem;
279db3659e5SSurya Prakki 		if (bytes <= (cacheptr->mt_size - OVERHEAD))
280db3659e5SSurya Prakki 			return (ptr);
281db3659e5SSurya Prakki 	}
282db3659e5SSurya Prakki 
2837c478bd9Sstevel@tonic-gate 	new = malloc(bytes);
2847c478bd9Sstevel@tonic-gate 
2857c478bd9Sstevel@tonic-gate 	if (new == NULL)
2867c478bd9Sstevel@tonic-gate 		return (NULL);
2877c478bd9Sstevel@tonic-gate 
2887c478bd9Sstevel@tonic-gate 	/*
2897c478bd9Sstevel@tonic-gate 	 * If new == ptr, ptr has previously been freed. Passing a freed pointer
2907c478bd9Sstevel@tonic-gate 	 * to realloc() is not allowed - unless the caller specifically states
2917c478bd9Sstevel@tonic-gate 	 * otherwise, in which case we must avoid freeing ptr (ie new) before we
2927c478bd9Sstevel@tonic-gate 	 * return new. There is (obviously) no requirement to memcpy() ptr to
2937c478bd9Sstevel@tonic-gate 	 * new before we return.
2947c478bd9Sstevel@tonic-gate 	 */
2957c478bd9Sstevel@tonic-gate 	if (new == ptr) {
2967c478bd9Sstevel@tonic-gate 		if (!(debugopt & MTDOUBLEFREE))
2977c478bd9Sstevel@tonic-gate 			abort();
2987c478bd9Sstevel@tonic-gate 		return (new);
2997c478bd9Sstevel@tonic-gate 	}
3007c478bd9Sstevel@tonic-gate 
3017c478bd9Sstevel@tonic-gate 	if (*(uintptr_t *)mem == MTMALLOC_MEMALIGN_MAGIC) {
3027c478bd9Sstevel@tonic-gate 		mem -= OVERHEAD;
3037c478bd9Sstevel@tonic-gate 		ptr = (void *)*(uintptr_t *)mem;
3047c478bd9Sstevel@tonic-gate 		mem = (caddr_t)ptr - OVERHEAD;
3057c478bd9Sstevel@tonic-gate 		shift = (size_t)((uintptr_t)data_ptr - (uintptr_t)ptr);
3067c478bd9Sstevel@tonic-gate 	} else if (*(uintptr_t *)mem == MTMALLOC_MEMALIGN_MIN_MAGIC) {
3077c478bd9Sstevel@tonic-gate 		ptr = (void *) mem;
3087c478bd9Sstevel@tonic-gate 		mem -= OVERHEAD;
3097c478bd9Sstevel@tonic-gate 		shift = OVERHEAD;
3107c478bd9Sstevel@tonic-gate 	}
3117c478bd9Sstevel@tonic-gate 
3127c478bd9Sstevel@tonic-gate 	if (*(uintptr_t *)mem == MTMALLOC_OVERSIZE_MAGIC) {
3137c478bd9Sstevel@tonic-gate 		oversize_t *old;
3147c478bd9Sstevel@tonic-gate 
3157c478bd9Sstevel@tonic-gate 		old = (oversize_t *)(mem - OVSZ_SIZE);
3167c478bd9Sstevel@tonic-gate 		(void) memcpy(new, data_ptr, MIN(bytes, old->size - shift));
3177c478bd9Sstevel@tonic-gate 		free(ptr);
3187c478bd9Sstevel@tonic-gate 		return (new);
3197c478bd9Sstevel@tonic-gate 	}
3207c478bd9Sstevel@tonic-gate 
3217c478bd9Sstevel@tonic-gate 	cacheptr = (cache_t *)*(uintptr_t *)mem;
3227c478bd9Sstevel@tonic-gate 
3237c478bd9Sstevel@tonic-gate 	(void) memcpy(new, data_ptr,
324db3659e5SSurya Prakki 	    MIN(cacheptr->mt_size - OVERHEAD - shift, bytes));
3257c478bd9Sstevel@tonic-gate 	free(ptr);
3267c478bd9Sstevel@tonic-gate 
3277c478bd9Sstevel@tonic-gate 	return (new);
3287c478bd9Sstevel@tonic-gate }
3297c478bd9Sstevel@tonic-gate 
3307c478bd9Sstevel@tonic-gate void *
calloc(size_t nelem,size_t bytes)3317c478bd9Sstevel@tonic-gate calloc(size_t nelem, size_t bytes)
3327c478bd9Sstevel@tonic-gate {
3337c478bd9Sstevel@tonic-gate 	void * ptr;
33431c6d826SRichard Lowe 	size_t size;
33531c6d826SRichard Lowe 
33631c6d826SRichard Lowe 	if (nelem == 0 || bytes == 0) {
33731c6d826SRichard Lowe 		size = 0;
33831c6d826SRichard Lowe 	} else {
33931c6d826SRichard Lowe 		size = nelem * bytes;
34031c6d826SRichard Lowe 
34131c6d826SRichard Lowe 		/* check for overflow */
34231c6d826SRichard Lowe 		if ((size / nelem) != bytes) {
34331c6d826SRichard Lowe 			errno = ENOMEM;
34431c6d826SRichard Lowe 			return (NULL);
34531c6d826SRichard Lowe 		}
34631c6d826SRichard Lowe 	}
3477c478bd9Sstevel@tonic-gate 
3487c478bd9Sstevel@tonic-gate 	ptr = malloc(size);
3497c478bd9Sstevel@tonic-gate 	if (ptr == NULL)
3507c478bd9Sstevel@tonic-gate 		return (NULL);
3511d530678Sraf 	(void) memset(ptr, 0, size);
3527c478bd9Sstevel@tonic-gate 
3537c478bd9Sstevel@tonic-gate 	return (ptr);
3547c478bd9Sstevel@tonic-gate }
3557c478bd9Sstevel@tonic-gate 
3567c478bd9Sstevel@tonic-gate void
free(void * ptr)3577c478bd9Sstevel@tonic-gate free(void * ptr)
3587c478bd9Sstevel@tonic-gate {
3597c478bd9Sstevel@tonic-gate 	cache_t *cacheptr;
3607c478bd9Sstevel@tonic-gate 	caddr_t mem;
3617c478bd9Sstevel@tonic-gate 	int32_t i;
3627c478bd9Sstevel@tonic-gate 	caddr_t freeblocks;
3637c478bd9Sstevel@tonic-gate 	uintptr_t offset;
3647c478bd9Sstevel@tonic-gate 	uchar_t mask;
3657c478bd9Sstevel@tonic-gate 	int32_t which_bit, num_bytes;
3667c478bd9Sstevel@tonic-gate 
3677c478bd9Sstevel@tonic-gate 	if (ptr == NULL)
3687c478bd9Sstevel@tonic-gate 		return;
3697c478bd9Sstevel@tonic-gate 
3707c478bd9Sstevel@tonic-gate 	mem = (caddr_t)ptr - OVERHEAD;
3717c478bd9Sstevel@tonic-gate 
3727c478bd9Sstevel@tonic-gate 	if (*(uintptr_t *)mem == MTMALLOC_MEMALIGN_MAGIC) {
3737c478bd9Sstevel@tonic-gate 		mem -= OVERHEAD;
3747c478bd9Sstevel@tonic-gate 		ptr = (void *)*(uintptr_t *)mem;
3757c478bd9Sstevel@tonic-gate 		mem = (caddr_t)ptr - OVERHEAD;
3767c478bd9Sstevel@tonic-gate 	} else if (*(uintptr_t *)mem == MTMALLOC_MEMALIGN_MIN_MAGIC) {
3777c478bd9Sstevel@tonic-gate 		ptr = (void *) mem;
3787c478bd9Sstevel@tonic-gate 		mem -= OVERHEAD;
3797c478bd9Sstevel@tonic-gate 	}
3807c478bd9Sstevel@tonic-gate 
3817c478bd9Sstevel@tonic-gate 	if (*(uintptr_t *)mem == MTMALLOC_OVERSIZE_MAGIC) {
3827c478bd9Sstevel@tonic-gate 		oversize_t *big, **opp;
3837c478bd9Sstevel@tonic-gate 		int bucket;
3847c478bd9Sstevel@tonic-gate 
3857c478bd9Sstevel@tonic-gate 		big = (oversize_t *)(mem - OVSZ_SIZE);
3867c478bd9Sstevel@tonic-gate 		(void) mutex_lock(&oversize_lock);
3877c478bd9Sstevel@tonic-gate 
3887c478bd9Sstevel@tonic-gate 		bucket = HASH_OVERSIZE(big->addr);
3897c478bd9Sstevel@tonic-gate 		for (opp = &ovsz_hashtab[bucket]; *opp != NULL;
3907c478bd9Sstevel@tonic-gate 		    opp = &(*opp)->hash_next)
3917c478bd9Sstevel@tonic-gate 			if (*opp == big)
3927c478bd9Sstevel@tonic-gate 				break;
3937c478bd9Sstevel@tonic-gate 
3947c478bd9Sstevel@tonic-gate 		if (*opp == NULL) {
3957c478bd9Sstevel@tonic-gate 			if (!(debugopt & MTDOUBLEFREE))
3967c478bd9Sstevel@tonic-gate 				abort();
3977c478bd9Sstevel@tonic-gate 			(void) mutex_unlock(&oversize_lock);
3987c478bd9Sstevel@tonic-gate 			return;
3997c478bd9Sstevel@tonic-gate 		}
4007c478bd9Sstevel@tonic-gate 
4017c478bd9Sstevel@tonic-gate 		*opp = big->hash_next;	/* remove big from the hash table */
4027c478bd9Sstevel@tonic-gate 		big->hash_next = NULL;
4037c478bd9Sstevel@tonic-gate 
4047c478bd9Sstevel@tonic-gate 		if (debugopt & MTDEBUGPATTERN)
4057c478bd9Sstevel@tonic-gate 			copy_pattern(FREEPATTERN, ptr, big->size);
4067c478bd9Sstevel@tonic-gate 		add_oversize(big);
4077c478bd9Sstevel@tonic-gate 		(void) mutex_unlock(&oversize_lock);
4087c478bd9Sstevel@tonic-gate 		return;
4097c478bd9Sstevel@tonic-gate 	}
4107c478bd9Sstevel@tonic-gate 
4117c478bd9Sstevel@tonic-gate 	cacheptr = (cache_t *)*(uintptr_t *)mem;
4127c478bd9Sstevel@tonic-gate 	freeblocks = cacheptr->mt_freelist;
4137c478bd9Sstevel@tonic-gate 
4147c478bd9Sstevel@tonic-gate 	/*
4157c478bd9Sstevel@tonic-gate 	 * This is the distance measured in bits into the arena.
4167c478bd9Sstevel@tonic-gate 	 * The value of offset is in bytes but there is a 1-1 correlation
4177c478bd9Sstevel@tonic-gate 	 * between distance into the arena and distance into the
4187c478bd9Sstevel@tonic-gate 	 * freelist bitmask.
4197c478bd9Sstevel@tonic-gate 	 */
4207c478bd9Sstevel@tonic-gate 	offset = mem - cacheptr->mt_arena;
4217c478bd9Sstevel@tonic-gate 
4227c478bd9Sstevel@tonic-gate 	/*
4237c478bd9Sstevel@tonic-gate 	 * i is total number of bits to offset into freelist bitmask.
4247c478bd9Sstevel@tonic-gate 	 */
4257c478bd9Sstevel@tonic-gate 
4267c478bd9Sstevel@tonic-gate 	i = offset / cacheptr->mt_size;
4277c478bd9Sstevel@tonic-gate 
4287c478bd9Sstevel@tonic-gate 	num_bytes = i >> 3;
4297c478bd9Sstevel@tonic-gate 
4307c478bd9Sstevel@tonic-gate 	/*
4317c478bd9Sstevel@tonic-gate 	 * which_bit is the bit offset into the byte in the freelist.
4327c478bd9Sstevel@tonic-gate 	 * if our freelist bitmask looks like 0xf3 and we are freeing
4337c478bd9Sstevel@tonic-gate 	 * block 5 (ie: the 6th block) our mask will be 0xf7 after
4347c478bd9Sstevel@tonic-gate 	 * the free. Things go left to right that's why the mask is 0x80
4357c478bd9Sstevel@tonic-gate 	 * and not 0x01.
4367c478bd9Sstevel@tonic-gate 	 */
4377c478bd9Sstevel@tonic-gate 	which_bit = i - (num_bytes << 3);
4387c478bd9Sstevel@tonic-gate 
4397c478bd9Sstevel@tonic-gate 	mask = 0x80 >> which_bit;
4407c478bd9Sstevel@tonic-gate 
4417c478bd9Sstevel@tonic-gate 	freeblocks += num_bytes;
4427c478bd9Sstevel@tonic-gate 
4437c478bd9Sstevel@tonic-gate 	if (debugopt & MTDEBUGPATTERN)
4447c478bd9Sstevel@tonic-gate 		copy_pattern(FREEPATTERN, ptr, cacheptr->mt_size - OVERHEAD);
4457c478bd9Sstevel@tonic-gate 
4467c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&cacheptr->mt_cache_lock);
4477c478bd9Sstevel@tonic-gate 
4487c478bd9Sstevel@tonic-gate 	if (*freeblocks & mask) {
4497c478bd9Sstevel@tonic-gate 		if (!(debugopt & MTDOUBLEFREE))
4507c478bd9Sstevel@tonic-gate 			abort();
4517c478bd9Sstevel@tonic-gate 	} else {
4527c478bd9Sstevel@tonic-gate 		*freeblocks |= mask;
4537c478bd9Sstevel@tonic-gate 		cacheptr->mt_nfree++;
4547c478bd9Sstevel@tonic-gate 	}
4557c478bd9Sstevel@tonic-gate 
4567c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&cacheptr->mt_cache_lock);
4577c478bd9Sstevel@tonic-gate }
4587c478bd9Sstevel@tonic-gate 
4597c478bd9Sstevel@tonic-gate void *
memalign(size_t alignment,size_t size)4607c478bd9Sstevel@tonic-gate memalign(size_t alignment, size_t size)
4617c478bd9Sstevel@tonic-gate {
4627c478bd9Sstevel@tonic-gate 	size_t alloc_size;
4637c478bd9Sstevel@tonic-gate 	uintptr_t offset;
4647c478bd9Sstevel@tonic-gate 	void *alloc_buf;
4657c478bd9Sstevel@tonic-gate 	void *ret_buf;
4667c478bd9Sstevel@tonic-gate 
467db3659e5SSurya Prakki 	if (size == 0 || alignment == 0 || misaligned(alignment) ||
468db3659e5SSurya Prakki 	    (alignment & (alignment - 1)) != 0) {
4697c478bd9Sstevel@tonic-gate 		errno = EINVAL;
4707c478bd9Sstevel@tonic-gate 		return (NULL);
4717c478bd9Sstevel@tonic-gate 	}
4727c478bd9Sstevel@tonic-gate 
4737c478bd9Sstevel@tonic-gate 	/* <= MTMALLOC_MIN_ALIGN, malloc can provide directly */
4747c478bd9Sstevel@tonic-gate 	if (alignment <= MTMALLOC_MIN_ALIGN)
4757c478bd9Sstevel@tonic-gate 		return (malloc(size));
4767c478bd9Sstevel@tonic-gate 
4777c478bd9Sstevel@tonic-gate 	alloc_size = size + alignment - MTMALLOC_MIN_ALIGN;
4787c478bd9Sstevel@tonic-gate 
4797c478bd9Sstevel@tonic-gate 	if (alloc_size < size) { /* overflow */
4807c478bd9Sstevel@tonic-gate 		errno = ENOMEM;
4817c478bd9Sstevel@tonic-gate 		return (NULL);
4827c478bd9Sstevel@tonic-gate 	}
4837c478bd9Sstevel@tonic-gate 
4847c478bd9Sstevel@tonic-gate 	alloc_buf = malloc(alloc_size);
4857c478bd9Sstevel@tonic-gate 
4867c478bd9Sstevel@tonic-gate 	if (alloc_buf == NULL)
4877c478bd9Sstevel@tonic-gate 		/* malloc sets errno */
4887c478bd9Sstevel@tonic-gate 		return (NULL);
4897c478bd9Sstevel@tonic-gate 
4907c478bd9Sstevel@tonic-gate 	/*
4917c478bd9Sstevel@tonic-gate 	 * If alloc_size > MAX_CACHED, malloc() will have returned a multiple of
4927c478bd9Sstevel@tonic-gate 	 * MTMALLOC_MIN_ALIGN, having rounded-up alloc_size if necessary. Since
4937c478bd9Sstevel@tonic-gate 	 * we will use alloc_size to return the excess fragments to the free
4947c478bd9Sstevel@tonic-gate 	 * list, we also round-up alloc_size if necessary.
4957c478bd9Sstevel@tonic-gate 	 */
4967c478bd9Sstevel@tonic-gate 	if ((alloc_size > MAX_CACHED) &&
4977c478bd9Sstevel@tonic-gate 	    (alloc_size & (MTMALLOC_MIN_ALIGN - 1)))
4987c478bd9Sstevel@tonic-gate 		alloc_size = ALIGN(alloc_size, MTMALLOC_MIN_ALIGN);
4997c478bd9Sstevel@tonic-gate 
5007c478bd9Sstevel@tonic-gate 	if ((offset = (uintptr_t)alloc_buf & (alignment - 1)) == 0) {
5017c478bd9Sstevel@tonic-gate 		/* aligned correctly */
5027c478bd9Sstevel@tonic-gate 
5037c478bd9Sstevel@tonic-gate 		size_t frag_size = alloc_size -
504db3659e5SSurya Prakki 		    (size + MTMALLOC_MIN_ALIGN + OVSZ_HEADER_SIZE);
5057c478bd9Sstevel@tonic-gate 
5067c478bd9Sstevel@tonic-gate 		/*
5077c478bd9Sstevel@tonic-gate 		 * If the leftover piece of the memory > MAX_CACHED,
5087c478bd9Sstevel@tonic-gate 		 * split off the piece and return it back to the freelist.
5097c478bd9Sstevel@tonic-gate 		 */
5107c478bd9Sstevel@tonic-gate 		if (IS_OVERSIZE(frag_size, alloc_size)) {
5117c478bd9Sstevel@tonic-gate 			oversize_t *orig, *tail;
5127c478bd9Sstevel@tonic-gate 			uintptr_t taddr;
5137c478bd9Sstevel@tonic-gate 			size_t data_size;
5147c478bd9Sstevel@tonic-gate 			taddr = ALIGN((uintptr_t)alloc_buf + size,
515db3659e5SSurya Prakki 			    MTMALLOC_MIN_ALIGN);
5167c478bd9Sstevel@tonic-gate 			data_size = taddr - (uintptr_t)alloc_buf;
5177c478bd9Sstevel@tonic-gate 			orig = (oversize_t *)((uintptr_t)alloc_buf -
518db3659e5SSurya Prakki 			    OVSZ_HEADER_SIZE);
5197c478bd9Sstevel@tonic-gate 			frag_size = orig->size - data_size -
520db3659e5SSurya Prakki 			    OVSZ_HEADER_SIZE;
5217c478bd9Sstevel@tonic-gate 			orig->size = data_size;
5227c478bd9Sstevel@tonic-gate 			tail = oversize_header_alloc(taddr, frag_size);
5237c478bd9Sstevel@tonic-gate 			free_oversize(tail);
5247c478bd9Sstevel@tonic-gate 		}
5257c478bd9Sstevel@tonic-gate 		ret_buf = alloc_buf;
5267c478bd9Sstevel@tonic-gate 	} else {
5277c478bd9Sstevel@tonic-gate 		uchar_t	oversize_bits = 0;
5287c478bd9Sstevel@tonic-gate 		size_t	head_sz, data_sz, tail_sz;
5297c478bd9Sstevel@tonic-gate 		uintptr_t ret_addr, taddr, shift, tshift;
5308f2954faSraf 		oversize_t *orig, *tail, *big;
5317c478bd9Sstevel@tonic-gate 		size_t tsize;
5327c478bd9Sstevel@tonic-gate 
5337c478bd9Sstevel@tonic-gate 		/* needs to be aligned */
5347c478bd9Sstevel@tonic-gate 		shift = alignment - offset;
5357c478bd9Sstevel@tonic-gate 
5367c478bd9Sstevel@tonic-gate 		assert(shift >= MTMALLOC_MIN_ALIGN);
5377c478bd9Sstevel@tonic-gate 
5387c478bd9Sstevel@tonic-gate 		ret_addr = ((uintptr_t)alloc_buf + shift);
5397c478bd9Sstevel@tonic-gate 		ret_buf = (void *)ret_addr;
5407c478bd9Sstevel@tonic-gate 
5417c478bd9Sstevel@tonic-gate 		if (alloc_size <= MAX_CACHED) {
5427c478bd9Sstevel@tonic-gate 			MEMALIGN_HEADER_ALLOC(ret_addr, shift, alloc_buf);
5437c478bd9Sstevel@tonic-gate 			return (ret_buf);
5447c478bd9Sstevel@tonic-gate 		}
5457c478bd9Sstevel@tonic-gate 
5467c478bd9Sstevel@tonic-gate 		/*
5477c478bd9Sstevel@tonic-gate 		 * Only check for the fragments when the memory is allocted
5487c478bd9Sstevel@tonic-gate 		 * from oversize_list.  Split off a fragment and return it
5497c478bd9Sstevel@tonic-gate 		 * to the oversize freelist when it's > MAX_CACHED.
5507c478bd9Sstevel@tonic-gate 		 */
5517c478bd9Sstevel@tonic-gate 
5527c478bd9Sstevel@tonic-gate 		head_sz = shift - MAX(MEMALIGN_HEADER_SIZE, OVSZ_HEADER_SIZE);
5537c478bd9Sstevel@tonic-gate 
5547c478bd9Sstevel@tonic-gate 		tail_sz = alloc_size -
555db3659e5SSurya Prakki 		    (shift + size + MTMALLOC_MIN_ALIGN + OVSZ_HEADER_SIZE);
5567c478bd9Sstevel@tonic-gate 
5577c478bd9Sstevel@tonic-gate 		oversize_bits |= IS_OVERSIZE(head_sz, alloc_size) |
558db3659e5SSurya Prakki 		    IS_OVERSIZE(size, alloc_size) << DATA_SHIFT |
559db3659e5SSurya Prakki 		    IS_OVERSIZE(tail_sz, alloc_size) << TAIL_SHIFT;
5607c478bd9Sstevel@tonic-gate 
5617c478bd9Sstevel@tonic-gate 		switch (oversize_bits) {
5627c478bd9Sstevel@tonic-gate 			case NONE_OVERSIZE:
5637c478bd9Sstevel@tonic-gate 			case DATA_OVERSIZE:
5647c478bd9Sstevel@tonic-gate 				MEMALIGN_HEADER_ALLOC(ret_addr, shift,
565db3659e5SSurya Prakki 				    alloc_buf);
5667c478bd9Sstevel@tonic-gate 				break;
5677c478bd9Sstevel@tonic-gate 			case HEAD_OVERSIZE:
5687c478bd9Sstevel@tonic-gate 				/*
5697c478bd9Sstevel@tonic-gate 				 * If we can extend data > MAX_CACHED and have
5707c478bd9Sstevel@tonic-gate 				 * head still > MAX_CACHED, we split head-end
5717c478bd9Sstevel@tonic-gate 				 * as the case of head-end and data oversized,
5727c478bd9Sstevel@tonic-gate 				 * otherwise just create memalign header.
5737c478bd9Sstevel@tonic-gate 				 */
5747c478bd9Sstevel@tonic-gate 				tsize = (shift + size) - (MAX_CACHED + 8 +
575db3659e5SSurya Prakki 				    MTMALLOC_MIN_ALIGN + OVSZ_HEADER_SIZE);
5767c478bd9Sstevel@tonic-gate 
5777c478bd9Sstevel@tonic-gate 				if (!IS_OVERSIZE(tsize, alloc_size)) {
5787c478bd9Sstevel@tonic-gate 					MEMALIGN_HEADER_ALLOC(ret_addr, shift,
579db3659e5SSurya Prakki 					    alloc_buf);
5807c478bd9Sstevel@tonic-gate 					break;
5817c478bd9Sstevel@tonic-gate 				} else {
5827c478bd9Sstevel@tonic-gate 					tsize += OVSZ_HEADER_SIZE;
5837c478bd9Sstevel@tonic-gate 					taddr = ALIGN((uintptr_t)alloc_buf +
584db3659e5SSurya Prakki 					    tsize, MTMALLOC_MIN_ALIGN);
5857c478bd9Sstevel@tonic-gate 					tshift = ret_addr - taddr;
5867c478bd9Sstevel@tonic-gate 					MEMALIGN_HEADER_ALLOC(ret_addr, tshift,
587db3659e5SSurya Prakki 					    taddr);
5887c478bd9Sstevel@tonic-gate 					ret_addr = taddr;
5897c478bd9Sstevel@tonic-gate 					shift = ret_addr - (uintptr_t)alloc_buf;
5907c478bd9Sstevel@tonic-gate 				}
5917c478bd9Sstevel@tonic-gate 				/* FALLTHROUGH */
5927c478bd9Sstevel@tonic-gate 			case HEAD_AND_DATA_OVERSIZE:
5937c478bd9Sstevel@tonic-gate 				/*
5947c478bd9Sstevel@tonic-gate 				 * Split off the head fragment and
5957c478bd9Sstevel@tonic-gate 				 * return it back to oversize freelist.
5967c478bd9Sstevel@tonic-gate 				 * Create oversize header for the piece
5977c478bd9Sstevel@tonic-gate 				 * of (data + tail fragment).
5987c478bd9Sstevel@tonic-gate 				 */
5997c478bd9Sstevel@tonic-gate 				orig = (oversize_t *)((uintptr_t)alloc_buf -
600db3659e5SSurya Prakki 				    OVSZ_HEADER_SIZE);
6018f2954faSraf 				big = oversize_header_alloc(ret_addr -
602db3659e5SSurya Prakki 				    OVSZ_HEADER_SIZE, (orig->size - shift));
6038f2954faSraf 				(void) mutex_lock(&oversize_lock);
6048f2954faSraf 				insert_hash(big);
6058f2954faSraf 				(void) mutex_unlock(&oversize_lock);
6067c478bd9Sstevel@tonic-gate 				orig->size = shift - OVSZ_HEADER_SIZE;
6077c478bd9Sstevel@tonic-gate 
6087c478bd9Sstevel@tonic-gate 				/* free up the head fragment */
6097c478bd9Sstevel@tonic-gate 				free_oversize(orig);
6107c478bd9Sstevel@tonic-gate 				break;
6117c478bd9Sstevel@tonic-gate 			case TAIL_OVERSIZE:
6127c478bd9Sstevel@tonic-gate 				/*
6137c478bd9Sstevel@tonic-gate 				 * If we can extend data > MAX_CACHED and have
6147c478bd9Sstevel@tonic-gate 				 * tail-end still > MAX_CACHED, we split tail
6157c478bd9Sstevel@tonic-gate 				 * end, otherwise just create memalign header.
6167c478bd9Sstevel@tonic-gate 				 */
6177c478bd9Sstevel@tonic-gate 				orig = (oversize_t *)((uintptr_t)alloc_buf -
618db3659e5SSurya Prakki 				    OVSZ_HEADER_SIZE);
6197c478bd9Sstevel@tonic-gate 				tsize =  orig->size - (MAX_CACHED + 8 +
620db3659e5SSurya Prakki 				    shift + OVSZ_HEADER_SIZE +
621db3659e5SSurya Prakki 				    MTMALLOC_MIN_ALIGN);
6227c478bd9Sstevel@tonic-gate 				if (!IS_OVERSIZE(tsize, alloc_size)) {
6237c478bd9Sstevel@tonic-gate 					MEMALIGN_HEADER_ALLOC(ret_addr, shift,
624db3659e5SSurya Prakki 					    alloc_buf);
6257c478bd9Sstevel@tonic-gate 					break;
6267c478bd9Sstevel@tonic-gate 				} else {
6277c478bd9Sstevel@tonic-gate 					size = MAX_CACHED + 8;
6287c478bd9Sstevel@tonic-gate 				}
6297c478bd9Sstevel@tonic-gate 				/* FALLTHROUGH */
6307c478bd9Sstevel@tonic-gate 			case DATA_AND_TAIL_OVERSIZE:
6317c478bd9Sstevel@tonic-gate 				/*
6327c478bd9Sstevel@tonic-gate 				 * Split off the tail fragment and
6337c478bd9Sstevel@tonic-gate 				 * return it back to oversize freelist.
6347c478bd9Sstevel@tonic-gate 				 * Create memalign header and adjust
6357c478bd9Sstevel@tonic-gate 				 * the size for the piece of
6367c478bd9Sstevel@tonic-gate 				 * (head fragment + data).
6377c478bd9Sstevel@tonic-gate 				 */
6387c478bd9Sstevel@tonic-gate 				taddr = ALIGN(ret_addr + size,
639db3659e5SSurya Prakki 				    MTMALLOC_MIN_ALIGN);
6407c478bd9Sstevel@tonic-gate 				data_sz = (size_t)(taddr -
641db3659e5SSurya Prakki 				    (uintptr_t)alloc_buf);
6427c478bd9Sstevel@tonic-gate 				orig = (oversize_t *)((uintptr_t)alloc_buf -
643db3659e5SSurya Prakki 				    OVSZ_HEADER_SIZE);
6447c478bd9Sstevel@tonic-gate 				tsize = orig->size - data_sz;
6457c478bd9Sstevel@tonic-gate 				orig->size = data_sz;
6467c478bd9Sstevel@tonic-gate 				MEMALIGN_HEADER_ALLOC(ret_buf, shift,
647db3659e5SSurya Prakki 				    alloc_buf);
6487c478bd9Sstevel@tonic-gate 				tsize -= OVSZ_HEADER_SIZE;
6497c478bd9Sstevel@tonic-gate 				tail = oversize_header_alloc(taddr,  tsize);
6507c478bd9Sstevel@tonic-gate 				free_oversize(tail);
6517c478bd9Sstevel@tonic-gate 				break;
6527c478bd9Sstevel@tonic-gate 			case HEAD_AND_TAIL_OVERSIZE:
6537c478bd9Sstevel@tonic-gate 				/*
6547c478bd9Sstevel@tonic-gate 				 * Split off the head fragment.
6557c478bd9Sstevel@tonic-gate 				 * We try to free up tail-end when we can
6567c478bd9Sstevel@tonic-gate 				 * extend data size to (MAX_CACHED + 8)
6577c478bd9Sstevel@tonic-gate 				 * and remain tail-end oversized.
6587c478bd9Sstevel@tonic-gate 				 * The bottom line is all split pieces
6597c478bd9Sstevel@tonic-gate 				 * should be oversize in size.
6607c478bd9Sstevel@tonic-gate 				 */
6617c478bd9Sstevel@tonic-gate 				orig = (oversize_t *)((uintptr_t)alloc_buf -
662db3659e5SSurya Prakki 				    OVSZ_HEADER_SIZE);
6637c478bd9Sstevel@tonic-gate 				tsize =  orig->size - (MAX_CACHED + 8 +
664db3659e5SSurya Prakki 				    OVSZ_HEADER_SIZE + shift +
665db3659e5SSurya Prakki 				    MTMALLOC_MIN_ALIGN);
6667c478bd9Sstevel@tonic-gate 
6677c478bd9Sstevel@tonic-gate 				if (!IS_OVERSIZE(tsize, alloc_size)) {
6687c478bd9Sstevel@tonic-gate 					/*
6697c478bd9Sstevel@tonic-gate 					 * If the chunk is not big enough
6707c478bd9Sstevel@tonic-gate 					 * to make both data and tail oversize
6717c478bd9Sstevel@tonic-gate 					 * we just keep them as one piece.
6727c478bd9Sstevel@tonic-gate 					 */
6738f2954faSraf 					big = oversize_header_alloc(ret_addr -
674db3659e5SSurya Prakki 					    OVSZ_HEADER_SIZE,
675db3659e5SSurya Prakki 					    orig->size - shift);
6768f2954faSraf 					(void) mutex_lock(&oversize_lock);
6778f2954faSraf 					insert_hash(big);
6788f2954faSraf 					(void) mutex_unlock(&oversize_lock);
679db3659e5SSurya Prakki 					orig->size = shift - OVSZ_HEADER_SIZE;
6807c478bd9Sstevel@tonic-gate 					free_oversize(orig);
6817c478bd9Sstevel@tonic-gate 					break;
6827c478bd9Sstevel@tonic-gate 				} else {
6837c478bd9Sstevel@tonic-gate 					/*
6847c478bd9Sstevel@tonic-gate 					 * extend data size > MAX_CACHED
6857c478bd9Sstevel@tonic-gate 					 * and handle it as head, data, tail
6867c478bd9Sstevel@tonic-gate 					 * are all oversized.
6877c478bd9Sstevel@tonic-gate 					 */
6887c478bd9Sstevel@tonic-gate 					size = MAX_CACHED + 8;
6897c478bd9Sstevel@tonic-gate 				}
6907c478bd9Sstevel@tonic-gate 				/* FALLTHROUGH */
6917c478bd9Sstevel@tonic-gate 			case ALL_OVERSIZE:
6927c478bd9Sstevel@tonic-gate 				/*
6937c478bd9Sstevel@tonic-gate 				 * split off the head and tail fragments,
6947c478bd9Sstevel@tonic-gate 				 * return them back to the oversize freelist.
6957c478bd9Sstevel@tonic-gate 				 * Alloc oversize header for data seg.
6967c478bd9Sstevel@tonic-gate 				 */
6977c478bd9Sstevel@tonic-gate 				orig = (oversize_t *)((uintptr_t)alloc_buf -
698db3659e5SSurya Prakki 				    OVSZ_HEADER_SIZE);
6997c478bd9Sstevel@tonic-gate 				tsize = orig->size;
7007c478bd9Sstevel@tonic-gate 				orig->size = shift - OVSZ_HEADER_SIZE;
7017c478bd9Sstevel@tonic-gate 				free_oversize(orig);
7027c478bd9Sstevel@tonic-gate 
7037c478bd9Sstevel@tonic-gate 				taddr = ALIGN(ret_addr + size,
704db3659e5SSurya Prakki 				    MTMALLOC_MIN_ALIGN);
7057c478bd9Sstevel@tonic-gate 				data_sz = taddr - ret_addr;
7067c478bd9Sstevel@tonic-gate 				assert(tsize > (shift + data_sz +
707db3659e5SSurya Prakki 				    OVSZ_HEADER_SIZE));
7087c478bd9Sstevel@tonic-gate 				tail_sz = tsize -
709db3659e5SSurya Prakki 				    (shift + data_sz + OVSZ_HEADER_SIZE);
7107c478bd9Sstevel@tonic-gate 
7117c478bd9Sstevel@tonic-gate 				/* create oversize header for data seg */
7128f2954faSraf 				big = oversize_header_alloc(ret_addr -
713db3659e5SSurya Prakki 				    OVSZ_HEADER_SIZE, data_sz);
7148f2954faSraf 				(void) mutex_lock(&oversize_lock);
7158f2954faSraf 				insert_hash(big);
7168f2954faSraf 				(void) mutex_unlock(&oversize_lock);
7177c478bd9Sstevel@tonic-gate 
7187c478bd9Sstevel@tonic-gate 				/* create oversize header for tail fragment */
7197c478bd9Sstevel@tonic-gate 				tail = oversize_header_alloc(taddr, tail_sz);
7207c478bd9Sstevel@tonic-gate 				free_oversize(tail);
7217c478bd9Sstevel@tonic-gate 				break;
7227c478bd9Sstevel@tonic-gate 			default:
7237c478bd9Sstevel@tonic-gate 				/* should not reach here */
7247c478bd9Sstevel@tonic-gate 				assert(0);
7257c478bd9Sstevel@tonic-gate 		}
7267c478bd9Sstevel@tonic-gate 	}
7277c478bd9Sstevel@tonic-gate 	return (ret_buf);
7287c478bd9Sstevel@tonic-gate }
7297c478bd9Sstevel@tonic-gate 
7307c478bd9Sstevel@tonic-gate 
7317c478bd9Sstevel@tonic-gate void *
valloc(size_t size)7327c478bd9Sstevel@tonic-gate valloc(size_t size)
7337c478bd9Sstevel@tonic-gate {
7347c478bd9Sstevel@tonic-gate 	static unsigned pagesize;
7357c478bd9Sstevel@tonic-gate 
7367c478bd9Sstevel@tonic-gate 	if (size == 0)
7377c478bd9Sstevel@tonic-gate 		return (NULL);
7387c478bd9Sstevel@tonic-gate 
7397c478bd9Sstevel@tonic-gate 	if (!pagesize)
7407c478bd9Sstevel@tonic-gate 		pagesize = sysconf(_SC_PAGESIZE);
7417c478bd9Sstevel@tonic-gate 
7427c478bd9Sstevel@tonic-gate 	return (memalign(pagesize, size));
7437c478bd9Sstevel@tonic-gate }
7447c478bd9Sstevel@tonic-gate 
7457c478bd9Sstevel@tonic-gate void
mallocctl(int cmd,long value)7467c478bd9Sstevel@tonic-gate mallocctl(int cmd, long value)
7477c478bd9Sstevel@tonic-gate {
7487c478bd9Sstevel@tonic-gate 	switch (cmd) {
7497c478bd9Sstevel@tonic-gate 
7507c478bd9Sstevel@tonic-gate 	case MTDEBUGPATTERN:
7517c478bd9Sstevel@tonic-gate 		/*
7527c478bd9Sstevel@tonic-gate 		 * Reinitialize free blocks in case malloc() is called prior
7537c478bd9Sstevel@tonic-gate 		 * to mallocctl().
7547c478bd9Sstevel@tonic-gate 		 */
7557c478bd9Sstevel@tonic-gate 		if (value && !(debugopt & cmd)) {
7567c478bd9Sstevel@tonic-gate 			reinit++;
7577c478bd9Sstevel@tonic-gate 			debugopt |= cmd;
7587c478bd9Sstevel@tonic-gate 			reinit_cpu_list();
7597c478bd9Sstevel@tonic-gate 		}
7607c478bd9Sstevel@tonic-gate 		/*FALLTHRU*/
7617c478bd9Sstevel@tonic-gate 	case MTDOUBLEFREE:
7627c478bd9Sstevel@tonic-gate 	case MTINITBUFFER:
7637c478bd9Sstevel@tonic-gate 		if (value)
7647c478bd9Sstevel@tonic-gate 			debugopt |= cmd;
7657c478bd9Sstevel@tonic-gate 		else
7667c478bd9Sstevel@tonic-gate 			debugopt &= ~cmd;
7677c478bd9Sstevel@tonic-gate 		break;
7687c478bd9Sstevel@tonic-gate 	case MTCHUNKSIZE:
7697c478bd9Sstevel@tonic-gate 		if (value >= MINSIZE && value <= MAXSIZE)
7707c478bd9Sstevel@tonic-gate 			requestsize = value;
7717c478bd9Sstevel@tonic-gate 		break;
7727c478bd9Sstevel@tonic-gate 	default:
7737c478bd9Sstevel@tonic-gate 		break;
7747c478bd9Sstevel@tonic-gate 	}
7757c478bd9Sstevel@tonic-gate }
7767c478bd9Sstevel@tonic-gate 
7777c478bd9Sstevel@tonic-gate /*
7781d530678Sraf  * Initialization function, called from the init section of the library.
7791d530678Sraf  * No locking is required here because we are single-threaded during
7801d530678Sraf  * library initialization.
7817c478bd9Sstevel@tonic-gate  */
7821d530678Sraf static void
setup_caches(void)7837c478bd9Sstevel@tonic-gate setup_caches(void)
7847c478bd9Sstevel@tonic-gate {
7857c478bd9Sstevel@tonic-gate 	uintptr_t oldbrk;
7867c478bd9Sstevel@tonic-gate 	uintptr_t newbrk;
7877c478bd9Sstevel@tonic-gate 
7887c478bd9Sstevel@tonic-gate 	size_t cache_space_needed;
7897c478bd9Sstevel@tonic-gate 	size_t padding;
7907c478bd9Sstevel@tonic-gate 
7917c478bd9Sstevel@tonic-gate 	curcpu_func new_curcpu;
7927c478bd9Sstevel@tonic-gate 	uint_t new_cpu_mask;
7937c478bd9Sstevel@tonic-gate 	percpu_t *new_cpu_list;
7947c478bd9Sstevel@tonic-gate 
7957c478bd9Sstevel@tonic-gate 	uint_t i, j;
7967c478bd9Sstevel@tonic-gate 	uintptr_t list_addr;
7977c478bd9Sstevel@tonic-gate 
7981d530678Sraf 	/*
7991d530678Sraf 	 * Get a decent "current cpu identifier", to be used to reduce
8001d530678Sraf 	 * contention.  Eventually, this should be replaced by an interface
8011d530678Sraf 	 * to get the actual CPU sequence number in libthread/liblwp.
8021d530678Sraf 	 */
8031d530678Sraf 	new_curcpu = (curcpu_func)thr_self;
8041d530678Sraf 	if ((ncpus = 2 * sysconf(_SC_NPROCESSORS_CONF)) <= 0)
8051d530678Sraf 		ncpus = 4; /* decent default value */
8067c478bd9Sstevel@tonic-gate 
8077c478bd9Sstevel@tonic-gate 	/* round ncpus up to a power of 2 */
8087c478bd9Sstevel@tonic-gate 	while (ncpus & (ncpus - 1))
8097c478bd9Sstevel@tonic-gate 		ncpus++;
8107c478bd9Sstevel@tonic-gate 
8117c478bd9Sstevel@tonic-gate 	new_cpu_mask = ncpus - 1;	/* create the cpu mask */
8127c478bd9Sstevel@tonic-gate 
8137c478bd9Sstevel@tonic-gate 	/*
8147c478bd9Sstevel@tonic-gate 	 * We now do some magic with the brk.  What we want to get in the
8157c478bd9Sstevel@tonic-gate 	 * end is a bunch of well-aligned stuff in a big initial allocation.
8167c478bd9Sstevel@tonic-gate 	 * Along the way, we do sanity checks to make sure no one else has
8177c478bd9Sstevel@tonic-gate 	 * touched the brk (which shouldn't happen, but it's always good to
8187c478bd9Sstevel@tonic-gate 	 * check)
8197c478bd9Sstevel@tonic-gate 	 *
8207c478bd9Sstevel@tonic-gate 	 * First, make sure sbrk is sane, and store the current brk in oldbrk.
8217c478bd9Sstevel@tonic-gate 	 */
8227c478bd9Sstevel@tonic-gate 	oldbrk = (uintptr_t)sbrk(0);
8231d530678Sraf 	if ((void *)oldbrk == (void *)-1)
8241d530678Sraf 		abort();	/* sbrk is broken -- we're doomed. */
8257c478bd9Sstevel@tonic-gate 
8267c478bd9Sstevel@tonic-gate 	/*
8277c478bd9Sstevel@tonic-gate 	 * Now, align the brk to a multiple of CACHE_COHERENCY_UNIT, so that
8287c478bd9Sstevel@tonic-gate 	 * the percpu structures and cache lists will be properly aligned.
8297c478bd9Sstevel@tonic-gate 	 *
8307c478bd9Sstevel@tonic-gate 	 *   2.  All hunks will be page-aligned, assuming HUNKSIZE >= PAGESIZE,
8317c478bd9Sstevel@tonic-gate 	 *	so they can be paged out individually.
8327c478bd9Sstevel@tonic-gate 	 */
8337c478bd9Sstevel@tonic-gate 	newbrk = ALIGN(oldbrk, CACHE_COHERENCY_UNIT);
8341d530678Sraf 	if (newbrk != oldbrk && (uintptr_t)sbrk(newbrk - oldbrk) != oldbrk)
8351d530678Sraf 		abort();	/* sbrk is broken -- we're doomed. */
8367c478bd9Sstevel@tonic-gate 
8377c478bd9Sstevel@tonic-gate 	/*
8387c478bd9Sstevel@tonic-gate 	 * For each cpu, there is one percpu_t and a list of caches
8397c478bd9Sstevel@tonic-gate 	 */
8407c478bd9Sstevel@tonic-gate 	cache_space_needed = ncpus * (sizeof (percpu_t) + CACHELIST_SIZE);
8417c478bd9Sstevel@tonic-gate 
8427c478bd9Sstevel@tonic-gate 	new_cpu_list = (percpu_t *)sbrk(cache_space_needed);
8437c478bd9Sstevel@tonic-gate 
8447c478bd9Sstevel@tonic-gate 	if (new_cpu_list == (percpu_t *)-1 ||
8451d530678Sraf 	    (uintptr_t)new_cpu_list != newbrk)
8461d530678Sraf 		abort();	/* sbrk is broken -- we're doomed. */
8477c478bd9Sstevel@tonic-gate 
8487c478bd9Sstevel@tonic-gate 	/*
8497c478bd9Sstevel@tonic-gate 	 * Finally, align the brk to HUNKSIZE so that all hunks are
8507c478bd9Sstevel@tonic-gate 	 * page-aligned, to avoid edge-effects.
8517c478bd9Sstevel@tonic-gate 	 */
8527c478bd9Sstevel@tonic-gate 
8537c478bd9Sstevel@tonic-gate 	newbrk = (uintptr_t)new_cpu_list + cache_space_needed;
8547c478bd9Sstevel@tonic-gate 
8557c478bd9Sstevel@tonic-gate 	padding = ALIGN(newbrk, HUNKSIZE) - newbrk;
8567c478bd9Sstevel@tonic-gate 
8571d530678Sraf 	if (padding > 0 && (uintptr_t)sbrk(padding) != newbrk)
8581d530678Sraf 		abort();	/* sbrk is broken -- we're doomed. */
8597c478bd9Sstevel@tonic-gate 
8607c478bd9Sstevel@tonic-gate 	list_addr = ((uintptr_t)new_cpu_list + (sizeof (percpu_t) * ncpus));
8617c478bd9Sstevel@tonic-gate 
8627c478bd9Sstevel@tonic-gate 	/* initialize the percpu list */
8637c478bd9Sstevel@tonic-gate 	for (i = 0; i < ncpus; i++) {
8647c478bd9Sstevel@tonic-gate 		new_cpu_list[i].mt_caches = (cache_head_t *)list_addr;
8657c478bd9Sstevel@tonic-gate 		for (j = 0; j < NUM_CACHES; j++) {
8667c478bd9Sstevel@tonic-gate 			new_cpu_list[i].mt_caches[j].mt_cache = NULL;
8677c478bd9Sstevel@tonic-gate 			new_cpu_list[i].mt_caches[j].mt_hint = NULL;
8687c478bd9Sstevel@tonic-gate 		}
8697c478bd9Sstevel@tonic-gate 
8701d530678Sraf 		(void) mutex_init(&new_cpu_list[i].mt_parent_lock,
8711d530678Sraf 		    USYNC_THREAD, NULL);
8727c478bd9Sstevel@tonic-gate 
8737c478bd9Sstevel@tonic-gate 		/* get the correct cache list alignment */
8747c478bd9Sstevel@tonic-gate 		list_addr += CACHELIST_SIZE;
8757c478bd9Sstevel@tonic-gate 	}
8767c478bd9Sstevel@tonic-gate 
8777c478bd9Sstevel@tonic-gate 	/*
8787c478bd9Sstevel@tonic-gate 	 * Initialize oversize listhead
8797c478bd9Sstevel@tonic-gate 	 */
8807c478bd9Sstevel@tonic-gate 	oversize_list.next_bysize = &oversize_list;
8817c478bd9Sstevel@tonic-gate 	oversize_list.prev_bysize = &oversize_list;
8827c478bd9Sstevel@tonic-gate 	oversize_list.next_byaddr = &oversize_list;
8837c478bd9Sstevel@tonic-gate 	oversize_list.prev_byaddr = &oversize_list;
8847c478bd9Sstevel@tonic-gate 	oversize_list.addr = NULL;
8857c478bd9Sstevel@tonic-gate 	oversize_list.size = 0;		/* sentinal */
8867c478bd9Sstevel@tonic-gate 
8877c478bd9Sstevel@tonic-gate 	/*
8881d530678Sraf 	 * Now install the global variables.
8897c478bd9Sstevel@tonic-gate 	 */
8907c478bd9Sstevel@tonic-gate 	curcpu = new_curcpu;
8917c478bd9Sstevel@tonic-gate 	cpu_mask = new_cpu_mask;
8927c478bd9Sstevel@tonic-gate 	cpu_list = new_cpu_list;
8937c478bd9Sstevel@tonic-gate }
8947c478bd9Sstevel@tonic-gate 
8957c478bd9Sstevel@tonic-gate static void
create_cache(cache_t * cp,size_t size,uint_t chunksize)8967c478bd9Sstevel@tonic-gate create_cache(cache_t *cp, size_t size, uint_t chunksize)
8977c478bd9Sstevel@tonic-gate {
8987c478bd9Sstevel@tonic-gate 	long nblocks;
8997c478bd9Sstevel@tonic-gate 
9001d530678Sraf 	(void) mutex_init(&cp->mt_cache_lock, USYNC_THREAD, NULL);
9017c478bd9Sstevel@tonic-gate 	cp->mt_size = size;
9027c478bd9Sstevel@tonic-gate 	cp->mt_freelist = ((caddr_t)cp + sizeof (cache_t));
9037c478bd9Sstevel@tonic-gate 	cp->mt_span = chunksize * HUNKSIZE - sizeof (cache_t);
9047c478bd9Sstevel@tonic-gate 	cp->mt_hunks = chunksize;
9057c478bd9Sstevel@tonic-gate 	/*
9067c478bd9Sstevel@tonic-gate 	 * rough calculation. We will need to adjust later.
9077c478bd9Sstevel@tonic-gate 	 */
9087c478bd9Sstevel@tonic-gate 	nblocks = cp->mt_span / cp->mt_size;
9097c478bd9Sstevel@tonic-gate 	nblocks >>= 3;
9107c478bd9Sstevel@tonic-gate 	if (nblocks == 0) { /* less than 8 free blocks in this pool */
9117c478bd9Sstevel@tonic-gate 		int32_t numblocks = 0;
9127c478bd9Sstevel@tonic-gate 		long i = cp->mt_span;
9137c478bd9Sstevel@tonic-gate 		size_t sub = cp->mt_size;
9147c478bd9Sstevel@tonic-gate 		uchar_t mask = 0;
9157c478bd9Sstevel@tonic-gate 
9167c478bd9Sstevel@tonic-gate 		while (i > sub) {
9177c478bd9Sstevel@tonic-gate 			numblocks++;
9187c478bd9Sstevel@tonic-gate 			i -= sub;
9197c478bd9Sstevel@tonic-gate 		}
9207c478bd9Sstevel@tonic-gate 		nblocks = numblocks;
9217c478bd9Sstevel@tonic-gate 		cp->mt_arena = (caddr_t)ALIGN(cp->mt_freelist + 8, 8);
9227c478bd9Sstevel@tonic-gate 		cp->mt_nfree = numblocks;
9237c478bd9Sstevel@tonic-gate 		while (numblocks--) {
9247c478bd9Sstevel@tonic-gate 			mask |= 0x80 >> numblocks;
9257c478bd9Sstevel@tonic-gate 		}
9267c478bd9Sstevel@tonic-gate 		*(cp->mt_freelist) = mask;
9277c478bd9Sstevel@tonic-gate 	} else {
9287c478bd9Sstevel@tonic-gate 		cp->mt_arena = (caddr_t)ALIGN((caddr_t)cp->mt_freelist +
929db3659e5SSurya Prakki 		    nblocks, 32);
9307c478bd9Sstevel@tonic-gate 		/* recompute nblocks */
9317c478bd9Sstevel@tonic-gate 		nblocks = (uintptr_t)((caddr_t)cp->mt_freelist +
932db3659e5SSurya Prakki 		    cp->mt_span - cp->mt_arena) / cp->mt_size;
9337c478bd9Sstevel@tonic-gate 		cp->mt_nfree = ((nblocks >> 3) << 3);
9347c478bd9Sstevel@tonic-gate 		/* Set everything to free */
9357c478bd9Sstevel@tonic-gate 		(void) memset(cp->mt_freelist, 0xff, nblocks >> 3);
9367c478bd9Sstevel@tonic-gate 	}
9377c478bd9Sstevel@tonic-gate 
9387c478bd9Sstevel@tonic-gate 	if (debugopt & MTDEBUGPATTERN)
9397c478bd9Sstevel@tonic-gate 		copy_pattern(FREEPATTERN, cp->mt_arena, cp->mt_size * nblocks);
9407c478bd9Sstevel@tonic-gate 
9417c478bd9Sstevel@tonic-gate 	cp->mt_next = NULL;
9427c478bd9Sstevel@tonic-gate }
9437c478bd9Sstevel@tonic-gate 
9447c478bd9Sstevel@tonic-gate static void
reinit_cpu_list(void)9457c478bd9Sstevel@tonic-gate reinit_cpu_list(void)
9467c478bd9Sstevel@tonic-gate {
9477c478bd9Sstevel@tonic-gate 	oversize_t *wp = oversize_list.next_bysize;
9487c478bd9Sstevel@tonic-gate 	percpu_t *cpuptr;
9497c478bd9Sstevel@tonic-gate 	cache_t *thiscache;
9507c478bd9Sstevel@tonic-gate 	cache_head_t *cachehead;
9517c478bd9Sstevel@tonic-gate 
9527c478bd9Sstevel@tonic-gate 	/* Reinitialize free oversize blocks. */
9537c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&oversize_lock);
9547c478bd9Sstevel@tonic-gate 	if (debugopt & MTDEBUGPATTERN)
9557c478bd9Sstevel@tonic-gate 		for (; wp != &oversize_list; wp = wp->next_bysize)
9567c478bd9Sstevel@tonic-gate 			copy_pattern(FREEPATTERN, wp->addr, wp->size);
9577c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&oversize_lock);
9587c478bd9Sstevel@tonic-gate 
9597c478bd9Sstevel@tonic-gate 	/* Reinitialize free blocks. */
9607c478bd9Sstevel@tonic-gate 	for (cpuptr = &cpu_list[0]; cpuptr < &cpu_list[ncpus]; cpuptr++) {
9617c478bd9Sstevel@tonic-gate 		(void) mutex_lock(&cpuptr->mt_parent_lock);
9627c478bd9Sstevel@tonic-gate 		for (cachehead = &cpuptr->mt_caches[0]; cachehead <
963db3659e5SSurya Prakki 		    &cpuptr->mt_caches[NUM_CACHES]; cachehead++) {
9647c478bd9Sstevel@tonic-gate 			for (thiscache = cachehead->mt_cache; thiscache != NULL;
965db3659e5SSurya Prakki 			    thiscache = thiscache->mt_next) {
9667c478bd9Sstevel@tonic-gate 				(void) mutex_lock(&thiscache->mt_cache_lock);
9677c478bd9Sstevel@tonic-gate 				if (thiscache->mt_nfree == 0) {
9687c478bd9Sstevel@tonic-gate 					(void) mutex_unlock(
9697c478bd9Sstevel@tonic-gate 					    &thiscache->mt_cache_lock);
9707c478bd9Sstevel@tonic-gate 					continue;
9717c478bd9Sstevel@tonic-gate 				}
9727c478bd9Sstevel@tonic-gate 				if (thiscache != NULL)
9737c478bd9Sstevel@tonic-gate 					reinit_cache(thiscache);
9747c478bd9Sstevel@tonic-gate 				(void) mutex_unlock(&thiscache->mt_cache_lock);
9757c478bd9Sstevel@tonic-gate 			}
9767c478bd9Sstevel@tonic-gate 		}
9777c478bd9Sstevel@tonic-gate 		(void) mutex_unlock(&cpuptr->mt_parent_lock);
9787c478bd9Sstevel@tonic-gate 	}
9797c478bd9Sstevel@tonic-gate 	reinit = 0;
9807c478bd9Sstevel@tonic-gate }
9817c478bd9Sstevel@tonic-gate 
9827c478bd9Sstevel@tonic-gate static void
reinit_cache(cache_t * thiscache)9837c478bd9Sstevel@tonic-gate reinit_cache(cache_t *thiscache)
9847c478bd9Sstevel@tonic-gate {
9857c478bd9Sstevel@tonic-gate 	uint32_t *freeblocks; /* not a uintptr_t on purpose */
9867c478bd9Sstevel@tonic-gate 	int32_t i, n;
9877c478bd9Sstevel@tonic-gate 	caddr_t ret;
9887c478bd9Sstevel@tonic-gate 
9897c478bd9Sstevel@tonic-gate 	freeblocks = (uint32_t *)thiscache->mt_freelist;
9907c478bd9Sstevel@tonic-gate 	while (freeblocks < (uint32_t *)thiscache->mt_arena) {
9917c478bd9Sstevel@tonic-gate 		if (*freeblocks & 0xffffffff) {
992db3659e5SSurya Prakki 			for (i = 0; i < 32; i++) {
993db3659e5SSurya Prakki 				if (FLIP_EM(*freeblocks) & (0x80000000 >> i)) {
994db3659e5SSurya Prakki 					n = (uintptr_t)(((freeblocks -
995db3659e5SSurya Prakki 					    (uint32_t *)thiscache->mt_freelist)
996db3659e5SSurya Prakki 					    << 5) + i) * thiscache->mt_size;
997db3659e5SSurya Prakki 					ret = thiscache->mt_arena + n;
998db3659e5SSurya Prakki 					ret += OVERHEAD;
999db3659e5SSurya Prakki 					copy_pattern(FREEPATTERN, ret,
1000db3659e5SSurya Prakki 					    thiscache->mt_size);
1001db3659e5SSurya Prakki 				}
10027c478bd9Sstevel@tonic-gate 			}
10037c478bd9Sstevel@tonic-gate 		}
10047c478bd9Sstevel@tonic-gate 		freeblocks++;
10057c478bd9Sstevel@tonic-gate 	}
10067c478bd9Sstevel@tonic-gate }
10077c478bd9Sstevel@tonic-gate 
10087c478bd9Sstevel@tonic-gate static void *
malloc_internal(size_t size,percpu_t * cpuptr)10097c478bd9Sstevel@tonic-gate malloc_internal(size_t size, percpu_t *cpuptr)
10107c478bd9Sstevel@tonic-gate {
10117c478bd9Sstevel@tonic-gate 	cache_head_t *cachehead;
10127c478bd9Sstevel@tonic-gate 	cache_t *thiscache, *hintcache;
10137c478bd9Sstevel@tonic-gate 	int32_t i, n, logsz, bucket;
10147c478bd9Sstevel@tonic-gate 	uint32_t index;
10157c478bd9Sstevel@tonic-gate 	uint32_t *freeblocks; /* not a uintptr_t on purpose */
10167c478bd9Sstevel@tonic-gate 	caddr_t ret;
10177c478bd9Sstevel@tonic-gate 
10187c478bd9Sstevel@tonic-gate 	logsz = MIN_CACHED_SHIFT;
10197c478bd9Sstevel@tonic-gate 
10207c478bd9Sstevel@tonic-gate 	while (size > (1 << logsz))
10217c478bd9Sstevel@tonic-gate 		logsz++;
10227c478bd9Sstevel@tonic-gate 
10237c478bd9Sstevel@tonic-gate 	bucket = logsz - MIN_CACHED_SHIFT;
10247c478bd9Sstevel@tonic-gate 
10257c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&cpuptr->mt_parent_lock);
10267c478bd9Sstevel@tonic-gate 
10277c478bd9Sstevel@tonic-gate 	/*
10287c478bd9Sstevel@tonic-gate 	 * Find a cache of the appropriate size with free buffers.
10297c478bd9Sstevel@tonic-gate 	 *
10307c478bd9Sstevel@tonic-gate 	 * We don't need to lock each cache as we check their mt_nfree count,
10317c478bd9Sstevel@tonic-gate 	 * since:
10327c478bd9Sstevel@tonic-gate 	 *	1.  We are only looking for caches with mt_nfree > 0.  If a
10337c478bd9Sstevel@tonic-gate 	 *	   free happens during our search, it will increment mt_nfree,
10347c478bd9Sstevel@tonic-gate 	 *	   which will not effect the test.
10357c478bd9Sstevel@tonic-gate 	 *	2.  Allocations can decrement mt_nfree, but they can't happen
10367c478bd9Sstevel@tonic-gate 	 *	   as long as we hold mt_parent_lock.
10377c478bd9Sstevel@tonic-gate 	 */
10387c478bd9Sstevel@tonic-gate 
10397c478bd9Sstevel@tonic-gate 	cachehead = &cpuptr->mt_caches[bucket];
10407c478bd9Sstevel@tonic-gate 
10417c478bd9Sstevel@tonic-gate 	/* Search through the list, starting at the mt_hint */
10427c478bd9Sstevel@tonic-gate 	thiscache = cachehead->mt_hint;
10437c478bd9Sstevel@tonic-gate 
10447c478bd9Sstevel@tonic-gate 	while (thiscache != NULL && thiscache->mt_nfree == 0)
10457c478bd9Sstevel@tonic-gate 		thiscache = thiscache->mt_next;
10467c478bd9Sstevel@tonic-gate 
10477c478bd9Sstevel@tonic-gate 	if (thiscache == NULL) {
10487c478bd9Sstevel@tonic-gate 		/* wrap around -- search up to the hint */
10497c478bd9Sstevel@tonic-gate 		thiscache = cachehead->mt_cache;
10507c478bd9Sstevel@tonic-gate 		hintcache = cachehead->mt_hint;
10517c478bd9Sstevel@tonic-gate 
10527c478bd9Sstevel@tonic-gate 		while (thiscache != NULL && thiscache != hintcache &&
10537c478bd9Sstevel@tonic-gate 		    thiscache->mt_nfree == 0)
10547c478bd9Sstevel@tonic-gate 			thiscache = thiscache->mt_next;
10557c478bd9Sstevel@tonic-gate 
10567c478bd9Sstevel@tonic-gate 		if (thiscache == hintcache)
10577c478bd9Sstevel@tonic-gate 			thiscache = NULL;
10587c478bd9Sstevel@tonic-gate 	}
10597c478bd9Sstevel@tonic-gate 
10607c478bd9Sstevel@tonic-gate 
10617c478bd9Sstevel@tonic-gate 	if (thiscache == NULL) { /* there are no free caches */
10627c478bd9Sstevel@tonic-gate 		int32_t thisrequest = requestsize;
10637c478bd9Sstevel@tonic-gate 		int32_t buffer_size = (1 << logsz) + OVERHEAD;
10647c478bd9Sstevel@tonic-gate 
10657c478bd9Sstevel@tonic-gate 		thiscache = (cache_t *)morecore(thisrequest * HUNKSIZE);
10667c478bd9Sstevel@tonic-gate 
10677c478bd9Sstevel@tonic-gate 		if (thiscache == (cache_t *)-1) {
1068db3659e5SSurya Prakki 			(void) mutex_unlock(&cpuptr->mt_parent_lock);
1069db3659e5SSurya Prakki 			errno = EAGAIN;
1070db3659e5SSurya Prakki 			return (NULL);
10717c478bd9Sstevel@tonic-gate 		}
10727c478bd9Sstevel@tonic-gate 		create_cache(thiscache, buffer_size, thisrequest);
10737c478bd9Sstevel@tonic-gate 
10747c478bd9Sstevel@tonic-gate 		/* link in the new block at the beginning of the list */
10757c478bd9Sstevel@tonic-gate 		thiscache->mt_next = cachehead->mt_cache;
10767c478bd9Sstevel@tonic-gate 		cachehead->mt_cache = thiscache;
10777c478bd9Sstevel@tonic-gate 	}
10787c478bd9Sstevel@tonic-gate 
10797c478bd9Sstevel@tonic-gate 	/* update the hint to the cache we found or created */
10807c478bd9Sstevel@tonic-gate 	cachehead->mt_hint = thiscache;
10817c478bd9Sstevel@tonic-gate 
10827c478bd9Sstevel@tonic-gate 	/* thiscache now points to a cache with available space */
10837c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&thiscache->mt_cache_lock);
10847c478bd9Sstevel@tonic-gate 
10857c478bd9Sstevel@tonic-gate 	freeblocks = (uint32_t *)thiscache->mt_freelist;
10867c478bd9Sstevel@tonic-gate 	while (freeblocks < (uint32_t *)thiscache->mt_arena) {
10877c478bd9Sstevel@tonic-gate 		if (*freeblocks & 0xffffffff)
10887c478bd9Sstevel@tonic-gate 			break;
10897c478bd9Sstevel@tonic-gate 		freeblocks++;
10907c478bd9Sstevel@tonic-gate 		if (freeblocks < (uint32_t *)thiscache->mt_arena &&
10917c478bd9Sstevel@tonic-gate 		    *freeblocks & 0xffffffff)
10927c478bd9Sstevel@tonic-gate 			break;
10937c478bd9Sstevel@tonic-gate 		freeblocks++;
10947c478bd9Sstevel@tonic-gate 		if (freeblocks < (uint32_t *)thiscache->mt_arena &&
10957c478bd9Sstevel@tonic-gate 		    *freeblocks & 0xffffffff)
10967c478bd9Sstevel@tonic-gate 			break;
10977c478bd9Sstevel@tonic-gate 		freeblocks++;
10987c478bd9Sstevel@tonic-gate 		if (freeblocks < (uint32_t *)thiscache->mt_arena &&
10997c478bd9Sstevel@tonic-gate 		    *freeblocks & 0xffffffff)
11007c478bd9Sstevel@tonic-gate 			break;
11017c478bd9Sstevel@tonic-gate 		freeblocks++;
11027c478bd9Sstevel@tonic-gate 	}
11037c478bd9Sstevel@tonic-gate 
11047c478bd9Sstevel@tonic-gate 	/*
11057c478bd9Sstevel@tonic-gate 	 * the offset from mt_freelist to freeblocks is the offset into
11067c478bd9Sstevel@tonic-gate 	 * the arena. Be sure to include the offset into freeblocks
11077c478bd9Sstevel@tonic-gate 	 * of the bitmask. n is the offset.
11087c478bd9Sstevel@tonic-gate 	 */
11097c478bd9Sstevel@tonic-gate 	for (i = 0; i < 32; ) {
11107c478bd9Sstevel@tonic-gate 		if (FLIP_EM(*freeblocks) & (0x80000000 >> i++))
11117c478bd9Sstevel@tonic-gate 			break;
11127c478bd9Sstevel@tonic-gate 		if (FLIP_EM(*freeblocks) & (0x80000000 >> i++))
11137c478bd9Sstevel@tonic-gate 			break;
11147c478bd9Sstevel@tonic-gate 		if (FLIP_EM(*freeblocks) & (0x80000000 >> i++))
11157c478bd9Sstevel@tonic-gate 			break;
11167c478bd9Sstevel@tonic-gate 		if (FLIP_EM(*freeblocks) & (0x80000000 >> i++))
11177c478bd9Sstevel@tonic-gate 			break;
11187c478bd9Sstevel@tonic-gate 	}
11197c478bd9Sstevel@tonic-gate 	index = 0x80000000 >> --i;
11207c478bd9Sstevel@tonic-gate 
11217c478bd9Sstevel@tonic-gate 
11227c478bd9Sstevel@tonic-gate 	*freeblocks &= FLIP_EM(~index);
11237c478bd9Sstevel@tonic-gate 
11247c478bd9Sstevel@tonic-gate 	thiscache->mt_nfree--;
11257c478bd9Sstevel@tonic-gate 
11267c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&thiscache->mt_cache_lock);
11277c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&cpuptr->mt_parent_lock);
11287c478bd9Sstevel@tonic-gate 
11297c478bd9Sstevel@tonic-gate 	n = (uintptr_t)(((freeblocks - (uint32_t *)thiscache->mt_freelist) << 5)
1130db3659e5SSurya Prakki 	    + i) * thiscache->mt_size;
11317c478bd9Sstevel@tonic-gate 	/*
11327c478bd9Sstevel@tonic-gate 	 * Now you have the offset in n, you've changed the free mask
11337c478bd9Sstevel@tonic-gate 	 * in the freelist. Nothing left to do but find the block
11347c478bd9Sstevel@tonic-gate 	 * in the arena and put the value of thiscache in the word
11357c478bd9Sstevel@tonic-gate 	 * ahead of the handed out address and return the memory
11367c478bd9Sstevel@tonic-gate 	 * back to the user.
11377c478bd9Sstevel@tonic-gate 	 */
11387c478bd9Sstevel@tonic-gate 	ret = thiscache->mt_arena + n;
11397c478bd9Sstevel@tonic-gate 
11407c478bd9Sstevel@tonic-gate 	/* Store the cache addr for this buf. Makes free go fast. */
11417c478bd9Sstevel@tonic-gate 	*(uintptr_t *)ret = (uintptr_t)thiscache;
11427c478bd9Sstevel@tonic-gate 
11437c478bd9Sstevel@tonic-gate 	/*
11447c478bd9Sstevel@tonic-gate 	 * This assert makes sure we don't hand out memory that is not
11457c478bd9Sstevel@tonic-gate 	 * owned by this cache.
11467c478bd9Sstevel@tonic-gate 	 */
11477c478bd9Sstevel@tonic-gate 	assert(ret + thiscache->mt_size <= thiscache->mt_freelist +
1148db3659e5SSurya Prakki 	    thiscache->mt_span);
11497c478bd9Sstevel@tonic-gate 
11507c478bd9Sstevel@tonic-gate 	ret += OVERHEAD;
11517c478bd9Sstevel@tonic-gate 
11527c478bd9Sstevel@tonic-gate 	assert(((uintptr_t)ret & 7) == 0); /* are we 8 byte aligned */
11537c478bd9Sstevel@tonic-gate 
11547c478bd9Sstevel@tonic-gate 	if (reinit == 0 && (debugopt & MTDEBUGPATTERN))
11557c478bd9Sstevel@tonic-gate 		if (verify_pattern(FREEPATTERN, ret, size))
11567c478bd9Sstevel@tonic-gate 			abort();	/* reference after free */
11577c478bd9Sstevel@tonic-gate 
11587c478bd9Sstevel@tonic-gate 	if (debugopt & MTINITBUFFER)
11597c478bd9Sstevel@tonic-gate 		copy_pattern(INITPATTERN, ret, size);
11607c478bd9Sstevel@tonic-gate 	return ((void *)ret);
11617c478bd9Sstevel@tonic-gate }
11627c478bd9Sstevel@tonic-gate 
11637c478bd9Sstevel@tonic-gate static void *
morecore(size_t bytes)11647c478bd9Sstevel@tonic-gate morecore(size_t bytes)
11657c478bd9Sstevel@tonic-gate {
11667c478bd9Sstevel@tonic-gate 	void * ret;
11677c478bd9Sstevel@tonic-gate 
11687c478bd9Sstevel@tonic-gate 	if (bytes > LONG_MAX) {
11697c478bd9Sstevel@tonic-gate 		intptr_t wad;
11707c478bd9Sstevel@tonic-gate 		/*
11717c478bd9Sstevel@tonic-gate 		 * The request size is too big. We need to do this in
11727c478bd9Sstevel@tonic-gate 		 * chunks. Sbrk only takes an int for an arg.
11737c478bd9Sstevel@tonic-gate 		 */
11747c478bd9Sstevel@tonic-gate 		if (bytes == ULONG_MAX)
11757c478bd9Sstevel@tonic-gate 			return ((void *)-1);
11767c478bd9Sstevel@tonic-gate 
11777c478bd9Sstevel@tonic-gate 		ret = sbrk(0);
11787c478bd9Sstevel@tonic-gate 		wad = LONG_MAX;
11797c478bd9Sstevel@tonic-gate 		while (wad > 0) {
11807c478bd9Sstevel@tonic-gate 			if (sbrk(wad) == (void *)-1) {
11817c478bd9Sstevel@tonic-gate 				if (ret != sbrk(0))
11827c478bd9Sstevel@tonic-gate 					(void) sbrk(-LONG_MAX);
11837c478bd9Sstevel@tonic-gate 				return ((void *)-1);
11847c478bd9Sstevel@tonic-gate 			}
11857c478bd9Sstevel@tonic-gate 			bytes -= LONG_MAX;
11867c478bd9Sstevel@tonic-gate 			wad = bytes;
11877c478bd9Sstevel@tonic-gate 		}
11887c478bd9Sstevel@tonic-gate 	} else
11897c478bd9Sstevel@tonic-gate 		ret = sbrk(bytes);
11907c478bd9Sstevel@tonic-gate 
11917c478bd9Sstevel@tonic-gate 	return (ret);
11927c478bd9Sstevel@tonic-gate }
11937c478bd9Sstevel@tonic-gate 
11947c478bd9Sstevel@tonic-gate 
11957c478bd9Sstevel@tonic-gate static void *
oversize(size_t size)11967c478bd9Sstevel@tonic-gate oversize(size_t size)
11977c478bd9Sstevel@tonic-gate {
11987c478bd9Sstevel@tonic-gate 	caddr_t ret;
11997c478bd9Sstevel@tonic-gate 	oversize_t *big;
12007c478bd9Sstevel@tonic-gate 
120170911a0dSrm 	/* make sure we will not overflow */
120270911a0dSrm 	if (size > MAX_MTMALLOC) {
120370911a0dSrm 		errno = ENOMEM;
120470911a0dSrm 		return (NULL);
120570911a0dSrm 	}
12067c478bd9Sstevel@tonic-gate 
12077c478bd9Sstevel@tonic-gate 	/*
12087c478bd9Sstevel@tonic-gate 	 * Since we ensure every address we hand back is
12097c478bd9Sstevel@tonic-gate 	 * MTMALLOC_MIN_ALIGN-byte aligned, ALIGNing size ensures that the
12107c478bd9Sstevel@tonic-gate 	 * memory handed out is MTMALLOC_MIN_ALIGN-byte aligned at both ends.
12117c478bd9Sstevel@tonic-gate 	 * This eases the implementation of MTDEBUGPATTERN and MTINITPATTERN,
12127c478bd9Sstevel@tonic-gate 	 * particularly where coalescing occurs.
12137c478bd9Sstevel@tonic-gate 	 */
12147c478bd9Sstevel@tonic-gate 	size = ALIGN(size, MTMALLOC_MIN_ALIGN);
12157c478bd9Sstevel@tonic-gate 
121670911a0dSrm 	/*
121770911a0dSrm 	 * The idea with the global lock is that we are sure to
121870911a0dSrm 	 * block in the kernel anyway since given an oversize alloc
121970911a0dSrm 	 * we are sure to have to call morecore();
122070911a0dSrm 	 */
122170911a0dSrm 	(void) mutex_lock(&oversize_lock);
122270911a0dSrm 
12237c478bd9Sstevel@tonic-gate 	if ((big = find_oversize(size)) != NULL) {
12247c478bd9Sstevel@tonic-gate 		if (reinit == 0 && (debugopt & MTDEBUGPATTERN))
12257c478bd9Sstevel@tonic-gate 			if (verify_pattern(FREEPATTERN, big->addr, size))
12267c478bd9Sstevel@tonic-gate 				abort();	/* reference after free */
12277c478bd9Sstevel@tonic-gate 	} else {
12287c478bd9Sstevel@tonic-gate 		/* Get more 8-byte aligned memory from heap */
12297c478bd9Sstevel@tonic-gate 		ret = morecore(size + OVSZ_HEADER_SIZE);
12307c478bd9Sstevel@tonic-gate 		if (ret == (caddr_t)-1) {
12317c478bd9Sstevel@tonic-gate 			(void) mutex_unlock(&oversize_lock);
12327c478bd9Sstevel@tonic-gate 			errno = ENOMEM;
12337c478bd9Sstevel@tonic-gate 			return (NULL);
12347c478bd9Sstevel@tonic-gate 		}
12357c478bd9Sstevel@tonic-gate 		big = oversize_header_alloc((uintptr_t)ret, size);
12367c478bd9Sstevel@tonic-gate 	}
12377c478bd9Sstevel@tonic-gate 	ret = big->addr;
12387c478bd9Sstevel@tonic-gate 
12398f2954faSraf 	insert_hash(big);
12407c478bd9Sstevel@tonic-gate 
12417c478bd9Sstevel@tonic-gate 	if (debugopt & MTINITBUFFER)
12427c478bd9Sstevel@tonic-gate 		copy_pattern(INITPATTERN, ret, size);
12437c478bd9Sstevel@tonic-gate 
12447c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&oversize_lock);
12457c478bd9Sstevel@tonic-gate 	assert(((uintptr_t)ret & 7) == 0); /* are we 8 byte aligned */
12467c478bd9Sstevel@tonic-gate 	return ((void *)ret);
12477c478bd9Sstevel@tonic-gate }
12487c478bd9Sstevel@tonic-gate 
12497c478bd9Sstevel@tonic-gate static void
insert_oversize(oversize_t * op,oversize_t * nx)12507c478bd9Sstevel@tonic-gate insert_oversize(oversize_t *op, oversize_t *nx)
12517c478bd9Sstevel@tonic-gate {
12527c478bd9Sstevel@tonic-gate 	oversize_t *sp;
12537c478bd9Sstevel@tonic-gate 
12547c478bd9Sstevel@tonic-gate 	/* locate correct insertion point in size-ordered list */
12557c478bd9Sstevel@tonic-gate 	for (sp = oversize_list.next_bysize;
12567c478bd9Sstevel@tonic-gate 	    sp != &oversize_list && (op->size > sp->size);
12577c478bd9Sstevel@tonic-gate 	    sp = sp->next_bysize)
12587c478bd9Sstevel@tonic-gate 		;
12597c478bd9Sstevel@tonic-gate 
12607c478bd9Sstevel@tonic-gate 	/* link into size-ordered list */
12617c478bd9Sstevel@tonic-gate 	op->next_bysize = sp;
12627c478bd9Sstevel@tonic-gate 	op->prev_bysize = sp->prev_bysize;
12637c478bd9Sstevel@tonic-gate 	op->prev_bysize->next_bysize = op;
12647c478bd9Sstevel@tonic-gate 	op->next_bysize->prev_bysize = op;
12657c478bd9Sstevel@tonic-gate 
12667c478bd9Sstevel@tonic-gate 	/*
12677c478bd9Sstevel@tonic-gate 	 * link item into address-ordered list
12687c478bd9Sstevel@tonic-gate 	 * (caller provides insertion point as an optimization)
12697c478bd9Sstevel@tonic-gate 	 */
12707c478bd9Sstevel@tonic-gate 	op->next_byaddr = nx;
12717c478bd9Sstevel@tonic-gate 	op->prev_byaddr = nx->prev_byaddr;
12727c478bd9Sstevel@tonic-gate 	op->prev_byaddr->next_byaddr = op;
12737c478bd9Sstevel@tonic-gate 	op->next_byaddr->prev_byaddr = op;
12747c478bd9Sstevel@tonic-gate 
12757c478bd9Sstevel@tonic-gate }
12767c478bd9Sstevel@tonic-gate 
12777c478bd9Sstevel@tonic-gate static void
unlink_oversize(oversize_t * lp)12787c478bd9Sstevel@tonic-gate unlink_oversize(oversize_t *lp)
12797c478bd9Sstevel@tonic-gate {
12807c478bd9Sstevel@tonic-gate 	/* unlink from address list */
12817c478bd9Sstevel@tonic-gate 	lp->prev_byaddr->next_byaddr = lp->next_byaddr;
12827c478bd9Sstevel@tonic-gate 	lp->next_byaddr->prev_byaddr = lp->prev_byaddr;
12837c478bd9Sstevel@tonic-gate 
12847c478bd9Sstevel@tonic-gate 	/* unlink from size list */
12857c478bd9Sstevel@tonic-gate 	lp->prev_bysize->next_bysize = lp->next_bysize;
12867c478bd9Sstevel@tonic-gate 	lp->next_bysize->prev_bysize = lp->prev_bysize;
12877c478bd9Sstevel@tonic-gate }
12887c478bd9Sstevel@tonic-gate 
12897c478bd9Sstevel@tonic-gate static void
position_oversize_by_size(oversize_t * op)12907c478bd9Sstevel@tonic-gate position_oversize_by_size(oversize_t *op)
12917c478bd9Sstevel@tonic-gate {
12927c478bd9Sstevel@tonic-gate 	oversize_t *sp;
12937c478bd9Sstevel@tonic-gate 
12947c478bd9Sstevel@tonic-gate 	if (op->size > op->next_bysize->size ||
12957c478bd9Sstevel@tonic-gate 	    op->size < op->prev_bysize->size) {
12967c478bd9Sstevel@tonic-gate 
12977c478bd9Sstevel@tonic-gate 		/* unlink from size list */
12987c478bd9Sstevel@tonic-gate 		op->prev_bysize->next_bysize = op->next_bysize;
12997c478bd9Sstevel@tonic-gate 		op->next_bysize->prev_bysize = op->prev_bysize;
13007c478bd9Sstevel@tonic-gate 
13017c478bd9Sstevel@tonic-gate 		/* locate correct insertion point in size-ordered list */
13027c478bd9Sstevel@tonic-gate 		for (sp = oversize_list.next_bysize;
13037c478bd9Sstevel@tonic-gate 		    sp != &oversize_list && (op->size > sp->size);
13047c478bd9Sstevel@tonic-gate 		    sp = sp->next_bysize)
13057c478bd9Sstevel@tonic-gate 			;
13067c478bd9Sstevel@tonic-gate 
13077c478bd9Sstevel@tonic-gate 		/* link into size-ordered list */
13087c478bd9Sstevel@tonic-gate 		op->next_bysize = sp;
13097c478bd9Sstevel@tonic-gate 		op->prev_bysize = sp->prev_bysize;
13107c478bd9Sstevel@tonic-gate 		op->prev_bysize->next_bysize = op;
13117c478bd9Sstevel@tonic-gate 		op->next_bysize->prev_bysize = op;
13127c478bd9Sstevel@tonic-gate 	}
13137c478bd9Sstevel@tonic-gate }
13147c478bd9Sstevel@tonic-gate 
13157c478bd9Sstevel@tonic-gate static void
add_oversize(oversize_t * lp)13167c478bd9Sstevel@tonic-gate add_oversize(oversize_t *lp)
13177c478bd9Sstevel@tonic-gate {
13187c478bd9Sstevel@tonic-gate 	int merge_flags = INSERT_ONLY;
1319*b5358705SToomas Soome 	oversize_t *nx;		/* ptr to item right of insertion point */
1320*b5358705SToomas Soome 	oversize_t *pv;		/* ptr to item left of insertion point */
1321*b5358705SToomas Soome 	uint_t size_lp, size_pv, size_nx = 0;
13227c478bd9Sstevel@tonic-gate 	uintptr_t endp_lp, endp_pv, endp_nx;
13237c478bd9Sstevel@tonic-gate 
13247c478bd9Sstevel@tonic-gate 	/*
13257c478bd9Sstevel@tonic-gate 	 * Locate insertion point in address-ordered list
13267c478bd9Sstevel@tonic-gate 	 */
13277c478bd9Sstevel@tonic-gate 
13287c478bd9Sstevel@tonic-gate 	for (nx = oversize_list.next_byaddr;
13297c478bd9Sstevel@tonic-gate 	    nx != &oversize_list && (lp->addr > nx->addr);
13307c478bd9Sstevel@tonic-gate 	    nx = nx->next_byaddr)
13317c478bd9Sstevel@tonic-gate 		;
13327c478bd9Sstevel@tonic-gate 
13337c478bd9Sstevel@tonic-gate 	/*
13347c478bd9Sstevel@tonic-gate 	 * Determine how to add chunk to oversize freelist
13357c478bd9Sstevel@tonic-gate 	 */
13367c478bd9Sstevel@tonic-gate 
13377c478bd9Sstevel@tonic-gate 	size_lp = OVSZ_HEADER_SIZE + lp->size;
13387c478bd9Sstevel@tonic-gate 	endp_lp = ALIGN((uintptr_t)lp + size_lp, MTMALLOC_MIN_ALIGN);
13397c478bd9Sstevel@tonic-gate 	size_lp = endp_lp - (uintptr_t)lp;
13407c478bd9Sstevel@tonic-gate 
13417c478bd9Sstevel@tonic-gate 	pv = nx->prev_byaddr;
13427c478bd9Sstevel@tonic-gate 
13437c478bd9Sstevel@tonic-gate 	if (pv->size) {
13447c478bd9Sstevel@tonic-gate 
13457c478bd9Sstevel@tonic-gate 		size_pv = OVSZ_HEADER_SIZE + pv->size;
13467c478bd9Sstevel@tonic-gate 		endp_pv = ALIGN((uintptr_t)pv + size_pv,
13477c478bd9Sstevel@tonic-gate 		    MTMALLOC_MIN_ALIGN);
13487c478bd9Sstevel@tonic-gate 		size_pv = endp_pv - (uintptr_t)pv;
13497c478bd9Sstevel@tonic-gate 
13507c478bd9Sstevel@tonic-gate 		/* Check for adjacency with left chunk */
13517c478bd9Sstevel@tonic-gate 		if ((uintptr_t)lp == endp_pv)
13527c478bd9Sstevel@tonic-gate 			merge_flags |= COALESCE_LEFT;
13537c478bd9Sstevel@tonic-gate 	}
13547c478bd9Sstevel@tonic-gate 
13557c478bd9Sstevel@tonic-gate 	if (nx->size) {
13567c478bd9Sstevel@tonic-gate 
1357db3659e5SSurya Prakki 		/* Check for adjacency with right chunk */
1358db3659e5SSurya Prakki 		if ((uintptr_t)nx == endp_lp) {
1359db3659e5SSurya Prakki 			size_nx = OVSZ_HEADER_SIZE + nx->size;
1360db3659e5SSurya Prakki 			endp_nx = ALIGN((uintptr_t)nx + size_nx,
1361db3659e5SSurya Prakki 			    MTMALLOC_MIN_ALIGN);
1362db3659e5SSurya Prakki 			size_nx = endp_nx - (uintptr_t)nx;
1363db3659e5SSurya Prakki 			merge_flags |= COALESCE_RIGHT;
1364db3659e5SSurya Prakki 		}
13657c478bd9Sstevel@tonic-gate 	}
13667c478bd9Sstevel@tonic-gate 
13677c478bd9Sstevel@tonic-gate 	/*
13687c478bd9Sstevel@tonic-gate 	 * If MTDEBUGPATTERN==1, lp->addr will have been overwritten with
13697c478bd9Sstevel@tonic-gate 	 * FREEPATTERN for lp->size bytes. If we can merge, the oversize
13707c478bd9Sstevel@tonic-gate 	 * header(s) that will also become part of the memory available for
13717c478bd9Sstevel@tonic-gate 	 * reallocation (ie lp and/or nx) must also be overwritten with
13727c478bd9Sstevel@tonic-gate 	 * FREEPATTERN or we will SIGABRT when this memory is next reallocated.
13737c478bd9Sstevel@tonic-gate 	 */
13747c478bd9Sstevel@tonic-gate 	switch (merge_flags) {
13757c478bd9Sstevel@tonic-gate 
13767c478bd9Sstevel@tonic-gate 	case INSERT_ONLY:		/* Coalescing not possible */
13777c478bd9Sstevel@tonic-gate 		insert_oversize(lp, nx);
13787c478bd9Sstevel@tonic-gate 		break;
13797c478bd9Sstevel@tonic-gate 	case COALESCE_LEFT:
13807c478bd9Sstevel@tonic-gate 		pv->size += size_lp;
13817c478bd9Sstevel@tonic-gate 		position_oversize_by_size(pv);
13827c478bd9Sstevel@tonic-gate 		if (debugopt & MTDEBUGPATTERN)
13837c478bd9Sstevel@tonic-gate 			copy_pattern(FREEPATTERN, lp, OVSZ_HEADER_SIZE);
13847c478bd9Sstevel@tonic-gate 		break;
13857c478bd9Sstevel@tonic-gate 	case COALESCE_RIGHT:
13867c478bd9Sstevel@tonic-gate 		unlink_oversize(nx);
13877c478bd9Sstevel@tonic-gate 		lp->size += size_nx;
13887c478bd9Sstevel@tonic-gate 		insert_oversize(lp, pv->next_byaddr);
13897c478bd9Sstevel@tonic-gate 		if (debugopt & MTDEBUGPATTERN)
13907c478bd9Sstevel@tonic-gate 			copy_pattern(FREEPATTERN, nx, OVSZ_HEADER_SIZE);
13917c478bd9Sstevel@tonic-gate 		break;
13927c478bd9Sstevel@tonic-gate 	case COALESCE_WITH_BOTH_SIDES:	/* Merge (with right) to the left */
13937c478bd9Sstevel@tonic-gate 		pv->size += size_lp + size_nx;
13947c478bd9Sstevel@tonic-gate 		unlink_oversize(nx);
13957c478bd9Sstevel@tonic-gate 		position_oversize_by_size(pv);
13967c478bd9Sstevel@tonic-gate 		if (debugopt & MTDEBUGPATTERN) {
13977c478bd9Sstevel@tonic-gate 			copy_pattern(FREEPATTERN, lp, OVSZ_HEADER_SIZE);
13987c478bd9Sstevel@tonic-gate 			copy_pattern(FREEPATTERN, nx, OVSZ_HEADER_SIZE);
13997c478bd9Sstevel@tonic-gate 		}
14007c478bd9Sstevel@tonic-gate 		break;
14017c478bd9Sstevel@tonic-gate 	}
14027c478bd9Sstevel@tonic-gate }
14037c478bd9Sstevel@tonic-gate 
14047c478bd9Sstevel@tonic-gate /*
14057c478bd9Sstevel@tonic-gate  * Find memory on our list that is at least size big. If we find a block that is
14067c478bd9Sstevel@tonic-gate  * big enough, we break it up and return the associated oversize_t struct back
14077c478bd9Sstevel@tonic-gate  * to the calling client. Any leftover piece of that block is returned to the
14087c478bd9Sstevel@tonic-gate  * freelist.
14097c478bd9Sstevel@tonic-gate  */
14107c478bd9Sstevel@tonic-gate static oversize_t *
find_oversize(size_t size)14117c478bd9Sstevel@tonic-gate find_oversize(size_t size)
14127c478bd9Sstevel@tonic-gate {
14137c478bd9Sstevel@tonic-gate 	oversize_t *wp = oversize_list.next_bysize;
14147c478bd9Sstevel@tonic-gate 	while (wp != &oversize_list && size > wp->size)
14157c478bd9Sstevel@tonic-gate 		wp = wp->next_bysize;
14167c478bd9Sstevel@tonic-gate 
14177c478bd9Sstevel@tonic-gate 	if (wp == &oversize_list) /* empty list or nothing big enough */
14187c478bd9Sstevel@tonic-gate 		return (NULL);
14197c478bd9Sstevel@tonic-gate 	/* breaking up a chunk of memory */
14207c478bd9Sstevel@tonic-gate 	if ((long)((wp->size - (size + OVSZ_HEADER_SIZE + MTMALLOC_MIN_ALIGN)))
14217c478bd9Sstevel@tonic-gate 	    > MAX_CACHED) {
14227c478bd9Sstevel@tonic-gate 		caddr_t off;
14237c478bd9Sstevel@tonic-gate 		oversize_t *np;
14247c478bd9Sstevel@tonic-gate 		size_t osize;
14257c478bd9Sstevel@tonic-gate 		off = (caddr_t)ALIGN(wp->addr + size,
14267c478bd9Sstevel@tonic-gate 		    MTMALLOC_MIN_ALIGN);
14277c478bd9Sstevel@tonic-gate 		osize = wp->size;
14287c478bd9Sstevel@tonic-gate 		wp->size = (size_t)(off - wp->addr);
14297c478bd9Sstevel@tonic-gate 		np = oversize_header_alloc((uintptr_t)off,
14307c478bd9Sstevel@tonic-gate 		    osize - (wp->size + OVSZ_HEADER_SIZE));
14317c478bd9Sstevel@tonic-gate 		if ((long)np->size < 0)
14327c478bd9Sstevel@tonic-gate 			abort();
14337c478bd9Sstevel@tonic-gate 		unlink_oversize(wp);
14347c478bd9Sstevel@tonic-gate 		add_oversize(np);
14357c478bd9Sstevel@tonic-gate 	} else {
14367c478bd9Sstevel@tonic-gate 		unlink_oversize(wp);
14377c478bd9Sstevel@tonic-gate 	}
14387c478bd9Sstevel@tonic-gate 	return (wp);
14397c478bd9Sstevel@tonic-gate }
14407c478bd9Sstevel@tonic-gate 
14417c478bd9Sstevel@tonic-gate static void
copy_pattern(uint32_t pattern,void * buf_arg,size_t size)14427c478bd9Sstevel@tonic-gate copy_pattern(uint32_t pattern, void *buf_arg, size_t size)
14437c478bd9Sstevel@tonic-gate {
14447c478bd9Sstevel@tonic-gate 	uint32_t *bufend = (uint32_t *)((char *)buf_arg + size);
14457c478bd9Sstevel@tonic-gate 	uint32_t *buf = buf_arg;
14467c478bd9Sstevel@tonic-gate 
14477c478bd9Sstevel@tonic-gate 	while (buf < bufend - 3) {
14487c478bd9Sstevel@tonic-gate 		buf[3] = buf[2] = buf[1] = buf[0] = pattern;
14497c478bd9Sstevel@tonic-gate 		buf += 4;
14507c478bd9Sstevel@tonic-gate 	}
14517c478bd9Sstevel@tonic-gate 	while (buf < bufend)
14527c478bd9Sstevel@tonic-gate 		*buf++ = pattern;
14537c478bd9Sstevel@tonic-gate }
14547c478bd9Sstevel@tonic-gate 
14557c478bd9Sstevel@tonic-gate static void *
verify_pattern(uint32_t pattern,void * buf_arg,size_t size)14567c478bd9Sstevel@tonic-gate verify_pattern(uint32_t pattern, void *buf_arg, size_t size)
14577c478bd9Sstevel@tonic-gate {
14587c478bd9Sstevel@tonic-gate 	uint32_t *bufend = (uint32_t *)((char *)buf_arg + size);
14597c478bd9Sstevel@tonic-gate 	uint32_t *buf;
14607c478bd9Sstevel@tonic-gate 
14617c478bd9Sstevel@tonic-gate 	for (buf = buf_arg; buf < bufend; buf++)
14627c478bd9Sstevel@tonic-gate 		if (*buf != pattern)
14637c478bd9Sstevel@tonic-gate 			return (buf);
14647c478bd9Sstevel@tonic-gate 	return (NULL);
14657c478bd9Sstevel@tonic-gate }
14667c478bd9Sstevel@tonic-gate 
14677c478bd9Sstevel@tonic-gate static void
free_oversize(oversize_t * ovp)14687c478bd9Sstevel@tonic-gate free_oversize(oversize_t *ovp)
14697c478bd9Sstevel@tonic-gate {
14707c478bd9Sstevel@tonic-gate 	assert(((uintptr_t)ovp->addr & 7) == 0); /* are we 8 byte aligned */
14717c478bd9Sstevel@tonic-gate 	assert(ovp->size > MAX_CACHED);
14727c478bd9Sstevel@tonic-gate 
14737c478bd9Sstevel@tonic-gate 	ovp->next_bysize = ovp->prev_bysize = NULL;
14747c478bd9Sstevel@tonic-gate 	ovp->next_byaddr = ovp->prev_byaddr = NULL;
14757c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&oversize_lock);
14767c478bd9Sstevel@tonic-gate 	add_oversize(ovp);
14777c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&oversize_lock);
14787c478bd9Sstevel@tonic-gate }
14797c478bd9Sstevel@tonic-gate 
14807c478bd9Sstevel@tonic-gate static oversize_t *
oversize_header_alloc(uintptr_t mem,size_t size)14817c478bd9Sstevel@tonic-gate oversize_header_alloc(uintptr_t mem, size_t size)
14827c478bd9Sstevel@tonic-gate {
14837c478bd9Sstevel@tonic-gate 	oversize_t *ovsz_hdr;
14847c478bd9Sstevel@tonic-gate 
14857c478bd9Sstevel@tonic-gate 	assert(size > MAX_CACHED);
14867c478bd9Sstevel@tonic-gate 
14877c478bd9Sstevel@tonic-gate 	ovsz_hdr = (oversize_t *)mem;
14887c478bd9Sstevel@tonic-gate 	ovsz_hdr->prev_bysize = NULL;
14897c478bd9Sstevel@tonic-gate 	ovsz_hdr->next_bysize = NULL;
14907c478bd9Sstevel@tonic-gate 	ovsz_hdr->prev_byaddr = NULL;
14917c478bd9Sstevel@tonic-gate 	ovsz_hdr->next_byaddr = NULL;
14927c478bd9Sstevel@tonic-gate 	ovsz_hdr->hash_next = NULL;
14937c478bd9Sstevel@tonic-gate 	ovsz_hdr->size = size;
14947c478bd9Sstevel@tonic-gate 	mem += OVSZ_SIZE;
14957c478bd9Sstevel@tonic-gate 	*(uintptr_t *)mem = MTMALLOC_OVERSIZE_MAGIC;
14967c478bd9Sstevel@tonic-gate 	mem += OVERHEAD;
14977c478bd9Sstevel@tonic-gate 	assert(((uintptr_t)mem & 7) == 0); /* are we 8 byte aligned */
14987c478bd9Sstevel@tonic-gate 	ovsz_hdr->addr = (caddr_t)mem;
14997c478bd9Sstevel@tonic-gate 	return (ovsz_hdr);
15007c478bd9Sstevel@tonic-gate }
15011d530678Sraf 
15021d530678Sraf static void
malloc_prepare()15031d530678Sraf malloc_prepare()
15041d530678Sraf {
15051d530678Sraf 	percpu_t *cpuptr;
15061d530678Sraf 	cache_head_t *cachehead;
15071d530678Sraf 	cache_t *thiscache;
15081d530678Sraf 
15091d530678Sraf 	(void) mutex_lock(&oversize_lock);
15101d530678Sraf 	for (cpuptr = &cpu_list[0]; cpuptr < &cpu_list[ncpus]; cpuptr++) {
15111d530678Sraf 		(void) mutex_lock(&cpuptr->mt_parent_lock);
15121d530678Sraf 		for (cachehead = &cpuptr->mt_caches[0];
15131d530678Sraf 		    cachehead < &cpuptr->mt_caches[NUM_CACHES];
15141d530678Sraf 		    cachehead++) {
15151d530678Sraf 			for (thiscache = cachehead->mt_cache;
15161d530678Sraf 			    thiscache != NULL;
15171d530678Sraf 			    thiscache = thiscache->mt_next) {
15181d530678Sraf 				(void) mutex_lock(
15191d530678Sraf 				    &thiscache->mt_cache_lock);
15201d530678Sraf 			}
15211d530678Sraf 		}
15221d530678Sraf 	}
15231d530678Sraf }
15241d530678Sraf 
15251d530678Sraf static void
malloc_release()15261d530678Sraf malloc_release()
15271d530678Sraf {
15281d530678Sraf 	percpu_t *cpuptr;
15291d530678Sraf 	cache_head_t *cachehead;
15301d530678Sraf 	cache_t *thiscache;
15311d530678Sraf 
15321d530678Sraf 	for (cpuptr = &cpu_list[ncpus - 1]; cpuptr >= &cpu_list[0]; cpuptr--) {
15331d530678Sraf 		for (cachehead = &cpuptr->mt_caches[NUM_CACHES - 1];
15341d530678Sraf 		    cachehead >= &cpuptr->mt_caches[0];
15351d530678Sraf 		    cachehead--) {
15361d530678Sraf 			for (thiscache = cachehead->mt_cache;
15371d530678Sraf 			    thiscache != NULL;
15381d530678Sraf 			    thiscache = thiscache->mt_next) {
15391d530678Sraf 				(void) mutex_unlock(
15401d530678Sraf 				    &thiscache->mt_cache_lock);
15411d530678Sraf 			}
15421d530678Sraf 		}
15431d530678Sraf 		(void) mutex_unlock(&cpuptr->mt_parent_lock);
15441d530678Sraf 	}
15451d530678Sraf 	(void) mutex_unlock(&oversize_lock);
15461d530678Sraf }
15471d530678Sraf 
15481d530678Sraf #pragma init(malloc_init)
15491d530678Sraf static void
malloc_init(void)15501d530678Sraf malloc_init(void)
15511d530678Sraf {
15521d530678Sraf 	/*
15531d530678Sraf 	 * This works in the init section for this library
15541d530678Sraf 	 * because setup_caches() doesn't call anything in libc
15551d530678Sraf 	 * that calls malloc().  If it did, disaster would ensue.
15561d530678Sraf 	 *
15571d530678Sraf 	 * For this to work properly, this library must be the first
15581d530678Sraf 	 * one to have its init section called (after libc) by the
15591d530678Sraf 	 * dynamic linker.  If some other library's init section
15601d530678Sraf 	 * ran first and called malloc(), disaster would ensue.
15611d530678Sraf 	 * Because this is an interposer library for malloc(), the
15621d530678Sraf 	 * dynamic linker arranges for its init section to run first.
15631d530678Sraf 	 */
15641d530678Sraf 	(void) setup_caches();
15651d530678Sraf 
15661d530678Sraf 	(void) pthread_atfork(malloc_prepare, malloc_release, malloc_release);
15671d530678Sraf }
1568