1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
27 /*	  All Rights Reserved  	*/
28 
29 
30 #pragma ident	"%Z%%M%	%I%	%E% SMI"
31 
32 #include <sys/types.h>
33 #include <sys/bitmap.h>
34 #include <sys/sysmacros.h>
35 #include <sys/kmem.h>
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/user.h>
39 #include <sys/unistd.h>
40 #include <sys/errno.h>
41 #include <sys/proc.h>
42 #include <sys/mman.h>
43 #include <sys/tuneable.h>
44 #include <sys/cmn_err.h>
45 #include <sys/cred.h>
46 #include <sys/vmsystm.h>
47 #include <sys/debug.h>
48 #include <sys/policy.h>
49 
50 #include <vm/as.h>
51 #include <vm/seg.h>
52 
53 static uint_t mem_getpgszc(size_t);
54 
55 /*
56  * Memory control operations
57  */
58 int
59 memcntl(caddr_t addr, size_t len, int cmd, caddr_t arg, int attr, int mask)
60 {
61 	struct as *as = ttoproc(curthread)->p_as;
62 	struct proc *p = ttoproc(curthread);
63 	size_t pgsz;
64 	uint_t szc, oszc, pgcmd;
65 	int error = 0;
66 	faultcode_t fc;
67 	uintptr_t iarg;
68 	STRUCT_DECL(memcntl_mha, mha);
69 
70 	if (mask)
71 		return (set_errno(EINVAL));
72 	if ((cmd == MC_LOCKAS) || (cmd == MC_UNLOCKAS)) {
73 		if ((addr != 0) || (len != 0)) {
74 			return (set_errno(EINVAL));
75 		}
76 	} else if (cmd != MC_HAT_ADVISE) {
77 		if (((uintptr_t)addr & PAGEOFFSET) != 0 || len == 0) {
78 			return (set_errno(EINVAL));
79 		}
80 		/*
81 		 * We're only concerned with the address range
82 		 * here, not the protections.  The protections
83 		 * are only used as a "filter" in this code,
84 		 * they aren't set or modified here.
85 		 */
86 		if (valid_usr_range(addr, len, 0, as,
87 		    as->a_userlimit) != RANGE_OKAY) {
88 			return (set_errno(ENOMEM));
89 		}
90 	}
91 
92 	if (cmd == MC_HAT_ADVISE) {
93 		if (attr != 0 || mask != 0) {
94 			return (set_errno(EINVAL));
95 		}
96 
97 	} else {
98 		if ((VALID_ATTR & attr) != attr) {
99 			return (set_errno(EINVAL));
100 		}
101 		if ((attr & SHARED) && (attr & PRIVATE)) {
102 			return (set_errno(EINVAL));
103 		}
104 		if (((cmd == MC_LOCKAS) || (cmd == MC_LOCK) ||
105 		    (cmd == MC_UNLOCKAS) || (cmd == MC_UNLOCK)) &&
106 		    (error = secpolicy_lock_memory(CRED())) != 0)
107 			return (set_errno(error));
108 	}
109 	if (attr) {
110 		attr |= PROT_USER;
111 	}
112 
113 	switch (cmd) {
114 	case MC_SYNC:
115 		/*
116 		 * MS_SYNC used to be defined to be zero but is now non-zero.
117 		 * For binary compatibility we still accept zero
118 		 * (the absence of MS_ASYNC) to mean the same thing.
119 		 */
120 		iarg = (uintptr_t)arg;
121 		if ((iarg & ~MS_INVALIDATE) == 0)
122 			iarg |= MS_SYNC;
123 
124 		if (((iarg & ~(MS_SYNC|MS_ASYNC|MS_INVALIDATE)) != 0) ||
125 			((iarg & (MS_SYNC|MS_ASYNC)) == (MS_SYNC|MS_ASYNC))) {
126 			error = set_errno(EINVAL);
127 		} else {
128 			error = as_ctl(as, addr, len, cmd, attr, iarg, NULL, 0);
129 			if (error) {
130 				(void) set_errno(error);
131 			}
132 		}
133 		return (error);
134 	case MC_LOCKAS:
135 		if ((uintptr_t)arg & ~(MCL_FUTURE|MCL_CURRENT) ||
136 		    (uintptr_t)arg == 0) {
137 			return (set_errno(EINVAL));
138 		}
139 		break;
140 	case MC_LOCK:
141 	case MC_UNLOCKAS:
142 	case MC_UNLOCK:
143 		break;
144 	case MC_HAT_ADVISE:
145 		/*
146 		 * Set prefered page size.
147 		 */
148 		STRUCT_INIT(mha, get_udatamodel());
149 		if (copyin(arg, STRUCT_BUF(mha), STRUCT_SIZE(mha))) {
150 			return (set_errno(EFAULT));
151 		}
152 
153 		pgcmd = STRUCT_FGET(mha, mha_cmd);
154 
155 		/*
156 		 * Currently only MHA_MAPSIZE_VA, MHA_MAPSIZE_STACK
157 		 * and MHA_MAPSIZE_BSSBRK are supported. Only one
158 		 * command may be specified at a time.
159 		 */
160 		if ((~(MHA_MAPSIZE_VA|MHA_MAPSIZE_STACK|MHA_MAPSIZE_BSSBRK) &
161 		    pgcmd) || pgcmd == 0 || !ISP2(pgcmd) ||
162 		    STRUCT_FGET(mha, mha_flags))
163 			return (set_errno(EINVAL));
164 
165 		pgsz = STRUCT_FGET(mha, mha_pagesize);
166 
167 		/*
168 		 * call platform specific map_pgsz() routine to get the
169 		 * optimal pgsz if pgsz is 0.
170 		 *
171 		 * For stack and heap operations addr and len must be zero.
172 		 */
173 		if ((pgcmd & (MHA_MAPSIZE_BSSBRK|MHA_MAPSIZE_STACK)) != 0) {
174 			if (addr != NULL || len != 0) {
175 				return (set_errno(EINVAL));
176 			}
177 
178 			/*
179 			 * Disable autompss for this process unless pgsz == 0,
180 			 * which means the system should pick.  In the
181 			 * pgsz == 0 case, leave the SAUTOLPG setting alone, as
182 			 * we don't want to enable it when someone has
183 			 * disabled automatic large page selection for the
184 			 * whole system.
185 			 */
186 			mutex_enter(&p->p_lock);
187 			if (pgsz != 0) {
188 				p->p_flag &= ~SAUTOLPG;
189 			}
190 			mutex_exit(&p->p_lock);
191 
192 			as_rangelock(as);
193 
194 			if (pgsz == 0) {
195 				int	type;
196 
197 				if (pgcmd == MHA_MAPSIZE_BSSBRK)
198 					type = MAPPGSZ_HEAP;
199 				else
200 					type = MAPPGSZ_STK;
201 
202 				pgsz = map_pgsz(type, p, 0, 0, NULL);
203 			}
204 		} else {
205 			/*
206 			 * Note that we don't disable automatic large page
207 			 * selection for anon segments based on use of
208 			 * memcntl().
209 			 */
210 			if (pgsz == 0) {
211 				pgsz = map_pgsz(MAPPGSZ_VA, p, addr, len,
212 				    NULL);
213 			}
214 
215 			/*
216 			 * addr and len must be prefered page size aligned
217 			 * and valid for range specified.
218 			 */
219 			if (!IS_P2ALIGNED(addr, pgsz) ||
220 			    !IS_P2ALIGNED(len, pgsz)) {
221 				return (set_errno(EINVAL));
222 			}
223 			if (valid_usr_range(addr, len, 0, as,
224 			    as->a_userlimit) != RANGE_OKAY) {
225 				return (set_errno(ENOMEM));
226 			}
227 		}
228 
229 		szc = mem_getpgszc(pgsz);
230 		if (szc == (uint_t)-1) {
231 			if ((pgcmd & (MHA_MAPSIZE_BSSBRK|MHA_MAPSIZE_STACK))
232 			    != 0) {
233 				as_rangeunlock(as);
234 			}
235 			return (set_errno(EINVAL));
236 		}
237 
238 		/*
239 		 * For stack and heap operations we first need to pad
240 		 * out existing range (create new mappings) to the new
241 		 * prefered page size boundary. Also the start of the
242 		 * .bss for the heap or user's stack base may not be on
243 		 * the new prefered page size boundary. For these cases
244 		 * we align the base of the request on the new prefered
245 		 * page size.
246 		 */
247 		if (pgcmd & MHA_MAPSIZE_BSSBRK) {
248 			if (szc == p->p_brkpageszc) {
249 				as_rangeunlock(as);
250 				return (0);
251 			}
252 			if (szc > p->p_brkpageszc) {
253 				error = brk_internal(p->p_brkbase
254 				    + p->p_brksize, szc);
255 				if (error) {
256 					as_rangeunlock(as);
257 					return (set_errno(error));
258 				}
259 			}
260 			oszc = p->p_brkpageszc;
261 			p->p_brkpageszc = szc;
262 
263 			ASSERT(IS_P2ALIGNED(p->p_brkbase + p->p_brksize, pgsz));
264 			addr = (caddr_t)P2ROUNDUP((uintptr_t)p->p_bssbase,
265 			    pgsz);
266 			len = (p->p_brkbase + p->p_brksize) - addr;
267 			ASSERT(IS_P2ALIGNED(len, pgsz));
268 			/*
269 			 * Perhaps no existing pages to promote.
270 			 */
271 			if (len == 0) {
272 				as_rangeunlock(as);
273 				return (0);
274 			}
275 		}
276 		/*
277 		 * The code below, as does grow.c, assumes stacks always grow
278 		 * downward.
279 		 */
280 		if (pgcmd & MHA_MAPSIZE_STACK) {
281 			/*
282 			 * Some boxes (x86) have a top of stack that
283 			 * is not large page aligned. Since stacks are
284 			 * usually small we'll just return and do nothing
285 			 * for theses cases. Prefeered page size is advisory
286 			 * so no need to return an error.
287 			 */
288 			if (szc == p->p_stkpageszc ||
289 			    !IS_P2ALIGNED(p->p_usrstack, pgsz)) {
290 				as_rangeunlock(as);
291 				return (0);
292 			}
293 
294 			if (szc > p->p_stkpageszc) {
295 				error = grow_internal(p->p_usrstack
296 				    - p->p_stksize, szc);
297 				if (error) {
298 					as_rangeunlock(as);
299 					return (set_errno(error));
300 				}
301 			}
302 			oszc = p->p_stkpageszc;
303 			p->p_stkpageszc = szc;
304 
305 			ASSERT(IS_P2ALIGNED(p->p_usrstack, pgsz));
306 			addr = p->p_usrstack - p->p_stksize;
307 			len = p->p_stksize;
308 
309 			/*
310 			 * Perhaps nothing to promote, we wrapped around
311 			 * or grow did not not grow the stack to a large
312 			 * page boundary.
313 			 */
314 			if (!IS_P2ALIGNED(len, pgsz) || len == 0 ||
315 			    addr >= p->p_usrstack || (addr + len) < addr) {
316 				as_rangeunlock(as);
317 				return (0);
318 			}
319 		}
320 		ASSERT(IS_P2ALIGNED(addr, pgsz));
321 		ASSERT(IS_P2ALIGNED(len, pgsz));
322 		error = as_setpagesize(as, addr, len, szc, B_TRUE);
323 
324 		/*
325 		 * On stack or heap failures restore original
326 		 * pg size code.
327 		 */
328 		if (error) {
329 			if ((pgcmd & MHA_MAPSIZE_BSSBRK) != 0) {
330 				p->p_brkpageszc = oszc;
331 			}
332 			if ((pgcmd & MHA_MAPSIZE_STACK) != 0) {
333 				p->p_stkpageszc = oszc;
334 			}
335 			(void) set_errno(error);
336 		}
337 		if ((pgcmd & (MHA_MAPSIZE_BSSBRK|MHA_MAPSIZE_STACK)) != 0) {
338 			as_rangeunlock(as);
339 		}
340 		return (error);
341 	case MC_ADVISE:
342 		if ((uintptr_t)arg == MADV_FREE) {
343 			len &= PAGEMASK;
344 		}
345 		switch ((uintptr_t)arg) {
346 		case MADV_WILLNEED:
347 			fc = as_faulta(as, addr, len);
348 			if (fc) {
349 				if (FC_CODE(fc) == FC_OBJERR)
350 					error = set_errno(FC_ERRNO(fc));
351 				else if (FC_CODE(fc) == FC_NOMAP)
352 					error = set_errno(ENOMEM);
353 				else
354 					error = set_errno(EINVAL);
355 				return (error);
356 			}
357 			break;
358 
359 		case MADV_DONTNEED:
360 			/*
361 			 * For now, don't need is turned into an as_ctl(MC_SYNC)
362 			 * operation flagged for async invalidate.
363 			 */
364 			error = as_ctl(as, addr, len, MC_SYNC, attr,
365 			    MS_ASYNC | MS_INVALIDATE, NULL, 0);
366 			if (error)
367 				(void) set_errno(error);
368 			return (error);
369 
370 		default:
371 			error = as_ctl(as, addr, len, cmd, attr,
372 			    (uintptr_t)arg, NULL, 0);
373 			if (error)
374 				(void) set_errno(error);
375 			return (error);
376 		}
377 		break;
378 	default:
379 		return (set_errno(EINVAL));
380 	}
381 
382 	error = as_ctl(as, addr, len, cmd, attr, (uintptr_t)arg, NULL, 0);
383 
384 	if (error)
385 		(void) set_errno(error);
386 	return (error);
387 }
388 
389 /*
390  * Return page size code for page size passed in. If
391  * matching page size not found or supported, return -1.
392  */
393 static uint_t
394 mem_getpgszc(size_t pgsz) {
395 	return ((uint_t)page_szc_user_filtered(pgsz));
396 }
397