17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5aa042c4bSkchow * Common Development and Distribution License (the "License"). 6aa042c4bSkchow * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22e21bae1bSkchow * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 277c478bd9Sstevel@tonic-gate /* All Rights Reserved */ 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate /* 307c478bd9Sstevel@tonic-gate * Portions of this source code were derived from Berkeley 4.3 BSD 317c478bd9Sstevel@tonic-gate * under license from the Regents of the University of California. 327c478bd9Sstevel@tonic-gate */ 337c478bd9Sstevel@tonic-gate 347c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 357c478bd9Sstevel@tonic-gate 367c478bd9Sstevel@tonic-gate /* 377c478bd9Sstevel@tonic-gate * UNIX machine dependent virtual memory support. 387c478bd9Sstevel@tonic-gate */ 397c478bd9Sstevel@tonic-gate 407c478bd9Sstevel@tonic-gate #include <sys/types.h> 417c478bd9Sstevel@tonic-gate #include <sys/param.h> 427c478bd9Sstevel@tonic-gate #include <sys/systm.h> 437c478bd9Sstevel@tonic-gate #include <sys/user.h> 447c478bd9Sstevel@tonic-gate #include <sys/proc.h> 457c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 467c478bd9Sstevel@tonic-gate #include <sys/vmem.h> 477c478bd9Sstevel@tonic-gate #include <sys/buf.h> 487c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 497c478bd9Sstevel@tonic-gate #include <sys/lgrp.h> 507c478bd9Sstevel@tonic-gate #include <sys/disp.h> 517c478bd9Sstevel@tonic-gate #include <sys/vm.h> 527c478bd9Sstevel@tonic-gate #include <sys/mman.h> 537c478bd9Sstevel@tonic-gate #include <sys/vnode.h> 547c478bd9Sstevel@tonic-gate #include <sys/cred.h> 557c478bd9Sstevel@tonic-gate #include <sys/exec.h> 567c478bd9Sstevel@tonic-gate #include <sys/exechdr.h> 577c478bd9Sstevel@tonic-gate #include <sys/debug.h> 587c478bd9Sstevel@tonic-gate 597c478bd9Sstevel@tonic-gate #include <vm/hat.h> 607c478bd9Sstevel@tonic-gate #include <vm/as.h> 617c478bd9Sstevel@tonic-gate #include <vm/seg.h> 627c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h> 637c478bd9Sstevel@tonic-gate #include <vm/seg_vn.h> 647c478bd9Sstevel@tonic-gate #include <vm/page.h> 657c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 667c478bd9Sstevel@tonic-gate #include <vm/seg_kpm.h> 677c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h> 687c478bd9Sstevel@tonic-gate 697c478bd9Sstevel@tonic-gate #include <sys/cpu.h> 707c478bd9Sstevel@tonic-gate #include <sys/vm_machparam.h> 717c478bd9Sstevel@tonic-gate #include <sys/memlist.h> 727c478bd9Sstevel@tonic-gate #include <sys/bootconf.h> /* XXX the memlist stuff belongs in memlist_plat.h */ 737c478bd9Sstevel@tonic-gate #include <vm/hat_i86.h> 747c478bd9Sstevel@tonic-gate #include <sys/x86_archext.h> 757c478bd9Sstevel@tonic-gate #include <sys/elf_386.h> 767c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 777c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> 787c478bd9Sstevel@tonic-gate #include <sys/machsystm.h> 797c478bd9Sstevel@tonic-gate 807c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 817c478bd9Sstevel@tonic-gate #include <sys/ddidmareq.h> 827c478bd9Sstevel@tonic-gate #include <sys/promif.h> 837c478bd9Sstevel@tonic-gate #include <sys/memnode.h> 847c478bd9Sstevel@tonic-gate #include <sys/stack.h> 857c478bd9Sstevel@tonic-gate 86*5d07b933Sdp uint_t vac_colors = 1; 877c478bd9Sstevel@tonic-gate 887c478bd9Sstevel@tonic-gate int largepagesupport = 0; 897c478bd9Sstevel@tonic-gate extern uint_t page_create_new; 907c478bd9Sstevel@tonic-gate extern uint_t page_create_exists; 917c478bd9Sstevel@tonic-gate extern uint_t page_create_putbacks; 927c478bd9Sstevel@tonic-gate extern uint_t page_create_putbacks; 937c478bd9Sstevel@tonic-gate extern uintptr_t eprom_kernelbase; 947c478bd9Sstevel@tonic-gate extern int use_sse_pagecopy, use_sse_pagezero; /* in ml/float.s */ 957c478bd9Sstevel@tonic-gate 967c478bd9Sstevel@tonic-gate /* 4g memory management */ 977c478bd9Sstevel@tonic-gate pgcnt_t maxmem4g; 987c478bd9Sstevel@tonic-gate pgcnt_t freemem4g; 997c478bd9Sstevel@tonic-gate int physmax4g; 1007c478bd9Sstevel@tonic-gate int desfree4gshift = 4; /* maxmem4g shift to derive DESFREE4G */ 1017c478bd9Sstevel@tonic-gate int lotsfree4gshift = 3; 1027c478bd9Sstevel@tonic-gate 10307ad560dSkchow /* 16m memory management: desired number of free pages below 16m. */ 10407ad560dSkchow pgcnt_t desfree16m = 0x380; 10507ad560dSkchow 1067c478bd9Sstevel@tonic-gate #ifdef VM_STATS 1077c478bd9Sstevel@tonic-gate struct { 1087c478bd9Sstevel@tonic-gate ulong_t pga_alloc; 1097c478bd9Sstevel@tonic-gate ulong_t pga_notfullrange; 1107c478bd9Sstevel@tonic-gate ulong_t pga_nulldmaattr; 1117c478bd9Sstevel@tonic-gate ulong_t pga_allocok; 1127c478bd9Sstevel@tonic-gate ulong_t pga_allocfailed; 1137c478bd9Sstevel@tonic-gate ulong_t pgma_alloc; 1147c478bd9Sstevel@tonic-gate ulong_t pgma_allocok; 1157c478bd9Sstevel@tonic-gate ulong_t pgma_allocfailed; 1167c478bd9Sstevel@tonic-gate ulong_t pgma_allocempty; 1177c478bd9Sstevel@tonic-gate } pga_vmstats; 1187c478bd9Sstevel@tonic-gate #endif 1197c478bd9Sstevel@tonic-gate 1207c478bd9Sstevel@tonic-gate uint_t mmu_page_sizes; 1217c478bd9Sstevel@tonic-gate 1227c478bd9Sstevel@tonic-gate /* How many page sizes the users can see */ 1237c478bd9Sstevel@tonic-gate uint_t mmu_exported_page_sizes; 1247c478bd9Sstevel@tonic-gate 1257c478bd9Sstevel@tonic-gate size_t auto_lpg_va_default = MMU_PAGESIZE; /* used by zmap() */ 126beb1bda0Sdavemq /* 127beb1bda0Sdavemq * Number of pages in 1 GB. Don't enable automatic large pages if we have 128beb1bda0Sdavemq * fewer than this many pages. 129beb1bda0Sdavemq */ 130beb1bda0Sdavemq pgcnt_t auto_lpg_min_physmem = 1 << (30 - MMU_PAGESHIFT); 1317c478bd9Sstevel@tonic-gate 1327c478bd9Sstevel@tonic-gate /* 1337c478bd9Sstevel@tonic-gate * Return the optimum page size for a given mapping 1347c478bd9Sstevel@tonic-gate */ 1357c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 1367c478bd9Sstevel@tonic-gate size_t 1377c478bd9Sstevel@tonic-gate map_pgsz(int maptype, struct proc *p, caddr_t addr, size_t len, int *remap) 1387c478bd9Sstevel@tonic-gate { 1397c478bd9Sstevel@tonic-gate level_t l; 1407c478bd9Sstevel@tonic-gate 1417c478bd9Sstevel@tonic-gate if (remap) 1427c478bd9Sstevel@tonic-gate *remap = 0; 1437c478bd9Sstevel@tonic-gate 1447c478bd9Sstevel@tonic-gate switch (maptype) { 1457c478bd9Sstevel@tonic-gate 1467c478bd9Sstevel@tonic-gate case MAPPGSZ_STK: 1477c478bd9Sstevel@tonic-gate case MAPPGSZ_HEAP: 1487c478bd9Sstevel@tonic-gate case MAPPGSZ_VA: 1497c478bd9Sstevel@tonic-gate /* 1507c478bd9Sstevel@tonic-gate * use the pages size that best fits len 1517c478bd9Sstevel@tonic-gate */ 1527c478bd9Sstevel@tonic-gate for (l = mmu.max_page_level; l > 0; --l) { 1537c478bd9Sstevel@tonic-gate if (len < LEVEL_SIZE(l)) 1547c478bd9Sstevel@tonic-gate continue; 1557c478bd9Sstevel@tonic-gate break; 1567c478bd9Sstevel@tonic-gate } 1577c478bd9Sstevel@tonic-gate return (LEVEL_SIZE(l)); 1587c478bd9Sstevel@tonic-gate 1597c478bd9Sstevel@tonic-gate /* 1607c478bd9Sstevel@tonic-gate * for ISM use the 1st large page size. 1617c478bd9Sstevel@tonic-gate */ 1627c478bd9Sstevel@tonic-gate case MAPPGSZ_ISM: 1637c478bd9Sstevel@tonic-gate if (mmu.max_page_level == 0) 1647c478bd9Sstevel@tonic-gate return (MMU_PAGESIZE); 1657c478bd9Sstevel@tonic-gate return (LEVEL_SIZE(1)); 1667c478bd9Sstevel@tonic-gate } 1677c478bd9Sstevel@tonic-gate return (0); 1687c478bd9Sstevel@tonic-gate } 1697c478bd9Sstevel@tonic-gate 1707c478bd9Sstevel@tonic-gate /* 1717c478bd9Sstevel@tonic-gate * This can be patched via /etc/system to allow large pages 1727c478bd9Sstevel@tonic-gate * to be used for mapping application and libraries text segments. 1737c478bd9Sstevel@tonic-gate */ 1747c478bd9Sstevel@tonic-gate int use_text_largepages = 0; 17507b65a64Saguzovsk int use_shm_largepages = 0; 1767c478bd9Sstevel@tonic-gate 1777c478bd9Sstevel@tonic-gate /* 1787c478bd9Sstevel@tonic-gate * Return a bit vector of large page size codes that 1797c478bd9Sstevel@tonic-gate * can be used to map [addr, addr + len) region. 1807c478bd9Sstevel@tonic-gate */ 1817c478bd9Sstevel@tonic-gate 1827c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 1837c478bd9Sstevel@tonic-gate uint_t 1847c478bd9Sstevel@tonic-gate map_execseg_pgszcvec(int text, caddr_t addr, size_t len) 1857c478bd9Sstevel@tonic-gate { 1867c478bd9Sstevel@tonic-gate size_t pgsz; 1877c478bd9Sstevel@tonic-gate caddr_t a; 1887c478bd9Sstevel@tonic-gate 1897c478bd9Sstevel@tonic-gate if (!text || !use_text_largepages || 1907c478bd9Sstevel@tonic-gate mmu.max_page_level == 0) 1917c478bd9Sstevel@tonic-gate return (0); 1927c478bd9Sstevel@tonic-gate 1937c478bd9Sstevel@tonic-gate pgsz = LEVEL_SIZE(1); 1947c478bd9Sstevel@tonic-gate a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz); 1957c478bd9Sstevel@tonic-gate if (a < addr || a >= addr + len) { 1967c478bd9Sstevel@tonic-gate return (0); 1977c478bd9Sstevel@tonic-gate } 1987c478bd9Sstevel@tonic-gate len -= (a - addr); 1997c478bd9Sstevel@tonic-gate if (len < pgsz) { 2007c478bd9Sstevel@tonic-gate return (0); 2017c478bd9Sstevel@tonic-gate } 2027c478bd9Sstevel@tonic-gate return (1 << 1); 2037c478bd9Sstevel@tonic-gate } 2047c478bd9Sstevel@tonic-gate 20507b65a64Saguzovsk uint_t 20607b65a64Saguzovsk map_shm_pgszcvec(caddr_t addr, size_t len, uintptr_t off) 20707b65a64Saguzovsk { 20807b65a64Saguzovsk size_t pgsz; 20907b65a64Saguzovsk caddr_t a; 21007b65a64Saguzovsk 21107b65a64Saguzovsk if (!use_shm_largepages || mmu.max_page_level == 0) { 21207b65a64Saguzovsk return (0); 21307b65a64Saguzovsk } 21407b65a64Saguzovsk 21507b65a64Saguzovsk pgsz = LEVEL_SIZE(1); 21607b65a64Saguzovsk a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz); 21707b65a64Saguzovsk if (a < addr || a >= addr + len || 21807b65a64Saguzovsk P2PHASE((uintptr_t)addr ^ off, pgsz)) { 21907b65a64Saguzovsk return (0); 22007b65a64Saguzovsk } 22107b65a64Saguzovsk len -= (a - addr); 22207b65a64Saguzovsk if (len < pgsz) { 22307b65a64Saguzovsk return (0); 22407b65a64Saguzovsk } 22507b65a64Saguzovsk return (1 << 1); 22607b65a64Saguzovsk } 22707b65a64Saguzovsk 2287c478bd9Sstevel@tonic-gate /* 2297c478bd9Sstevel@tonic-gate * Handle a pagefault. 2307c478bd9Sstevel@tonic-gate */ 2317c478bd9Sstevel@tonic-gate faultcode_t 2327c478bd9Sstevel@tonic-gate pagefault( 2337c478bd9Sstevel@tonic-gate caddr_t addr, 2347c478bd9Sstevel@tonic-gate enum fault_type type, 2357c478bd9Sstevel@tonic-gate enum seg_rw rw, 2367c478bd9Sstevel@tonic-gate int iskernel) 2377c478bd9Sstevel@tonic-gate { 2387c478bd9Sstevel@tonic-gate struct as *as; 2397c478bd9Sstevel@tonic-gate struct hat *hat; 2407c478bd9Sstevel@tonic-gate struct proc *p; 2417c478bd9Sstevel@tonic-gate kthread_t *t; 2427c478bd9Sstevel@tonic-gate faultcode_t res; 2437c478bd9Sstevel@tonic-gate caddr_t base; 2447c478bd9Sstevel@tonic-gate size_t len; 2457c478bd9Sstevel@tonic-gate int err; 2467c478bd9Sstevel@tonic-gate int mapped_red; 2477c478bd9Sstevel@tonic-gate uintptr_t ea; 2487c478bd9Sstevel@tonic-gate 2497c478bd9Sstevel@tonic-gate ASSERT_STACK_ALIGNED(); 2507c478bd9Sstevel@tonic-gate 2517c478bd9Sstevel@tonic-gate if (INVALID_VADDR(addr)) 2527c478bd9Sstevel@tonic-gate return (FC_NOMAP); 2537c478bd9Sstevel@tonic-gate 2547c478bd9Sstevel@tonic-gate mapped_red = segkp_map_red(); 2557c478bd9Sstevel@tonic-gate 2567c478bd9Sstevel@tonic-gate if (iskernel) { 2577c478bd9Sstevel@tonic-gate as = &kas; 2587c478bd9Sstevel@tonic-gate hat = as->a_hat; 2597c478bd9Sstevel@tonic-gate } else { 2607c478bd9Sstevel@tonic-gate t = curthread; 2617c478bd9Sstevel@tonic-gate p = ttoproc(t); 2627c478bd9Sstevel@tonic-gate as = p->p_as; 2637c478bd9Sstevel@tonic-gate hat = as->a_hat; 2647c478bd9Sstevel@tonic-gate } 2657c478bd9Sstevel@tonic-gate 2667c478bd9Sstevel@tonic-gate /* 2677c478bd9Sstevel@tonic-gate * Dispatch pagefault. 2687c478bd9Sstevel@tonic-gate */ 2697c478bd9Sstevel@tonic-gate res = as_fault(hat, as, addr, 1, type, rw); 2707c478bd9Sstevel@tonic-gate 2717c478bd9Sstevel@tonic-gate /* 2727c478bd9Sstevel@tonic-gate * If this isn't a potential unmapped hole in the user's 2737c478bd9Sstevel@tonic-gate * UNIX data or stack segments, just return status info. 2747c478bd9Sstevel@tonic-gate */ 2757c478bd9Sstevel@tonic-gate if (res != FC_NOMAP || iskernel) 2767c478bd9Sstevel@tonic-gate goto out; 2777c478bd9Sstevel@tonic-gate 2787c478bd9Sstevel@tonic-gate /* 2797c478bd9Sstevel@tonic-gate * Check to see if we happened to faulted on a currently unmapped 2807c478bd9Sstevel@tonic-gate * part of the UNIX data or stack segments. If so, create a zfod 2817c478bd9Sstevel@tonic-gate * mapping there and then try calling the fault routine again. 2827c478bd9Sstevel@tonic-gate */ 2837c478bd9Sstevel@tonic-gate base = p->p_brkbase; 2847c478bd9Sstevel@tonic-gate len = p->p_brksize; 2857c478bd9Sstevel@tonic-gate 2867c478bd9Sstevel@tonic-gate if (addr < base || addr >= base + len) { /* data seg? */ 2877c478bd9Sstevel@tonic-gate base = (caddr_t)p->p_usrstack - p->p_stksize; 2887c478bd9Sstevel@tonic-gate len = p->p_stksize; 2897c478bd9Sstevel@tonic-gate if (addr < base || addr >= p->p_usrstack) { /* stack seg? */ 2907c478bd9Sstevel@tonic-gate /* not in either UNIX data or stack segments */ 2917c478bd9Sstevel@tonic-gate res = FC_NOMAP; 2927c478bd9Sstevel@tonic-gate goto out; 2937c478bd9Sstevel@tonic-gate } 2947c478bd9Sstevel@tonic-gate } 2957c478bd9Sstevel@tonic-gate 2967c478bd9Sstevel@tonic-gate /* 2977c478bd9Sstevel@tonic-gate * the rest of this function implements a 3.X 4.X 5.X compatibility 2987c478bd9Sstevel@tonic-gate * This code is probably not needed anymore 2997c478bd9Sstevel@tonic-gate */ 3007c478bd9Sstevel@tonic-gate if (p->p_model == DATAMODEL_ILP32) { 3017c478bd9Sstevel@tonic-gate 3027c478bd9Sstevel@tonic-gate /* expand the gap to the page boundaries on each side */ 3037c478bd9Sstevel@tonic-gate ea = P2ROUNDUP((uintptr_t)base + len, MMU_PAGESIZE); 3047c478bd9Sstevel@tonic-gate base = (caddr_t)P2ALIGN((uintptr_t)base, MMU_PAGESIZE); 3057c478bd9Sstevel@tonic-gate len = ea - (uintptr_t)base; 3067c478bd9Sstevel@tonic-gate 3077c478bd9Sstevel@tonic-gate as_rangelock(as); 3087c478bd9Sstevel@tonic-gate if (as_gap(as, MMU_PAGESIZE, &base, &len, AH_CONTAIN, addr) == 3097c478bd9Sstevel@tonic-gate 0) { 3107c478bd9Sstevel@tonic-gate err = as_map(as, base, len, segvn_create, zfod_argsp); 3117c478bd9Sstevel@tonic-gate as_rangeunlock(as); 3127c478bd9Sstevel@tonic-gate if (err) { 3137c478bd9Sstevel@tonic-gate res = FC_MAKE_ERR(err); 3147c478bd9Sstevel@tonic-gate goto out; 3157c478bd9Sstevel@tonic-gate } 3167c478bd9Sstevel@tonic-gate } else { 3177c478bd9Sstevel@tonic-gate /* 3187c478bd9Sstevel@tonic-gate * This page is already mapped by another thread after 3197c478bd9Sstevel@tonic-gate * we returned from as_fault() above. We just fall 3207c478bd9Sstevel@tonic-gate * through as_fault() below. 3217c478bd9Sstevel@tonic-gate */ 3227c478bd9Sstevel@tonic-gate as_rangeunlock(as); 3237c478bd9Sstevel@tonic-gate } 3247c478bd9Sstevel@tonic-gate 3257c478bd9Sstevel@tonic-gate res = as_fault(hat, as, addr, 1, F_INVAL, rw); 3267c478bd9Sstevel@tonic-gate } 3277c478bd9Sstevel@tonic-gate 3287c478bd9Sstevel@tonic-gate out: 3297c478bd9Sstevel@tonic-gate if (mapped_red) 3307c478bd9Sstevel@tonic-gate segkp_unmap_red(); 3317c478bd9Sstevel@tonic-gate 3327c478bd9Sstevel@tonic-gate return (res); 3337c478bd9Sstevel@tonic-gate } 3347c478bd9Sstevel@tonic-gate 3357c478bd9Sstevel@tonic-gate void 3367c478bd9Sstevel@tonic-gate map_addr(caddr_t *addrp, size_t len, offset_t off, int vacalign, uint_t flags) 3377c478bd9Sstevel@tonic-gate { 3387c478bd9Sstevel@tonic-gate struct proc *p = curproc; 3397c478bd9Sstevel@tonic-gate caddr_t userlimit = (flags & _MAP_LOW32) ? 3407c478bd9Sstevel@tonic-gate (caddr_t)_userlimit32 : p->p_as->a_userlimit; 3417c478bd9Sstevel@tonic-gate 3427c478bd9Sstevel@tonic-gate map_addr_proc(addrp, len, off, vacalign, userlimit, curproc, flags); 3437c478bd9Sstevel@tonic-gate } 3447c478bd9Sstevel@tonic-gate 3457c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 3467c478bd9Sstevel@tonic-gate int 3477c478bd9Sstevel@tonic-gate map_addr_vacalign_check(caddr_t addr, u_offset_t off) 3487c478bd9Sstevel@tonic-gate { 3497c478bd9Sstevel@tonic-gate return (0); 3507c478bd9Sstevel@tonic-gate } 3517c478bd9Sstevel@tonic-gate 3527c478bd9Sstevel@tonic-gate /* 3537c478bd9Sstevel@tonic-gate * map_addr_proc() is the routine called when the system is to 3547c478bd9Sstevel@tonic-gate * choose an address for the user. We will pick an address 3557c478bd9Sstevel@tonic-gate * range which is the highest available below kernelbase. 3567c478bd9Sstevel@tonic-gate * 3577c478bd9Sstevel@tonic-gate * addrp is a value/result parameter. 3587c478bd9Sstevel@tonic-gate * On input it is a hint from the user to be used in a completely 3597c478bd9Sstevel@tonic-gate * machine dependent fashion. We decide to completely ignore this hint. 3607c478bd9Sstevel@tonic-gate * 3617c478bd9Sstevel@tonic-gate * On output it is NULL if no address can be found in the current 3627c478bd9Sstevel@tonic-gate * processes address space or else an address that is currently 3637c478bd9Sstevel@tonic-gate * not mapped for len bytes with a page of red zone on either side. 3647c478bd9Sstevel@tonic-gate * 3657c478bd9Sstevel@tonic-gate * align is not needed on x86 (it's for viturally addressed caches) 3667c478bd9Sstevel@tonic-gate */ 3677c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 3687c478bd9Sstevel@tonic-gate void 3697c478bd9Sstevel@tonic-gate map_addr_proc( 3707c478bd9Sstevel@tonic-gate caddr_t *addrp, 3717c478bd9Sstevel@tonic-gate size_t len, 3727c478bd9Sstevel@tonic-gate offset_t off, 3737c478bd9Sstevel@tonic-gate int vacalign, 3747c478bd9Sstevel@tonic-gate caddr_t userlimit, 3757c478bd9Sstevel@tonic-gate struct proc *p, 3767c478bd9Sstevel@tonic-gate uint_t flags) 3777c478bd9Sstevel@tonic-gate { 3787c478bd9Sstevel@tonic-gate struct as *as = p->p_as; 3797c478bd9Sstevel@tonic-gate caddr_t addr; 3807c478bd9Sstevel@tonic-gate caddr_t base; 3817c478bd9Sstevel@tonic-gate size_t slen; 3827c478bd9Sstevel@tonic-gate size_t align_amount; 3837c478bd9Sstevel@tonic-gate 3847c478bd9Sstevel@tonic-gate ASSERT32(userlimit == as->a_userlimit); 3857c478bd9Sstevel@tonic-gate 3867c478bd9Sstevel@tonic-gate base = p->p_brkbase; 3877c478bd9Sstevel@tonic-gate #if defined(__amd64) 3887c478bd9Sstevel@tonic-gate /* 3897c478bd9Sstevel@tonic-gate * XX64 Yes, this needs more work. 3907c478bd9Sstevel@tonic-gate */ 3917c478bd9Sstevel@tonic-gate if (p->p_model == DATAMODEL_NATIVE) { 3927c478bd9Sstevel@tonic-gate if (userlimit < as->a_userlimit) { 3937c478bd9Sstevel@tonic-gate /* 3947c478bd9Sstevel@tonic-gate * This happens when a program wants to map 3957c478bd9Sstevel@tonic-gate * something in a range that's accessible to a 3967c478bd9Sstevel@tonic-gate * program in a smaller address space. For example, 3977c478bd9Sstevel@tonic-gate * a 64-bit program calling mmap32(2) to guarantee 3987c478bd9Sstevel@tonic-gate * that the returned address is below 4Gbytes. 3997c478bd9Sstevel@tonic-gate */ 4007c478bd9Sstevel@tonic-gate ASSERT((uintptr_t)userlimit < ADDRESS_C(0xffffffff)); 4017c478bd9Sstevel@tonic-gate 4027c478bd9Sstevel@tonic-gate if (userlimit > base) 4037c478bd9Sstevel@tonic-gate slen = userlimit - base; 4047c478bd9Sstevel@tonic-gate else { 4057c478bd9Sstevel@tonic-gate *addrp = NULL; 4067c478bd9Sstevel@tonic-gate return; 4077c478bd9Sstevel@tonic-gate } 4087c478bd9Sstevel@tonic-gate } else { 4097c478bd9Sstevel@tonic-gate /* 4107c478bd9Sstevel@tonic-gate * XX64 This layout is probably wrong .. but in 4117c478bd9Sstevel@tonic-gate * the event we make the amd64 address space look 4127c478bd9Sstevel@tonic-gate * like sparcv9 i.e. with the stack -above- the 4137c478bd9Sstevel@tonic-gate * heap, this bit of code might even be correct. 4147c478bd9Sstevel@tonic-gate */ 4157c478bd9Sstevel@tonic-gate slen = p->p_usrstack - base - 4167c478bd9Sstevel@tonic-gate (((size_t)rctl_enforced_value( 4177c478bd9Sstevel@tonic-gate rctlproc_legacy[RLIMIT_STACK], 4187c478bd9Sstevel@tonic-gate p->p_rctls, p) + PAGEOFFSET) & PAGEMASK); 4197c478bd9Sstevel@tonic-gate } 4207c478bd9Sstevel@tonic-gate } else 4217c478bd9Sstevel@tonic-gate #endif 4227c478bd9Sstevel@tonic-gate slen = userlimit - base; 4237c478bd9Sstevel@tonic-gate 4247c478bd9Sstevel@tonic-gate len = (len + PAGEOFFSET) & PAGEMASK; 4257c478bd9Sstevel@tonic-gate 4267c478bd9Sstevel@tonic-gate /* 4277c478bd9Sstevel@tonic-gate * Redzone for each side of the request. This is done to leave 4287c478bd9Sstevel@tonic-gate * one page unmapped between segments. This is not required, but 4297c478bd9Sstevel@tonic-gate * it's useful for the user because if their program strays across 4307c478bd9Sstevel@tonic-gate * a segment boundary, it will catch a fault immediately making 4317c478bd9Sstevel@tonic-gate * debugging a little easier. 4327c478bd9Sstevel@tonic-gate */ 4337c478bd9Sstevel@tonic-gate len += 2 * MMU_PAGESIZE; 4347c478bd9Sstevel@tonic-gate 4357c478bd9Sstevel@tonic-gate /* 4367c478bd9Sstevel@tonic-gate * figure out what the alignment should be 4377c478bd9Sstevel@tonic-gate * 4387c478bd9Sstevel@tonic-gate * XX64 -- is there an ELF_AMD64_MAXPGSZ or is it the same???? 4397c478bd9Sstevel@tonic-gate */ 4407c478bd9Sstevel@tonic-gate if (len <= ELF_386_MAXPGSZ) { 4417c478bd9Sstevel@tonic-gate /* 4427c478bd9Sstevel@tonic-gate * Align virtual addresses to ensure that ELF shared libraries 4437c478bd9Sstevel@tonic-gate * are mapped with the appropriate alignment constraints by 4447c478bd9Sstevel@tonic-gate * the run-time linker. 4457c478bd9Sstevel@tonic-gate */ 4467c478bd9Sstevel@tonic-gate align_amount = ELF_386_MAXPGSZ; 4477c478bd9Sstevel@tonic-gate } else { 4487c478bd9Sstevel@tonic-gate int l = mmu.max_page_level; 4497c478bd9Sstevel@tonic-gate 4507c478bd9Sstevel@tonic-gate while (l && len < LEVEL_SIZE(l)) 4517c478bd9Sstevel@tonic-gate --l; 4527c478bd9Sstevel@tonic-gate 4537c478bd9Sstevel@tonic-gate align_amount = LEVEL_SIZE(l); 4547c478bd9Sstevel@tonic-gate } 4557c478bd9Sstevel@tonic-gate 4567c478bd9Sstevel@tonic-gate if ((flags & MAP_ALIGN) && ((uintptr_t)*addrp > align_amount)) 4577c478bd9Sstevel@tonic-gate align_amount = (uintptr_t)*addrp; 4587c478bd9Sstevel@tonic-gate 4597c478bd9Sstevel@tonic-gate len += align_amount; 4607c478bd9Sstevel@tonic-gate 4617c478bd9Sstevel@tonic-gate /* 4627c478bd9Sstevel@tonic-gate * Look for a large enough hole starting below userlimit. 4637c478bd9Sstevel@tonic-gate * After finding it, use the upper part. Addition of PAGESIZE 4647c478bd9Sstevel@tonic-gate * is for the redzone as described above. 4657c478bd9Sstevel@tonic-gate */ 4667c478bd9Sstevel@tonic-gate if (as_gap(as, len, &base, &slen, AH_HI, NULL) == 0) { 4677c478bd9Sstevel@tonic-gate caddr_t as_addr; 4687c478bd9Sstevel@tonic-gate 4697c478bd9Sstevel@tonic-gate addr = base + slen - len + MMU_PAGESIZE; 4707c478bd9Sstevel@tonic-gate as_addr = addr; 4717c478bd9Sstevel@tonic-gate /* 4727c478bd9Sstevel@tonic-gate * Round address DOWN to the alignment amount, 4737c478bd9Sstevel@tonic-gate * add the offset, and if this address is less 4747c478bd9Sstevel@tonic-gate * than the original address, add alignment amount. 4757c478bd9Sstevel@tonic-gate */ 4767c478bd9Sstevel@tonic-gate addr = (caddr_t)((uintptr_t)addr & (~(align_amount - 1))); 4777c478bd9Sstevel@tonic-gate addr += (uintptr_t)(off & (align_amount - 1)); 4787c478bd9Sstevel@tonic-gate if (addr < as_addr) 4797c478bd9Sstevel@tonic-gate addr += align_amount; 4807c478bd9Sstevel@tonic-gate 4817c478bd9Sstevel@tonic-gate ASSERT(addr <= (as_addr + align_amount)); 4827c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)addr & (align_amount - 1)) == 4837c478bd9Sstevel@tonic-gate ((uintptr_t)(off & (align_amount - 1)))); 4847c478bd9Sstevel@tonic-gate *addrp = addr; 4857c478bd9Sstevel@tonic-gate } else { 4867c478bd9Sstevel@tonic-gate *addrp = NULL; /* no more virtual space */ 4877c478bd9Sstevel@tonic-gate } 4887c478bd9Sstevel@tonic-gate } 4897c478bd9Sstevel@tonic-gate 4907c478bd9Sstevel@tonic-gate /* 4917c478bd9Sstevel@tonic-gate * Determine whether [base, base+len] contains a valid range of 4927c478bd9Sstevel@tonic-gate * addresses at least minlen long. base and len are adjusted if 4937c478bd9Sstevel@tonic-gate * required to provide a valid range. 4947c478bd9Sstevel@tonic-gate */ 4957c478bd9Sstevel@tonic-gate /*ARGSUSED3*/ 4967c478bd9Sstevel@tonic-gate int 4977c478bd9Sstevel@tonic-gate valid_va_range(caddr_t *basep, size_t *lenp, size_t minlen, int dir) 4987c478bd9Sstevel@tonic-gate { 4997c478bd9Sstevel@tonic-gate uintptr_t hi, lo; 5007c478bd9Sstevel@tonic-gate 5017c478bd9Sstevel@tonic-gate lo = (uintptr_t)*basep; 5027c478bd9Sstevel@tonic-gate hi = lo + *lenp; 5037c478bd9Sstevel@tonic-gate 5047c478bd9Sstevel@tonic-gate /* 5057c478bd9Sstevel@tonic-gate * If hi rolled over the top, try cutting back. 5067c478bd9Sstevel@tonic-gate */ 5077c478bd9Sstevel@tonic-gate if (hi < lo) { 5087c478bd9Sstevel@tonic-gate if (0 - lo + hi < minlen) 5097c478bd9Sstevel@tonic-gate return (0); 5107c478bd9Sstevel@tonic-gate if (0 - lo < minlen) 5117c478bd9Sstevel@tonic-gate return (0); 5127c478bd9Sstevel@tonic-gate *lenp = 0 - lo; 5137c478bd9Sstevel@tonic-gate } else if (hi - lo < minlen) { 5147c478bd9Sstevel@tonic-gate return (0); 5157c478bd9Sstevel@tonic-gate } 5167c478bd9Sstevel@tonic-gate #if defined(__amd64) 5177c478bd9Sstevel@tonic-gate /* 5187c478bd9Sstevel@tonic-gate * Deal with a possible hole in the address range between 5197c478bd9Sstevel@tonic-gate * hole_start and hole_end that should never be mapped. 5207c478bd9Sstevel@tonic-gate */ 5217c478bd9Sstevel@tonic-gate if (lo < hole_start) { 5227c478bd9Sstevel@tonic-gate if (hi > hole_start) { 5237c478bd9Sstevel@tonic-gate if (hi < hole_end) { 5247c478bd9Sstevel@tonic-gate hi = hole_start; 5257c478bd9Sstevel@tonic-gate } else { 5267c478bd9Sstevel@tonic-gate /* lo < hole_start && hi >= hole_end */ 5277c478bd9Sstevel@tonic-gate if (dir == AH_LO) { 5287c478bd9Sstevel@tonic-gate /* 5297c478bd9Sstevel@tonic-gate * prefer lowest range 5307c478bd9Sstevel@tonic-gate */ 5317c478bd9Sstevel@tonic-gate if (hole_start - lo >= minlen) 5327c478bd9Sstevel@tonic-gate hi = hole_start; 5337c478bd9Sstevel@tonic-gate else if (hi - hole_end >= minlen) 5347c478bd9Sstevel@tonic-gate lo = hole_end; 5357c478bd9Sstevel@tonic-gate else 5367c478bd9Sstevel@tonic-gate return (0); 5377c478bd9Sstevel@tonic-gate } else { 5387c478bd9Sstevel@tonic-gate /* 5397c478bd9Sstevel@tonic-gate * prefer highest range 5407c478bd9Sstevel@tonic-gate */ 5417c478bd9Sstevel@tonic-gate if (hi - hole_end >= minlen) 5427c478bd9Sstevel@tonic-gate lo = hole_end; 5437c478bd9Sstevel@tonic-gate else if (hole_start - lo >= minlen) 5447c478bd9Sstevel@tonic-gate hi = hole_start; 5457c478bd9Sstevel@tonic-gate else 5467c478bd9Sstevel@tonic-gate return (0); 5477c478bd9Sstevel@tonic-gate } 5487c478bd9Sstevel@tonic-gate } 5497c478bd9Sstevel@tonic-gate } 5507c478bd9Sstevel@tonic-gate } else { 5517c478bd9Sstevel@tonic-gate /* lo >= hole_start */ 5527c478bd9Sstevel@tonic-gate if (hi < hole_end) 5537c478bd9Sstevel@tonic-gate return (0); 5547c478bd9Sstevel@tonic-gate if (lo < hole_end) 5557c478bd9Sstevel@tonic-gate lo = hole_end; 5567c478bd9Sstevel@tonic-gate } 5577c478bd9Sstevel@tonic-gate 5587c478bd9Sstevel@tonic-gate if (hi - lo < minlen) 5597c478bd9Sstevel@tonic-gate return (0); 5607c478bd9Sstevel@tonic-gate 5617c478bd9Sstevel@tonic-gate *basep = (caddr_t)lo; 5627c478bd9Sstevel@tonic-gate *lenp = hi - lo; 5637c478bd9Sstevel@tonic-gate #endif 5647c478bd9Sstevel@tonic-gate return (1); 5657c478bd9Sstevel@tonic-gate } 5667c478bd9Sstevel@tonic-gate 5677c478bd9Sstevel@tonic-gate /* 5687c478bd9Sstevel@tonic-gate * Determine whether [addr, addr+len] are valid user addresses. 5697c478bd9Sstevel@tonic-gate */ 5707c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 5717c478bd9Sstevel@tonic-gate int 5727c478bd9Sstevel@tonic-gate valid_usr_range(caddr_t addr, size_t len, uint_t prot, struct as *as, 5737c478bd9Sstevel@tonic-gate caddr_t userlimit) 5747c478bd9Sstevel@tonic-gate { 5757c478bd9Sstevel@tonic-gate caddr_t eaddr = addr + len; 5767c478bd9Sstevel@tonic-gate 5777c478bd9Sstevel@tonic-gate if (eaddr <= addr || addr >= userlimit || eaddr > userlimit) 5787c478bd9Sstevel@tonic-gate return (RANGE_BADADDR); 5797c478bd9Sstevel@tonic-gate 5807c478bd9Sstevel@tonic-gate #if defined(__amd64) 5817c478bd9Sstevel@tonic-gate /* 5827c478bd9Sstevel@tonic-gate * Check for the VA hole 5837c478bd9Sstevel@tonic-gate */ 5847c478bd9Sstevel@tonic-gate if (eaddr > (caddr_t)hole_start && addr < (caddr_t)hole_end) 5857c478bd9Sstevel@tonic-gate return (RANGE_BADADDR); 5867c478bd9Sstevel@tonic-gate #endif 5877c478bd9Sstevel@tonic-gate 5887c478bd9Sstevel@tonic-gate return (RANGE_OKAY); 5897c478bd9Sstevel@tonic-gate } 5907c478bd9Sstevel@tonic-gate 5917c478bd9Sstevel@tonic-gate /* 5927c478bd9Sstevel@tonic-gate * Return 1 if the page frame is onboard memory, else 0. 5937c478bd9Sstevel@tonic-gate */ 5947c478bd9Sstevel@tonic-gate int 5957c478bd9Sstevel@tonic-gate pf_is_memory(pfn_t pf) 5967c478bd9Sstevel@tonic-gate { 5977c478bd9Sstevel@tonic-gate return (address_in_memlist(phys_install, mmu_ptob((uint64_t)pf), 1)); 5987c478bd9Sstevel@tonic-gate } 5997c478bd9Sstevel@tonic-gate 6007c478bd9Sstevel@tonic-gate 6017c478bd9Sstevel@tonic-gate /* 6027c478bd9Sstevel@tonic-gate * initialized by page_coloring_init(). 6037c478bd9Sstevel@tonic-gate */ 6047c478bd9Sstevel@tonic-gate uint_t page_colors; 6057c478bd9Sstevel@tonic-gate uint_t page_colors_mask; 6067c478bd9Sstevel@tonic-gate uint_t page_coloring_shift; 6077c478bd9Sstevel@tonic-gate int cpu_page_colors; 6087c478bd9Sstevel@tonic-gate static uint_t l2_colors; 6097c478bd9Sstevel@tonic-gate 6107c478bd9Sstevel@tonic-gate /* 6117c478bd9Sstevel@tonic-gate * Page freelists and cachelists are dynamically allocated once mnoderangecnt 6127c478bd9Sstevel@tonic-gate * and page_colors are calculated from the l2 cache n-way set size. Within a 6137c478bd9Sstevel@tonic-gate * mnode range, the page freelist and cachelist are hashed into bins based on 6147c478bd9Sstevel@tonic-gate * color. This makes it easier to search for a page within a specific memory 6157c478bd9Sstevel@tonic-gate * range. 6167c478bd9Sstevel@tonic-gate */ 6177c478bd9Sstevel@tonic-gate #define PAGE_COLORS_MIN 16 6187c478bd9Sstevel@tonic-gate 6197c478bd9Sstevel@tonic-gate page_t ****page_freelists; 6207c478bd9Sstevel@tonic-gate page_t ***page_cachelists; 6217c478bd9Sstevel@tonic-gate 6227c478bd9Sstevel@tonic-gate /* 6237c478bd9Sstevel@tonic-gate * As the PC architecture evolved memory up was clumped into several 6247c478bd9Sstevel@tonic-gate * ranges for various historical I/O devices to do DMA. 6257c478bd9Sstevel@tonic-gate * < 16Meg - ISA bus 6267c478bd9Sstevel@tonic-gate * < 2Gig - ??? 6277c478bd9Sstevel@tonic-gate * < 4Gig - PCI bus or drivers that don't understand PAE mode 6287c478bd9Sstevel@tonic-gate */ 6297c478bd9Sstevel@tonic-gate static pfn_t arch_memranges[NUM_MEM_RANGES] = { 6307c478bd9Sstevel@tonic-gate 0x100000, /* pfn range for 4G and above */ 6317c478bd9Sstevel@tonic-gate 0x80000, /* pfn range for 2G-4G */ 6327c478bd9Sstevel@tonic-gate 0x01000, /* pfn range for 16M-2G */ 6337c478bd9Sstevel@tonic-gate 0x00000, /* pfn range for 0-16M */ 6347c478bd9Sstevel@tonic-gate }; 6357c478bd9Sstevel@tonic-gate 6367c478bd9Sstevel@tonic-gate /* 6377c478bd9Sstevel@tonic-gate * These are changed during startup if the machine has limited memory. 6387c478bd9Sstevel@tonic-gate */ 6397c478bd9Sstevel@tonic-gate pfn_t *memranges = &arch_memranges[0]; 6407c478bd9Sstevel@tonic-gate int nranges = NUM_MEM_RANGES; 6417c478bd9Sstevel@tonic-gate 6427c478bd9Sstevel@tonic-gate /* 6437c478bd9Sstevel@tonic-gate * Used by page layer to know about page sizes 6447c478bd9Sstevel@tonic-gate */ 6457c478bd9Sstevel@tonic-gate hw_pagesize_t hw_page_array[MAX_NUM_LEVEL + 1]; 6467c478bd9Sstevel@tonic-gate 6477c478bd9Sstevel@tonic-gate /* 6487c478bd9Sstevel@tonic-gate * This can be patched via /etc/system to allow old non-PAE aware device 6497c478bd9Sstevel@tonic-gate * drivers to use kmem_alloc'd memory on 32 bit systems with > 4Gig RAM. 6507c478bd9Sstevel@tonic-gate */ 6517c478bd9Sstevel@tonic-gate #if defined(__i386) 652aa042c4bSkchow int restricted_kmemalloc = 0; 6537c478bd9Sstevel@tonic-gate #elif defined(__amd64) 6547c478bd9Sstevel@tonic-gate int restricted_kmemalloc = 0; 6557c478bd9Sstevel@tonic-gate #endif 6567c478bd9Sstevel@tonic-gate 6577c478bd9Sstevel@tonic-gate kmutex_t *fpc_mutex[NPC_MUTEX]; 6587c478bd9Sstevel@tonic-gate kmutex_t *cpc_mutex[NPC_MUTEX]; 6597c478bd9Sstevel@tonic-gate 6607c478bd9Sstevel@tonic-gate 6617c478bd9Sstevel@tonic-gate /* 6627c478bd9Sstevel@tonic-gate * return the memrange containing pfn 6637c478bd9Sstevel@tonic-gate */ 6647c478bd9Sstevel@tonic-gate int 6657c478bd9Sstevel@tonic-gate memrange_num(pfn_t pfn) 6667c478bd9Sstevel@tonic-gate { 6677c478bd9Sstevel@tonic-gate int n; 6687c478bd9Sstevel@tonic-gate 6697c478bd9Sstevel@tonic-gate for (n = 0; n < nranges - 1; ++n) { 6707c478bd9Sstevel@tonic-gate if (pfn >= memranges[n]) 6717c478bd9Sstevel@tonic-gate break; 6727c478bd9Sstevel@tonic-gate } 6737c478bd9Sstevel@tonic-gate return (n); 6747c478bd9Sstevel@tonic-gate } 6757c478bd9Sstevel@tonic-gate 6767c478bd9Sstevel@tonic-gate /* 6777c478bd9Sstevel@tonic-gate * return the mnoderange containing pfn 6787c478bd9Sstevel@tonic-gate */ 6797c478bd9Sstevel@tonic-gate int 6807c478bd9Sstevel@tonic-gate pfn_2_mtype(pfn_t pfn) 6817c478bd9Sstevel@tonic-gate { 6827c478bd9Sstevel@tonic-gate int n; 6837c478bd9Sstevel@tonic-gate 6847c478bd9Sstevel@tonic-gate for (n = mnoderangecnt - 1; n >= 0; n--) { 6857c478bd9Sstevel@tonic-gate if (pfn >= mnoderanges[n].mnr_pfnlo) { 6867c478bd9Sstevel@tonic-gate break; 6877c478bd9Sstevel@tonic-gate } 6887c478bd9Sstevel@tonic-gate } 6897c478bd9Sstevel@tonic-gate return (n); 6907c478bd9Sstevel@tonic-gate } 6917c478bd9Sstevel@tonic-gate 6927c478bd9Sstevel@tonic-gate /* 6937c478bd9Sstevel@tonic-gate * is_contigpage_free: 6947c478bd9Sstevel@tonic-gate * returns a page list of contiguous pages. It minimally has to return 6957c478bd9Sstevel@tonic-gate * minctg pages. Caller determines minctg based on the scatter-gather 6967c478bd9Sstevel@tonic-gate * list length. 6977c478bd9Sstevel@tonic-gate * 6987c478bd9Sstevel@tonic-gate * pfnp is set to the next page frame to search on return. 6997c478bd9Sstevel@tonic-gate */ 7007c478bd9Sstevel@tonic-gate static page_t * 7017c478bd9Sstevel@tonic-gate is_contigpage_free( 7027c478bd9Sstevel@tonic-gate pfn_t *pfnp, 7037c478bd9Sstevel@tonic-gate pgcnt_t *pgcnt, 7047c478bd9Sstevel@tonic-gate pgcnt_t minctg, 7057c478bd9Sstevel@tonic-gate uint64_t pfnseg, 7067c478bd9Sstevel@tonic-gate int iolock) 7077c478bd9Sstevel@tonic-gate { 7087c478bd9Sstevel@tonic-gate int i = 0; 7097c478bd9Sstevel@tonic-gate pfn_t pfn = *pfnp; 7107c478bd9Sstevel@tonic-gate page_t *pp; 7117c478bd9Sstevel@tonic-gate page_t *plist = NULL; 7127c478bd9Sstevel@tonic-gate 7137c478bd9Sstevel@tonic-gate /* 7147c478bd9Sstevel@tonic-gate * fail if pfn + minctg crosses a segment boundary. 7157c478bd9Sstevel@tonic-gate * Adjust for next starting pfn to begin at segment boundary. 7167c478bd9Sstevel@tonic-gate */ 7177c478bd9Sstevel@tonic-gate 7187c478bd9Sstevel@tonic-gate if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg)) { 7197c478bd9Sstevel@tonic-gate *pfnp = roundup(*pfnp, pfnseg + 1); 7207c478bd9Sstevel@tonic-gate return (NULL); 7217c478bd9Sstevel@tonic-gate } 7227c478bd9Sstevel@tonic-gate 7237c478bd9Sstevel@tonic-gate do { 7247c478bd9Sstevel@tonic-gate retry: 7257c478bd9Sstevel@tonic-gate pp = page_numtopp_nolock(pfn + i); 7267c478bd9Sstevel@tonic-gate if ((pp == NULL) || 7277c478bd9Sstevel@tonic-gate (page_trylock(pp, SE_EXCL) == 0)) { 7287c478bd9Sstevel@tonic-gate (*pfnp)++; 7297c478bd9Sstevel@tonic-gate break; 7307c478bd9Sstevel@tonic-gate } 7317c478bd9Sstevel@tonic-gate if (page_pptonum(pp) != pfn + i) { 7327c478bd9Sstevel@tonic-gate page_unlock(pp); 7337c478bd9Sstevel@tonic-gate goto retry; 7347c478bd9Sstevel@tonic-gate } 7357c478bd9Sstevel@tonic-gate 7367c478bd9Sstevel@tonic-gate if (!(PP_ISFREE(pp))) { 7377c478bd9Sstevel@tonic-gate page_unlock(pp); 7387c478bd9Sstevel@tonic-gate (*pfnp)++; 7397c478bd9Sstevel@tonic-gate break; 7407c478bd9Sstevel@tonic-gate } 7417c478bd9Sstevel@tonic-gate 7427c478bd9Sstevel@tonic-gate if (!PP_ISAGED(pp)) { 7437c478bd9Sstevel@tonic-gate page_list_sub(pp, PG_CACHE_LIST); 7447c478bd9Sstevel@tonic-gate page_hashout(pp, (kmutex_t *)NULL); 7457c478bd9Sstevel@tonic-gate } else { 7467c478bd9Sstevel@tonic-gate page_list_sub(pp, PG_FREE_LIST); 7477c478bd9Sstevel@tonic-gate } 7487c478bd9Sstevel@tonic-gate 7497c478bd9Sstevel@tonic-gate if (iolock) 7507c478bd9Sstevel@tonic-gate page_io_lock(pp); 7517c478bd9Sstevel@tonic-gate page_list_concat(&plist, &pp); 7527c478bd9Sstevel@tonic-gate 7537c478bd9Sstevel@tonic-gate /* 7547c478bd9Sstevel@tonic-gate * exit loop when pgcnt satisfied or segment boundary reached. 7557c478bd9Sstevel@tonic-gate */ 7567c478bd9Sstevel@tonic-gate 7577c478bd9Sstevel@tonic-gate } while ((++i < *pgcnt) && ((pfn + i) & pfnseg)); 7587c478bd9Sstevel@tonic-gate 7597c478bd9Sstevel@tonic-gate *pfnp += i; /* set to next pfn to search */ 7607c478bd9Sstevel@tonic-gate 7617c478bd9Sstevel@tonic-gate if (i >= minctg) { 7627c478bd9Sstevel@tonic-gate *pgcnt -= i; 7637c478bd9Sstevel@tonic-gate return (plist); 7647c478bd9Sstevel@tonic-gate } 7657c478bd9Sstevel@tonic-gate 7667c478bd9Sstevel@tonic-gate /* 7677c478bd9Sstevel@tonic-gate * failure: minctg not satisfied. 7687c478bd9Sstevel@tonic-gate * 7697c478bd9Sstevel@tonic-gate * if next request crosses segment boundary, set next pfn 7707c478bd9Sstevel@tonic-gate * to search from the segment boundary. 7717c478bd9Sstevel@tonic-gate */ 7727c478bd9Sstevel@tonic-gate if (((*pfnp + minctg - 1) & pfnseg) < (*pfnp & pfnseg)) 7737c478bd9Sstevel@tonic-gate *pfnp = roundup(*pfnp, pfnseg + 1); 7747c478bd9Sstevel@tonic-gate 7757c478bd9Sstevel@tonic-gate /* clean up any pages already allocated */ 7767c478bd9Sstevel@tonic-gate 7777c478bd9Sstevel@tonic-gate while (plist) { 7787c478bd9Sstevel@tonic-gate pp = plist; 7797c478bd9Sstevel@tonic-gate page_sub(&plist, pp); 7807c478bd9Sstevel@tonic-gate page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 7817c478bd9Sstevel@tonic-gate if (iolock) 7827c478bd9Sstevel@tonic-gate page_io_unlock(pp); 7837c478bd9Sstevel@tonic-gate page_unlock(pp); 7847c478bd9Sstevel@tonic-gate } 7857c478bd9Sstevel@tonic-gate 7867c478bd9Sstevel@tonic-gate return (NULL); 7877c478bd9Sstevel@tonic-gate } 7887c478bd9Sstevel@tonic-gate 7897c478bd9Sstevel@tonic-gate /* 7907c478bd9Sstevel@tonic-gate * verify that pages being returned from allocator have correct DMA attribute 7917c478bd9Sstevel@tonic-gate */ 7927c478bd9Sstevel@tonic-gate #ifndef DEBUG 7937c478bd9Sstevel@tonic-gate #define check_dma(a, b, c) (0) 7947c478bd9Sstevel@tonic-gate #else 7957c478bd9Sstevel@tonic-gate static void 7967c478bd9Sstevel@tonic-gate check_dma(ddi_dma_attr_t *dma_attr, page_t *pp, int cnt) 7977c478bd9Sstevel@tonic-gate { 7987c478bd9Sstevel@tonic-gate if (dma_attr == NULL) 7997c478bd9Sstevel@tonic-gate return; 8007c478bd9Sstevel@tonic-gate 8017c478bd9Sstevel@tonic-gate while (cnt-- > 0) { 8027c478bd9Sstevel@tonic-gate if (mmu_ptob((uint64_t)pp->p_pagenum) < 8037c478bd9Sstevel@tonic-gate dma_attr->dma_attr_addr_lo) 8047c478bd9Sstevel@tonic-gate panic("PFN (pp=%p) below dma_attr_addr_lo", pp); 8057c478bd9Sstevel@tonic-gate if (mmu_ptob((uint64_t)pp->p_pagenum) >= 8067c478bd9Sstevel@tonic-gate dma_attr->dma_attr_addr_hi) 8077c478bd9Sstevel@tonic-gate panic("PFN (pp=%p) above dma_attr_addr_hi", pp); 8087c478bd9Sstevel@tonic-gate pp = pp->p_next; 8097c478bd9Sstevel@tonic-gate } 8107c478bd9Sstevel@tonic-gate } 8117c478bd9Sstevel@tonic-gate #endif 8127c478bd9Sstevel@tonic-gate 8137c478bd9Sstevel@tonic-gate static kmutex_t contig_lock; 8147c478bd9Sstevel@tonic-gate 8157c478bd9Sstevel@tonic-gate #define CONTIG_LOCK() mutex_enter(&contig_lock); 8167c478bd9Sstevel@tonic-gate #define CONTIG_UNLOCK() mutex_exit(&contig_lock); 8177c478bd9Sstevel@tonic-gate 8187c478bd9Sstevel@tonic-gate #define PFN_16M (mmu_btop((uint64_t)0x1000000)) 8197c478bd9Sstevel@tonic-gate 8207c478bd9Sstevel@tonic-gate static page_t * 8217c478bd9Sstevel@tonic-gate page_get_contigpage(pgcnt_t *pgcnt, ddi_dma_attr_t *mattr, int iolock) 8227c478bd9Sstevel@tonic-gate { 8237c478bd9Sstevel@tonic-gate pfn_t pfn; 8247c478bd9Sstevel@tonic-gate int sgllen; 8257c478bd9Sstevel@tonic-gate uint64_t pfnseg; 8267c478bd9Sstevel@tonic-gate pgcnt_t minctg; 8277c478bd9Sstevel@tonic-gate page_t *pplist = NULL, *plist; 8287c478bd9Sstevel@tonic-gate uint64_t lo, hi; 8297c478bd9Sstevel@tonic-gate pgcnt_t pfnalign = 0; 8307c478bd9Sstevel@tonic-gate static pfn_t startpfn; 8317c478bd9Sstevel@tonic-gate static pgcnt_t lastctgcnt; 8327c478bd9Sstevel@tonic-gate uintptr_t align; 8337c478bd9Sstevel@tonic-gate 8347c478bd9Sstevel@tonic-gate CONTIG_LOCK(); 8357c478bd9Sstevel@tonic-gate 8367c478bd9Sstevel@tonic-gate if (mattr) { 8377c478bd9Sstevel@tonic-gate lo = mmu_btop((mattr->dma_attr_addr_lo + MMU_PAGEOFFSET)); 8387c478bd9Sstevel@tonic-gate hi = mmu_btop(mattr->dma_attr_addr_hi); 8397c478bd9Sstevel@tonic-gate if (hi >= physmax) 8407c478bd9Sstevel@tonic-gate hi = physmax - 1; 8417c478bd9Sstevel@tonic-gate sgllen = mattr->dma_attr_sgllen; 8427c478bd9Sstevel@tonic-gate pfnseg = mmu_btop(mattr->dma_attr_seg); 8437c478bd9Sstevel@tonic-gate 8447c478bd9Sstevel@tonic-gate align = maxbit(mattr->dma_attr_align, mattr->dma_attr_minxfer); 8457c478bd9Sstevel@tonic-gate if (align > MMU_PAGESIZE) 8467c478bd9Sstevel@tonic-gate pfnalign = mmu_btop(align); 8477c478bd9Sstevel@tonic-gate 8487c478bd9Sstevel@tonic-gate /* 8497c478bd9Sstevel@tonic-gate * in order to satisfy the request, must minimally 8507c478bd9Sstevel@tonic-gate * acquire minctg contiguous pages 8517c478bd9Sstevel@tonic-gate */ 8527c478bd9Sstevel@tonic-gate minctg = howmany(*pgcnt, sgllen); 8537c478bd9Sstevel@tonic-gate 8547c478bd9Sstevel@tonic-gate ASSERT(hi >= lo); 8557c478bd9Sstevel@tonic-gate 8567c478bd9Sstevel@tonic-gate /* 8577c478bd9Sstevel@tonic-gate * start from where last searched if the minctg >= lastctgcnt 8587c478bd9Sstevel@tonic-gate */ 8597c478bd9Sstevel@tonic-gate if (minctg < lastctgcnt || startpfn < lo || startpfn > hi) 8607c478bd9Sstevel@tonic-gate startpfn = lo; 8617c478bd9Sstevel@tonic-gate } else { 8627c478bd9Sstevel@tonic-gate hi = physmax - 1; 8637c478bd9Sstevel@tonic-gate lo = 0; 8647c478bd9Sstevel@tonic-gate sgllen = 1; 8657c478bd9Sstevel@tonic-gate pfnseg = mmu.highest_pfn; 8667c478bd9Sstevel@tonic-gate minctg = *pgcnt; 8677c478bd9Sstevel@tonic-gate 8687c478bd9Sstevel@tonic-gate if (minctg < lastctgcnt) 8697c478bd9Sstevel@tonic-gate startpfn = lo; 8707c478bd9Sstevel@tonic-gate } 8717c478bd9Sstevel@tonic-gate lastctgcnt = minctg; 8727c478bd9Sstevel@tonic-gate 8737c478bd9Sstevel@tonic-gate ASSERT(pfnseg + 1 >= (uint64_t)minctg); 8747c478bd9Sstevel@tonic-gate 8757c478bd9Sstevel@tonic-gate /* conserve 16m memory - start search above 16m when possible */ 8767c478bd9Sstevel@tonic-gate if (hi > PFN_16M && startpfn < PFN_16M) 8777c478bd9Sstevel@tonic-gate startpfn = PFN_16M; 8787c478bd9Sstevel@tonic-gate 8797c478bd9Sstevel@tonic-gate pfn = startpfn; 8807c478bd9Sstevel@tonic-gate if (pfnalign) 8817c478bd9Sstevel@tonic-gate pfn = P2ROUNDUP(pfn, pfnalign); 8827c478bd9Sstevel@tonic-gate 8837c478bd9Sstevel@tonic-gate while (pfn + minctg - 1 <= hi) { 8847c478bd9Sstevel@tonic-gate 8857c478bd9Sstevel@tonic-gate plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock); 8867c478bd9Sstevel@tonic-gate if (plist) { 8877c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &plist); 8887c478bd9Sstevel@tonic-gate sgllen--; 8897c478bd9Sstevel@tonic-gate /* 8907c478bd9Sstevel@tonic-gate * return when contig pages no longer needed 8917c478bd9Sstevel@tonic-gate */ 8927c478bd9Sstevel@tonic-gate if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) { 8937c478bd9Sstevel@tonic-gate startpfn = pfn; 8947c478bd9Sstevel@tonic-gate CONTIG_UNLOCK(); 8957c478bd9Sstevel@tonic-gate check_dma(mattr, pplist, *pgcnt); 8967c478bd9Sstevel@tonic-gate return (pplist); 8977c478bd9Sstevel@tonic-gate } 8987c478bd9Sstevel@tonic-gate minctg = howmany(*pgcnt, sgllen); 8997c478bd9Sstevel@tonic-gate } 9007c478bd9Sstevel@tonic-gate if (pfnalign) 9017c478bd9Sstevel@tonic-gate pfn = P2ROUNDUP(pfn, pfnalign); 9027c478bd9Sstevel@tonic-gate } 9037c478bd9Sstevel@tonic-gate 9047c478bd9Sstevel@tonic-gate /* cannot find contig pages in specified range */ 9057c478bd9Sstevel@tonic-gate if (startpfn == lo) { 9067c478bd9Sstevel@tonic-gate CONTIG_UNLOCK(); 9077c478bd9Sstevel@tonic-gate return (NULL); 9087c478bd9Sstevel@tonic-gate } 9097c478bd9Sstevel@tonic-gate 9107c478bd9Sstevel@tonic-gate /* did not start with lo previously */ 9117c478bd9Sstevel@tonic-gate pfn = lo; 9127c478bd9Sstevel@tonic-gate if (pfnalign) 9137c478bd9Sstevel@tonic-gate pfn = P2ROUNDUP(pfn, pfnalign); 9147c478bd9Sstevel@tonic-gate 9157c478bd9Sstevel@tonic-gate /* allow search to go above startpfn */ 9167c478bd9Sstevel@tonic-gate while (pfn < startpfn) { 9177c478bd9Sstevel@tonic-gate 9187c478bd9Sstevel@tonic-gate plist = is_contigpage_free(&pfn, pgcnt, minctg, pfnseg, iolock); 9197c478bd9Sstevel@tonic-gate if (plist != NULL) { 9207c478bd9Sstevel@tonic-gate 9217c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &plist); 9227c478bd9Sstevel@tonic-gate sgllen--; 9237c478bd9Sstevel@tonic-gate 9247c478bd9Sstevel@tonic-gate /* 9257c478bd9Sstevel@tonic-gate * return when contig pages no longer needed 9267c478bd9Sstevel@tonic-gate */ 9277c478bd9Sstevel@tonic-gate if (!*pgcnt || ((*pgcnt <= sgllen) && !pfnalign)) { 9287c478bd9Sstevel@tonic-gate startpfn = pfn; 9297c478bd9Sstevel@tonic-gate CONTIG_UNLOCK(); 9307c478bd9Sstevel@tonic-gate check_dma(mattr, pplist, *pgcnt); 9317c478bd9Sstevel@tonic-gate return (pplist); 9327c478bd9Sstevel@tonic-gate } 9337c478bd9Sstevel@tonic-gate minctg = howmany(*pgcnt, sgllen); 9347c478bd9Sstevel@tonic-gate } 9357c478bd9Sstevel@tonic-gate if (pfnalign) 9367c478bd9Sstevel@tonic-gate pfn = P2ROUNDUP(pfn, pfnalign); 9377c478bd9Sstevel@tonic-gate } 9387c478bd9Sstevel@tonic-gate CONTIG_UNLOCK(); 9397c478bd9Sstevel@tonic-gate return (NULL); 9407c478bd9Sstevel@tonic-gate } 9417c478bd9Sstevel@tonic-gate 9427c478bd9Sstevel@tonic-gate /* 9437c478bd9Sstevel@tonic-gate * combine mem_node_config and memrange memory ranges into one data 9447c478bd9Sstevel@tonic-gate * structure to be used for page list management. 9457c478bd9Sstevel@tonic-gate * 9467c478bd9Sstevel@tonic-gate * mnode_range_cnt() calculates the number of memory ranges for mnode and 9477c478bd9Sstevel@tonic-gate * memranges[]. Used to determine the size of page lists and mnoderanges. 9487c478bd9Sstevel@tonic-gate * 9497c478bd9Sstevel@tonic-gate * mnode_range_setup() initializes mnoderanges. 9507c478bd9Sstevel@tonic-gate */ 9517c478bd9Sstevel@tonic-gate mnoderange_t *mnoderanges; 9527c478bd9Sstevel@tonic-gate int mnoderangecnt; 9537c478bd9Sstevel@tonic-gate int mtype4g; 9547c478bd9Sstevel@tonic-gate 9557c478bd9Sstevel@tonic-gate int 956*5d07b933Sdp mnode_range_cnt(int mnode) 9577c478bd9Sstevel@tonic-gate { 9587c478bd9Sstevel@tonic-gate int mri; 9597c478bd9Sstevel@tonic-gate int mnrcnt = 0; 9607c478bd9Sstevel@tonic-gate 961*5d07b933Sdp if (mem_node_config[mnode].exists != 0) { 9627c478bd9Sstevel@tonic-gate mri = nranges - 1; 9637c478bd9Sstevel@tonic-gate 9647c478bd9Sstevel@tonic-gate /* find the memranges index below contained in mnode range */ 9657c478bd9Sstevel@tonic-gate 9667c478bd9Sstevel@tonic-gate while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase) 9677c478bd9Sstevel@tonic-gate mri--; 9687c478bd9Sstevel@tonic-gate 9697c478bd9Sstevel@tonic-gate /* 9707c478bd9Sstevel@tonic-gate * increment mnode range counter when memranges or mnode 9717c478bd9Sstevel@tonic-gate * boundary is reached. 9727c478bd9Sstevel@tonic-gate */ 9737c478bd9Sstevel@tonic-gate while (mri >= 0 && 9747c478bd9Sstevel@tonic-gate mem_node_config[mnode].physmax >= MEMRANGELO(mri)) { 9757c478bd9Sstevel@tonic-gate mnrcnt++; 9767c478bd9Sstevel@tonic-gate if (mem_node_config[mnode].physmax > MEMRANGEHI(mri)) 9777c478bd9Sstevel@tonic-gate mri--; 9787c478bd9Sstevel@tonic-gate else 9797c478bd9Sstevel@tonic-gate break; 9807c478bd9Sstevel@tonic-gate } 9817c478bd9Sstevel@tonic-gate } 982*5d07b933Sdp ASSERT(mnrcnt <= MAX_MNODE_MRANGES); 9837c478bd9Sstevel@tonic-gate return (mnrcnt); 9847c478bd9Sstevel@tonic-gate } 9857c478bd9Sstevel@tonic-gate 9867c478bd9Sstevel@tonic-gate void 9877c478bd9Sstevel@tonic-gate mnode_range_setup(mnoderange_t *mnoderanges) 9887c478bd9Sstevel@tonic-gate { 9897c478bd9Sstevel@tonic-gate int mnode, mri; 9907c478bd9Sstevel@tonic-gate 9917c478bd9Sstevel@tonic-gate for (mnode = 0; mnode < max_mem_nodes; mnode++) { 9927c478bd9Sstevel@tonic-gate if (mem_node_config[mnode].exists == 0) 9937c478bd9Sstevel@tonic-gate continue; 9947c478bd9Sstevel@tonic-gate 9957c478bd9Sstevel@tonic-gate mri = nranges - 1; 9967c478bd9Sstevel@tonic-gate 9977c478bd9Sstevel@tonic-gate while (MEMRANGEHI(mri) < mem_node_config[mnode].physbase) 9987c478bd9Sstevel@tonic-gate mri--; 9997c478bd9Sstevel@tonic-gate 10007c478bd9Sstevel@tonic-gate while (mri >= 0 && mem_node_config[mnode].physmax >= 10017c478bd9Sstevel@tonic-gate MEMRANGELO(mri)) { 10027c478bd9Sstevel@tonic-gate mnoderanges->mnr_pfnlo = 10037c478bd9Sstevel@tonic-gate MAX(MEMRANGELO(mri), 10047c478bd9Sstevel@tonic-gate mem_node_config[mnode].physbase); 10057c478bd9Sstevel@tonic-gate mnoderanges->mnr_pfnhi = 10067c478bd9Sstevel@tonic-gate MIN(MEMRANGEHI(mri), 10077c478bd9Sstevel@tonic-gate mem_node_config[mnode].physmax); 10087c478bd9Sstevel@tonic-gate mnoderanges->mnr_mnode = mnode; 10097c478bd9Sstevel@tonic-gate mnoderanges->mnr_memrange = mri; 10107c478bd9Sstevel@tonic-gate mnoderanges++; 10117c478bd9Sstevel@tonic-gate if (mem_node_config[mnode].physmax > MEMRANGEHI(mri)) 10127c478bd9Sstevel@tonic-gate mri--; 10137c478bd9Sstevel@tonic-gate else 10147c478bd9Sstevel@tonic-gate break; 10157c478bd9Sstevel@tonic-gate } 10167c478bd9Sstevel@tonic-gate } 10177c478bd9Sstevel@tonic-gate } 10187c478bd9Sstevel@tonic-gate 10197c478bd9Sstevel@tonic-gate /* 10207c478bd9Sstevel@tonic-gate * Determine if the mnode range specified in mtype contains memory belonging 10217c478bd9Sstevel@tonic-gate * to memory node mnode. If flags & PGI_MT_RANGE is set then mtype contains 102207ad560dSkchow * the range of indices from high pfn to 0, 16m or 4g. 10237c478bd9Sstevel@tonic-gate * 10247c478bd9Sstevel@tonic-gate * Return first mnode range type index found otherwise return -1 if none found. 10257c478bd9Sstevel@tonic-gate */ 10267c478bd9Sstevel@tonic-gate int 10277c478bd9Sstevel@tonic-gate mtype_func(int mnode, int mtype, uint_t flags) 10287c478bd9Sstevel@tonic-gate { 10297c478bd9Sstevel@tonic-gate if (flags & PGI_MT_RANGE) { 103007ad560dSkchow int mtlim; 10317c478bd9Sstevel@tonic-gate 10327c478bd9Sstevel@tonic-gate if (flags & PGI_MT_NEXT) 10337c478bd9Sstevel@tonic-gate mtype--; 103407ad560dSkchow if (flags & PGI_MT_RANGE0) 103507ad560dSkchow mtlim = 0; 103607ad560dSkchow else if (flags & PGI_MT_RANGE4G) 103707ad560dSkchow mtlim = mtype4g + 1; /* exclude 0-4g range */ 103807ad560dSkchow else if (flags & PGI_MT_RANGE16M) 103907ad560dSkchow mtlim = 1; /* exclude 0-16m range */ 10407c478bd9Sstevel@tonic-gate while (mtype >= mtlim) { 10417c478bd9Sstevel@tonic-gate if (mnoderanges[mtype].mnr_mnode == mnode) 10427c478bd9Sstevel@tonic-gate return (mtype); 10437c478bd9Sstevel@tonic-gate mtype--; 10447c478bd9Sstevel@tonic-gate } 10457c478bd9Sstevel@tonic-gate } else { 10467c478bd9Sstevel@tonic-gate if (mnoderanges[mtype].mnr_mnode == mnode) 10477c478bd9Sstevel@tonic-gate return (mtype); 10487c478bd9Sstevel@tonic-gate } 10497c478bd9Sstevel@tonic-gate return (-1); 10507c478bd9Sstevel@tonic-gate } 10517c478bd9Sstevel@tonic-gate 1052e21bae1bSkchow /* 1053e21bae1bSkchow * Update the page list max counts with the pfn range specified by the 1054e21bae1bSkchow * input parameters. Called from add_physmem() when physical memory with 1055e21bae1bSkchow * page_t's are initially added to the page lists. 1056e21bae1bSkchow */ 1057e21bae1bSkchow void 1058e21bae1bSkchow mtype_modify_max(pfn_t startpfn, long cnt) 1059e21bae1bSkchow { 1060e21bae1bSkchow int mtype = 0; 1061e21bae1bSkchow pfn_t endpfn = startpfn + cnt, pfn; 1062e21bae1bSkchow pgcnt_t inc; 1063e21bae1bSkchow 1064e21bae1bSkchow ASSERT(cnt > 0); 1065e21bae1bSkchow 1066e21bae1bSkchow for (pfn = startpfn; pfn < endpfn; ) { 1067e21bae1bSkchow if (pfn <= mnoderanges[mtype].mnr_pfnhi) { 1068e21bae1bSkchow if (endpfn < mnoderanges[mtype].mnr_pfnhi) { 1069e21bae1bSkchow inc = endpfn - pfn; 1070e21bae1bSkchow } else { 1071e21bae1bSkchow inc = mnoderanges[mtype].mnr_pfnhi - pfn + 1; 1072e21bae1bSkchow } 1073e21bae1bSkchow mnoderanges[mtype].mnr_mt_pgmax += inc; 1074e21bae1bSkchow if (physmax4g && mtype <= mtype4g) 1075e21bae1bSkchow maxmem4g += inc; 1076e21bae1bSkchow pfn += inc; 1077e21bae1bSkchow } 1078e21bae1bSkchow mtype++; 1079e21bae1bSkchow ASSERT(mtype < mnoderangecnt || pfn >= endpfn); 1080e21bae1bSkchow } 1081e21bae1bSkchow } 1082e21bae1bSkchow 1083affbd3ccSkchow /* 1084affbd3ccSkchow * Returns the free page count for mnode 1085affbd3ccSkchow */ 1086affbd3ccSkchow int 1087affbd3ccSkchow mnode_pgcnt(int mnode) 1088affbd3ccSkchow { 1089affbd3ccSkchow int mtype = mnoderangecnt - 1; 1090affbd3ccSkchow int flags = PGI_MT_RANGE0; 1091affbd3ccSkchow pgcnt_t pgcnt = 0; 1092affbd3ccSkchow 1093affbd3ccSkchow mtype = mtype_func(mnode, mtype, flags); 1094affbd3ccSkchow 1095affbd3ccSkchow while (mtype != -1) { 109607ad560dSkchow pgcnt += MTYPE_FREEMEM(mtype); 1097affbd3ccSkchow mtype = mtype_func(mnode, mtype, flags | PGI_MT_NEXT); 1098affbd3ccSkchow } 1099affbd3ccSkchow return (pgcnt); 1100affbd3ccSkchow } 1101affbd3ccSkchow 11027c478bd9Sstevel@tonic-gate /* 11037c478bd9Sstevel@tonic-gate * Initialize page coloring variables based on the l2 cache parameters. 11047c478bd9Sstevel@tonic-gate * Calculate and return memory needed for page coloring data structures. 11057c478bd9Sstevel@tonic-gate */ 11067c478bd9Sstevel@tonic-gate size_t 11077c478bd9Sstevel@tonic-gate page_coloring_init(uint_t l2_sz, int l2_linesz, int l2_assoc) 11087c478bd9Sstevel@tonic-gate { 11097c478bd9Sstevel@tonic-gate size_t colorsz = 0; 11107c478bd9Sstevel@tonic-gate int i; 11117c478bd9Sstevel@tonic-gate int colors; 11127c478bd9Sstevel@tonic-gate 11137c478bd9Sstevel@tonic-gate /* 11147c478bd9Sstevel@tonic-gate * Reduce the memory ranges lists if we don't have large amounts 11157c478bd9Sstevel@tonic-gate * of memory. This avoids searching known empty free lists. 11167c478bd9Sstevel@tonic-gate */ 11177c478bd9Sstevel@tonic-gate i = memrange_num(physmax); 11187c478bd9Sstevel@tonic-gate memranges += i; 11197c478bd9Sstevel@tonic-gate nranges -= i; 11207c478bd9Sstevel@tonic-gate #if defined(__i386) 11217c478bd9Sstevel@tonic-gate if (i > 0) 11227c478bd9Sstevel@tonic-gate restricted_kmemalloc = 0; 11237c478bd9Sstevel@tonic-gate #endif 11247c478bd9Sstevel@tonic-gate /* physmax greater than 4g */ 11257c478bd9Sstevel@tonic-gate if (i == 0) 11267c478bd9Sstevel@tonic-gate physmax4g = 1; 11277c478bd9Sstevel@tonic-gate 11287c478bd9Sstevel@tonic-gate ASSERT(ISP2(l2_sz)); 11297c478bd9Sstevel@tonic-gate ASSERT(ISP2(l2_linesz)); 11307c478bd9Sstevel@tonic-gate ASSERT(l2_sz > MMU_PAGESIZE); 11317c478bd9Sstevel@tonic-gate 11327c478bd9Sstevel@tonic-gate /* l2_assoc is 0 for fully associative l2 cache */ 11337c478bd9Sstevel@tonic-gate if (l2_assoc) 11347c478bd9Sstevel@tonic-gate l2_colors = MAX(1, l2_sz / (l2_assoc * MMU_PAGESIZE)); 11357c478bd9Sstevel@tonic-gate else 11367c478bd9Sstevel@tonic-gate l2_colors = 1; 11377c478bd9Sstevel@tonic-gate 11387c478bd9Sstevel@tonic-gate /* for scalability, configure at least PAGE_COLORS_MIN color bins */ 11397c478bd9Sstevel@tonic-gate page_colors = MAX(l2_colors, PAGE_COLORS_MIN); 11407c478bd9Sstevel@tonic-gate 11417c478bd9Sstevel@tonic-gate /* 11427c478bd9Sstevel@tonic-gate * cpu_page_colors is non-zero when a page color may be spread across 11437c478bd9Sstevel@tonic-gate * multiple bins. 11447c478bd9Sstevel@tonic-gate */ 11457c478bd9Sstevel@tonic-gate if (l2_colors < page_colors) 11467c478bd9Sstevel@tonic-gate cpu_page_colors = l2_colors; 11477c478bd9Sstevel@tonic-gate 11487c478bd9Sstevel@tonic-gate ASSERT(ISP2(page_colors)); 11497c478bd9Sstevel@tonic-gate 11507c478bd9Sstevel@tonic-gate page_colors_mask = page_colors - 1; 11517c478bd9Sstevel@tonic-gate 11527c478bd9Sstevel@tonic-gate ASSERT(ISP2(CPUSETSIZE())); 11537c478bd9Sstevel@tonic-gate page_coloring_shift = lowbit(CPUSETSIZE()); 11547c478bd9Sstevel@tonic-gate 1155*5d07b933Sdp /* initialize number of colors per page size */ 1156*5d07b933Sdp for (i = 0; i <= mmu.max_page_level; i++) { 1157*5d07b933Sdp hw_page_array[i].hp_size = LEVEL_SIZE(i); 1158*5d07b933Sdp hw_page_array[i].hp_shift = LEVEL_SHIFT(i); 1159*5d07b933Sdp hw_page_array[i].hp_pgcnt = LEVEL_SIZE(i) >> LEVEL_SHIFT(0); 1160*5d07b933Sdp hw_page_array[i].hp_colors = (page_colors_mask >> 1161*5d07b933Sdp (hw_page_array[i].hp_shift - hw_page_array[0].hp_shift)) 1162*5d07b933Sdp + 1; 1163*5d07b933Sdp } 1164*5d07b933Sdp 1165*5d07b933Sdp /* 1166*5d07b933Sdp * The value of cpu_page_colors determines if additional color bins 1167*5d07b933Sdp * need to be checked for a particular color in the page_get routines. 1168*5d07b933Sdp */ 1169*5d07b933Sdp if (cpu_page_colors != 0) { 1170*5d07b933Sdp 1171*5d07b933Sdp int a = lowbit(page_colors) - lowbit(cpu_page_colors); 1172*5d07b933Sdp ASSERT(a > 0); 1173*5d07b933Sdp ASSERT(a < 16); 1174*5d07b933Sdp 1175*5d07b933Sdp for (i = 0; i <= mmu.max_page_level; i++) { 1176*5d07b933Sdp if ((colors = hw_page_array[i].hp_colors) <= 1) { 1177*5d07b933Sdp colorequivszc[i] = 0; 1178*5d07b933Sdp continue; 1179*5d07b933Sdp } 1180*5d07b933Sdp while ((colors >> a) == 0) 1181*5d07b933Sdp a--; 1182*5d07b933Sdp ASSERT(a >= 0); 1183*5d07b933Sdp 1184*5d07b933Sdp /* higher 4 bits encodes color equiv mask */ 1185*5d07b933Sdp colorequivszc[i] = (a << 4); 1186*5d07b933Sdp } 1187*5d07b933Sdp } 1188*5d07b933Sdp 1189*5d07b933Sdp /* factor in colorequiv to check additional 'equivalent' bins. */ 1190*5d07b933Sdp if (colorequiv > 1) { 1191*5d07b933Sdp 1192*5d07b933Sdp int a = lowbit(colorequiv) - 1; 1193*5d07b933Sdp if (a > 15) 1194*5d07b933Sdp a = 15; 1195*5d07b933Sdp 1196*5d07b933Sdp for (i = 0; i <= mmu.max_page_level; i++) { 1197*5d07b933Sdp if ((colors = hw_page_array[i].hp_colors) <= 1) { 1198*5d07b933Sdp continue; 1199*5d07b933Sdp } 1200*5d07b933Sdp while ((colors >> a) == 0) 1201*5d07b933Sdp a--; 1202*5d07b933Sdp if ((a << 4) > colorequivszc[i]) { 1203*5d07b933Sdp colorequivszc[i] = (a << 4); 1204*5d07b933Sdp } 1205*5d07b933Sdp } 1206*5d07b933Sdp } 1207*5d07b933Sdp 12087c478bd9Sstevel@tonic-gate /* size for mnoderanges */ 1209*5d07b933Sdp for (mnoderangecnt = 0, i = 0; i < max_mem_nodes; i++) 1210*5d07b933Sdp mnoderangecnt += mnode_range_cnt(i); 12117c478bd9Sstevel@tonic-gate colorsz = mnoderangecnt * sizeof (mnoderange_t); 12127c478bd9Sstevel@tonic-gate 12137c478bd9Sstevel@tonic-gate /* size for fpc_mutex and cpc_mutex */ 12147c478bd9Sstevel@tonic-gate colorsz += (2 * max_mem_nodes * sizeof (kmutex_t) * NPC_MUTEX); 12157c478bd9Sstevel@tonic-gate 12167c478bd9Sstevel@tonic-gate /* size of page_freelists */ 12177c478bd9Sstevel@tonic-gate colorsz += mnoderangecnt * sizeof (page_t ***); 12187c478bd9Sstevel@tonic-gate colorsz += mnoderangecnt * mmu_page_sizes * sizeof (page_t **); 12197c478bd9Sstevel@tonic-gate 12207c478bd9Sstevel@tonic-gate for (i = 0; i < mmu_page_sizes; i++) { 12217c478bd9Sstevel@tonic-gate colors = page_get_pagecolors(i); 12227c478bd9Sstevel@tonic-gate colorsz += mnoderangecnt * colors * sizeof (page_t *); 12237c478bd9Sstevel@tonic-gate } 12247c478bd9Sstevel@tonic-gate 12257c478bd9Sstevel@tonic-gate /* size of page_cachelists */ 12267c478bd9Sstevel@tonic-gate colorsz += mnoderangecnt * sizeof (page_t **); 12277c478bd9Sstevel@tonic-gate colorsz += mnoderangecnt * page_colors * sizeof (page_t *); 12287c478bd9Sstevel@tonic-gate 12297c478bd9Sstevel@tonic-gate return (colorsz); 12307c478bd9Sstevel@tonic-gate } 12317c478bd9Sstevel@tonic-gate 12327c478bd9Sstevel@tonic-gate /* 12337c478bd9Sstevel@tonic-gate * Called once at startup to configure page_coloring data structures and 12347c478bd9Sstevel@tonic-gate * does the 1st page_free()/page_freelist_add(). 12357c478bd9Sstevel@tonic-gate */ 12367c478bd9Sstevel@tonic-gate void 12377c478bd9Sstevel@tonic-gate page_coloring_setup(caddr_t pcmemaddr) 12387c478bd9Sstevel@tonic-gate { 12397c478bd9Sstevel@tonic-gate int i; 12407c478bd9Sstevel@tonic-gate int j; 12417c478bd9Sstevel@tonic-gate int k; 12427c478bd9Sstevel@tonic-gate caddr_t addr; 12437c478bd9Sstevel@tonic-gate int colors; 12447c478bd9Sstevel@tonic-gate 12457c478bd9Sstevel@tonic-gate /* 12467c478bd9Sstevel@tonic-gate * do page coloring setup 12477c478bd9Sstevel@tonic-gate */ 12487c478bd9Sstevel@tonic-gate addr = pcmemaddr; 12497c478bd9Sstevel@tonic-gate 12507c478bd9Sstevel@tonic-gate mnoderanges = (mnoderange_t *)addr; 12517c478bd9Sstevel@tonic-gate addr += (mnoderangecnt * sizeof (mnoderange_t)); 12527c478bd9Sstevel@tonic-gate 12537c478bd9Sstevel@tonic-gate mnode_range_setup(mnoderanges); 12547c478bd9Sstevel@tonic-gate 12557c478bd9Sstevel@tonic-gate if (physmax4g) 12567c478bd9Sstevel@tonic-gate mtype4g = pfn_2_mtype(0xfffff); 12577c478bd9Sstevel@tonic-gate 12587c478bd9Sstevel@tonic-gate for (k = 0; k < NPC_MUTEX; k++) { 12597c478bd9Sstevel@tonic-gate fpc_mutex[k] = (kmutex_t *)addr; 12607c478bd9Sstevel@tonic-gate addr += (max_mem_nodes * sizeof (kmutex_t)); 12617c478bd9Sstevel@tonic-gate } 12627c478bd9Sstevel@tonic-gate for (k = 0; k < NPC_MUTEX; k++) { 12637c478bd9Sstevel@tonic-gate cpc_mutex[k] = (kmutex_t *)addr; 12647c478bd9Sstevel@tonic-gate addr += (max_mem_nodes * sizeof (kmutex_t)); 12657c478bd9Sstevel@tonic-gate } 12667c478bd9Sstevel@tonic-gate page_freelists = (page_t ****)addr; 12677c478bd9Sstevel@tonic-gate addr += (mnoderangecnt * sizeof (page_t ***)); 12687c478bd9Sstevel@tonic-gate 12697c478bd9Sstevel@tonic-gate page_cachelists = (page_t ***)addr; 12707c478bd9Sstevel@tonic-gate addr += (mnoderangecnt * sizeof (page_t **)); 12717c478bd9Sstevel@tonic-gate 12727c478bd9Sstevel@tonic-gate for (i = 0; i < mnoderangecnt; i++) { 12737c478bd9Sstevel@tonic-gate page_freelists[i] = (page_t ***)addr; 12747c478bd9Sstevel@tonic-gate addr += (mmu_page_sizes * sizeof (page_t **)); 12757c478bd9Sstevel@tonic-gate 12767c478bd9Sstevel@tonic-gate for (j = 0; j < mmu_page_sizes; j++) { 12777c478bd9Sstevel@tonic-gate colors = page_get_pagecolors(j); 12787c478bd9Sstevel@tonic-gate page_freelists[i][j] = (page_t **)addr; 12797c478bd9Sstevel@tonic-gate addr += (colors * sizeof (page_t *)); 12807c478bd9Sstevel@tonic-gate } 12817c478bd9Sstevel@tonic-gate page_cachelists[i] = (page_t **)addr; 12827c478bd9Sstevel@tonic-gate addr += (page_colors * sizeof (page_t *)); 12837c478bd9Sstevel@tonic-gate } 12847c478bd9Sstevel@tonic-gate } 12857c478bd9Sstevel@tonic-gate 12867c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 12877c478bd9Sstevel@tonic-gate int 12887c478bd9Sstevel@tonic-gate bp_color(struct buf *bp) 12897c478bd9Sstevel@tonic-gate { 12907c478bd9Sstevel@tonic-gate return (0); 12917c478bd9Sstevel@tonic-gate } 12927c478bd9Sstevel@tonic-gate 12937c478bd9Sstevel@tonic-gate /* 12947c478bd9Sstevel@tonic-gate * get a page from any list with the given mnode 12957c478bd9Sstevel@tonic-gate */ 12967c478bd9Sstevel@tonic-gate page_t * 12977c478bd9Sstevel@tonic-gate page_get_mnode_anylist(ulong_t origbin, uchar_t szc, uint_t flags, 12987c478bd9Sstevel@tonic-gate int mnode, int mtype, ddi_dma_attr_t *dma_attr) 12997c478bd9Sstevel@tonic-gate { 1300*5d07b933Sdp kmutex_t *pcm; 1301*5d07b933Sdp int i; 1302*5d07b933Sdp page_t *pp; 1303*5d07b933Sdp page_t *first_pp; 1304*5d07b933Sdp uint64_t pgaddr; 1305*5d07b933Sdp ulong_t bin; 1306*5d07b933Sdp int mtypestart; 1307*5d07b933Sdp int plw_initialized; 1308*5d07b933Sdp page_list_walker_t plw; 13097c478bd9Sstevel@tonic-gate 13107c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pgma_alloc); 13117c478bd9Sstevel@tonic-gate 13127c478bd9Sstevel@tonic-gate ASSERT((flags & PG_MATCH_COLOR) == 0); 13137c478bd9Sstevel@tonic-gate ASSERT(szc == 0); 13147c478bd9Sstevel@tonic-gate ASSERT(dma_attr != NULL); 13157c478bd9Sstevel@tonic-gate 13167c478bd9Sstevel@tonic-gate MTYPE_START(mnode, mtype, flags); 13177c478bd9Sstevel@tonic-gate if (mtype < 0) { 13187c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pgma_allocempty); 13197c478bd9Sstevel@tonic-gate return (NULL); 13207c478bd9Sstevel@tonic-gate } 13217c478bd9Sstevel@tonic-gate 13227c478bd9Sstevel@tonic-gate mtypestart = mtype; 13237c478bd9Sstevel@tonic-gate 13247c478bd9Sstevel@tonic-gate bin = origbin; 13257c478bd9Sstevel@tonic-gate 13267c478bd9Sstevel@tonic-gate /* 13277c478bd9Sstevel@tonic-gate * check up to page_colors + 1 bins - origbin may be checked twice 13287c478bd9Sstevel@tonic-gate * because of BIN_STEP skip 13297c478bd9Sstevel@tonic-gate */ 13307c478bd9Sstevel@tonic-gate do { 1331*5d07b933Sdp plw_initialized = 0; 1332*5d07b933Sdp 1333*5d07b933Sdp for (plw.plw_count = 0; 1334*5d07b933Sdp plw.plw_count < page_colors; plw.plw_count++) { 1335*5d07b933Sdp 13367c478bd9Sstevel@tonic-gate if (PAGE_FREELISTS(mnode, szc, bin, mtype) == NULL) 13377c478bd9Sstevel@tonic-gate goto nextfreebin; 13387c478bd9Sstevel@tonic-gate 13397c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST); 13407c478bd9Sstevel@tonic-gate mutex_enter(pcm); 13417c478bd9Sstevel@tonic-gate pp = PAGE_FREELISTS(mnode, szc, bin, mtype); 13427c478bd9Sstevel@tonic-gate first_pp = pp; 13437c478bd9Sstevel@tonic-gate while (pp != NULL) { 13447c478bd9Sstevel@tonic-gate if (page_trylock(pp, SE_EXCL) == 0) { 13457c478bd9Sstevel@tonic-gate pp = pp->p_next; 13467c478bd9Sstevel@tonic-gate if (pp == first_pp) { 13477c478bd9Sstevel@tonic-gate pp = NULL; 13487c478bd9Sstevel@tonic-gate } 13497c478bd9Sstevel@tonic-gate continue; 13507c478bd9Sstevel@tonic-gate } 13517c478bd9Sstevel@tonic-gate 13527c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 13537c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 13547c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode == NULL); 13557c478bd9Sstevel@tonic-gate ASSERT(pp->p_hash == NULL); 13567c478bd9Sstevel@tonic-gate ASSERT(pp->p_offset == (u_offset_t)-1); 13577c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == szc); 13587c478bd9Sstevel@tonic-gate ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 13597c478bd9Sstevel@tonic-gate /* check if page within DMA attributes */ 13607c478bd9Sstevel@tonic-gate pgaddr = mmu_ptob((uint64_t)(pp->p_pagenum)); 13617c478bd9Sstevel@tonic-gate 13627c478bd9Sstevel@tonic-gate if ((pgaddr >= dma_attr->dma_attr_addr_lo) && 13637c478bd9Sstevel@tonic-gate (pgaddr + MMU_PAGESIZE - 1 <= 13647c478bd9Sstevel@tonic-gate dma_attr->dma_attr_addr_hi)) { 13657c478bd9Sstevel@tonic-gate break; 13667c478bd9Sstevel@tonic-gate } 13677c478bd9Sstevel@tonic-gate 13687c478bd9Sstevel@tonic-gate /* continue looking */ 13697c478bd9Sstevel@tonic-gate page_unlock(pp); 13707c478bd9Sstevel@tonic-gate pp = pp->p_next; 13717c478bd9Sstevel@tonic-gate if (pp == first_pp) 13727c478bd9Sstevel@tonic-gate pp = NULL; 13737c478bd9Sstevel@tonic-gate 13747c478bd9Sstevel@tonic-gate } 13757c478bd9Sstevel@tonic-gate if (pp != NULL) { 13767c478bd9Sstevel@tonic-gate ASSERT(mtype == PP_2_MTYPE(pp)); 13777c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 13787c478bd9Sstevel@tonic-gate 13797c478bd9Sstevel@tonic-gate /* found a page with specified DMA attributes */ 13807c478bd9Sstevel@tonic-gate page_sub(&PAGE_FREELISTS(mnode, szc, bin, 13817c478bd9Sstevel@tonic-gate mtype), pp); 1382affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST); 13837c478bd9Sstevel@tonic-gate 13847c478bd9Sstevel@tonic-gate if ((PP_ISFREE(pp) == 0) || 13857c478bd9Sstevel@tonic-gate (PP_ISAGED(pp) == 0)) { 13867c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "page %p is not free", 13877c478bd9Sstevel@tonic-gate (void *)pp); 13887c478bd9Sstevel@tonic-gate } 13897c478bd9Sstevel@tonic-gate 13907c478bd9Sstevel@tonic-gate mutex_exit(pcm); 13917c478bd9Sstevel@tonic-gate check_dma(dma_attr, pp, 1); 13927c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pgma_allocok); 13937c478bd9Sstevel@tonic-gate return (pp); 13947c478bd9Sstevel@tonic-gate } 13957c478bd9Sstevel@tonic-gate mutex_exit(pcm); 13967c478bd9Sstevel@tonic-gate nextfreebin: 1397*5d07b933Sdp if (plw_initialized == 0) { 1398*5d07b933Sdp page_list_walk_init(szc, 0, bin, 1, 0, &plw); 1399*5d07b933Sdp ASSERT(plw.plw_ceq_dif == page_colors); 1400*5d07b933Sdp plw_initialized = 1; 1401*5d07b933Sdp } 14027c478bd9Sstevel@tonic-gate 1403*5d07b933Sdp if (plw.plw_do_split) { 1404*5d07b933Sdp pp = page_freelist_split(szc, bin, mnode, 1405*5d07b933Sdp mtype, 1406*5d07b933Sdp mmu_btop(dma_attr->dma_attr_addr_hi + 1), 1407*5d07b933Sdp &plw); 1408*5d07b933Sdp if (pp != NULL) 1409*5d07b933Sdp return (pp); 1410*5d07b933Sdp } 1411*5d07b933Sdp 1412*5d07b933Sdp bin = page_list_walk_next_bin(szc, bin, &plw); 14137c478bd9Sstevel@tonic-gate } 1414*5d07b933Sdp 1415affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags); 1416affbd3ccSkchow } while (mtype >= 0); 14177c478bd9Sstevel@tonic-gate 14187c478bd9Sstevel@tonic-gate /* failed to find a page in the freelist; try it in the cachelist */ 14197c478bd9Sstevel@tonic-gate 14207c478bd9Sstevel@tonic-gate /* reset mtype start for cachelist search */ 14217c478bd9Sstevel@tonic-gate mtype = mtypestart; 14227c478bd9Sstevel@tonic-gate ASSERT(mtype >= 0); 14237c478bd9Sstevel@tonic-gate 14247c478bd9Sstevel@tonic-gate /* start with the bin of matching color */ 14257c478bd9Sstevel@tonic-gate bin = origbin; 14267c478bd9Sstevel@tonic-gate 14277c478bd9Sstevel@tonic-gate do { 14287c478bd9Sstevel@tonic-gate for (i = 0; i <= page_colors; i++) { 14297c478bd9Sstevel@tonic-gate if (PAGE_CACHELISTS(mnode, bin, mtype) == NULL) 14307c478bd9Sstevel@tonic-gate goto nextcachebin; 14317c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_CACHE_LIST); 14327c478bd9Sstevel@tonic-gate mutex_enter(pcm); 14337c478bd9Sstevel@tonic-gate pp = PAGE_CACHELISTS(mnode, bin, mtype); 14347c478bd9Sstevel@tonic-gate first_pp = pp; 14357c478bd9Sstevel@tonic-gate while (pp != NULL) { 14367c478bd9Sstevel@tonic-gate if (page_trylock(pp, SE_EXCL) == 0) { 14377c478bd9Sstevel@tonic-gate pp = pp->p_next; 14387c478bd9Sstevel@tonic-gate if (pp == first_pp) 14397c478bd9Sstevel@tonic-gate break; 14407c478bd9Sstevel@tonic-gate continue; 14417c478bd9Sstevel@tonic-gate } 14427c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode); 14437c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp) == 0); 14447c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 14457c478bd9Sstevel@tonic-gate ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 14467c478bd9Sstevel@tonic-gate 14477c478bd9Sstevel@tonic-gate /* check if page within DMA attributes */ 14487c478bd9Sstevel@tonic-gate 14497c478bd9Sstevel@tonic-gate pgaddr = ptob((uint64_t)(pp->p_pagenum)); 14507c478bd9Sstevel@tonic-gate 14517c478bd9Sstevel@tonic-gate if ((pgaddr >= dma_attr->dma_attr_addr_lo) && 14527c478bd9Sstevel@tonic-gate (pgaddr + MMU_PAGESIZE - 1 <= 14537c478bd9Sstevel@tonic-gate dma_attr->dma_attr_addr_hi)) { 14547c478bd9Sstevel@tonic-gate break; 14557c478bd9Sstevel@tonic-gate } 14567c478bd9Sstevel@tonic-gate 14577c478bd9Sstevel@tonic-gate /* continue looking */ 14587c478bd9Sstevel@tonic-gate page_unlock(pp); 14597c478bd9Sstevel@tonic-gate pp = pp->p_next; 14607c478bd9Sstevel@tonic-gate if (pp == first_pp) 14617c478bd9Sstevel@tonic-gate pp = NULL; 14627c478bd9Sstevel@tonic-gate } 14637c478bd9Sstevel@tonic-gate 14647c478bd9Sstevel@tonic-gate if (pp != NULL) { 14657c478bd9Sstevel@tonic-gate ASSERT(mtype == PP_2_MTYPE(pp)); 14667c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 14677c478bd9Sstevel@tonic-gate 14687c478bd9Sstevel@tonic-gate /* found a page with specified DMA attributes */ 14697c478bd9Sstevel@tonic-gate page_sub(&PAGE_CACHELISTS(mnode, bin, 14707c478bd9Sstevel@tonic-gate mtype), pp); 1471affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST); 14727c478bd9Sstevel@tonic-gate 14737c478bd9Sstevel@tonic-gate mutex_exit(pcm); 14747c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode); 14757c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp) == 0); 14767c478bd9Sstevel@tonic-gate check_dma(dma_attr, pp, 1); 14777c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pgma_allocok); 14787c478bd9Sstevel@tonic-gate return (pp); 14797c478bd9Sstevel@tonic-gate } 14807c478bd9Sstevel@tonic-gate mutex_exit(pcm); 14817c478bd9Sstevel@tonic-gate nextcachebin: 14827c478bd9Sstevel@tonic-gate bin += (i == 0) ? BIN_STEP : 1; 14837c478bd9Sstevel@tonic-gate bin &= page_colors_mask; 14847c478bd9Sstevel@tonic-gate } 1485affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags); 1486affbd3ccSkchow } while (mtype >= 0); 14877c478bd9Sstevel@tonic-gate 14887c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pgma_allocfailed); 14897c478bd9Sstevel@tonic-gate return (NULL); 14907c478bd9Sstevel@tonic-gate } 14917c478bd9Sstevel@tonic-gate 14927c478bd9Sstevel@tonic-gate /* 14937c478bd9Sstevel@tonic-gate * This function is similar to page_get_freelist()/page_get_cachelist() 14947c478bd9Sstevel@tonic-gate * but it searches both the lists to find a page with the specified 14957c478bd9Sstevel@tonic-gate * color (or no color) and DMA attributes. The search is done in the 14967c478bd9Sstevel@tonic-gate * freelist first and then in the cache list within the highest memory 14977c478bd9Sstevel@tonic-gate * range (based on DMA attributes) before searching in the lower 14987c478bd9Sstevel@tonic-gate * memory ranges. 14997c478bd9Sstevel@tonic-gate * 15007c478bd9Sstevel@tonic-gate * Note: This function is called only by page_create_io(). 15017c478bd9Sstevel@tonic-gate */ 15027c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 15037c478bd9Sstevel@tonic-gate page_t * 15047c478bd9Sstevel@tonic-gate page_get_anylist(struct vnode *vp, u_offset_t off, struct as *as, caddr_t vaddr, 15057c478bd9Sstevel@tonic-gate size_t size, uint_t flags, ddi_dma_attr_t *dma_attr, lgrp_t *lgrp) 15067c478bd9Sstevel@tonic-gate { 15077c478bd9Sstevel@tonic-gate uint_t bin; 15087c478bd9Sstevel@tonic-gate int mtype; 15097c478bd9Sstevel@tonic-gate page_t *pp; 15107c478bd9Sstevel@tonic-gate int n; 15117c478bd9Sstevel@tonic-gate int m; 15127c478bd9Sstevel@tonic-gate int szc; 15137c478bd9Sstevel@tonic-gate int fullrange; 15147c478bd9Sstevel@tonic-gate int mnode; 15157c478bd9Sstevel@tonic-gate int local_failed_stat = 0; 15167c478bd9Sstevel@tonic-gate lgrp_mnode_cookie_t lgrp_cookie; 15177c478bd9Sstevel@tonic-gate 15187c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pga_alloc); 15197c478bd9Sstevel@tonic-gate 15207c478bd9Sstevel@tonic-gate /* only base pagesize currently supported */ 15217c478bd9Sstevel@tonic-gate if (size != MMU_PAGESIZE) 15227c478bd9Sstevel@tonic-gate return (NULL); 15237c478bd9Sstevel@tonic-gate 15247c478bd9Sstevel@tonic-gate /* 15257c478bd9Sstevel@tonic-gate * If we're passed a specific lgroup, we use it. Otherwise, 15267c478bd9Sstevel@tonic-gate * assume first-touch placement is desired. 15277c478bd9Sstevel@tonic-gate */ 15287c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp)) 15297c478bd9Sstevel@tonic-gate lgrp = lgrp_home_lgrp(); 15307c478bd9Sstevel@tonic-gate 15317c478bd9Sstevel@tonic-gate /* LINTED */ 1532*5d07b933Sdp AS_2_BIN(as, seg, vp, vaddr, bin, 0); 15337c478bd9Sstevel@tonic-gate 15347c478bd9Sstevel@tonic-gate /* 15357c478bd9Sstevel@tonic-gate * Only hold one freelist or cachelist lock at a time, that way we 15367c478bd9Sstevel@tonic-gate * can start anywhere and not have to worry about lock 15377c478bd9Sstevel@tonic-gate * ordering. 15387c478bd9Sstevel@tonic-gate */ 15397c478bd9Sstevel@tonic-gate if (dma_attr == NULL) { 15407c478bd9Sstevel@tonic-gate n = 0; 15417c478bd9Sstevel@tonic-gate m = mnoderangecnt - 1; 15427c478bd9Sstevel@tonic-gate fullrange = 1; 15437c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pga_nulldmaattr); 15447c478bd9Sstevel@tonic-gate } else { 15457c478bd9Sstevel@tonic-gate pfn_t pfnlo = mmu_btop(dma_attr->dma_attr_addr_lo); 15467c478bd9Sstevel@tonic-gate pfn_t pfnhi = mmu_btop(dma_attr->dma_attr_addr_hi); 15477c478bd9Sstevel@tonic-gate 15487c478bd9Sstevel@tonic-gate /* 15497c478bd9Sstevel@tonic-gate * We can guarantee alignment only for page boundary. 15507c478bd9Sstevel@tonic-gate */ 15517c478bd9Sstevel@tonic-gate if (dma_attr->dma_attr_align > MMU_PAGESIZE) 15527c478bd9Sstevel@tonic-gate return (NULL); 15537c478bd9Sstevel@tonic-gate 15547c478bd9Sstevel@tonic-gate n = pfn_2_mtype(pfnlo); 15557c478bd9Sstevel@tonic-gate m = pfn_2_mtype(pfnhi); 15567c478bd9Sstevel@tonic-gate 15577c478bd9Sstevel@tonic-gate fullrange = ((pfnlo == mnoderanges[n].mnr_pfnlo) && 15587c478bd9Sstevel@tonic-gate (pfnhi >= mnoderanges[m].mnr_pfnhi)); 15597c478bd9Sstevel@tonic-gate } 15607c478bd9Sstevel@tonic-gate VM_STAT_COND_ADD(fullrange == 0, pga_vmstats.pga_notfullrange); 15617c478bd9Sstevel@tonic-gate 15627c478bd9Sstevel@tonic-gate if (n > m) 15637c478bd9Sstevel@tonic-gate return (NULL); 15647c478bd9Sstevel@tonic-gate 15657c478bd9Sstevel@tonic-gate szc = 0; 15667c478bd9Sstevel@tonic-gate 15677c478bd9Sstevel@tonic-gate /* cylcing thru mtype handled by RANGE0 if n == 0 */ 15687c478bd9Sstevel@tonic-gate if (n == 0) { 15697c478bd9Sstevel@tonic-gate flags |= PGI_MT_RANGE0; 15707c478bd9Sstevel@tonic-gate n = m; 15717c478bd9Sstevel@tonic-gate } 15727c478bd9Sstevel@tonic-gate 15737c478bd9Sstevel@tonic-gate /* 15747c478bd9Sstevel@tonic-gate * Try local memory node first, but try remote if we can't 15757c478bd9Sstevel@tonic-gate * get a page of the right color. 15767c478bd9Sstevel@tonic-gate */ 15777c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_HIER); 15787c478bd9Sstevel@tonic-gate while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 15797c478bd9Sstevel@tonic-gate /* 15807c478bd9Sstevel@tonic-gate * allocate pages from high pfn to low. 15817c478bd9Sstevel@tonic-gate */ 15827c478bd9Sstevel@tonic-gate for (mtype = m; mtype >= n; mtype--) { 15837c478bd9Sstevel@tonic-gate if (fullrange != 0) { 15847c478bd9Sstevel@tonic-gate pp = page_get_mnode_freelist(mnode, 15857c478bd9Sstevel@tonic-gate bin, mtype, szc, flags); 15867c478bd9Sstevel@tonic-gate if (pp == NULL) { 15877c478bd9Sstevel@tonic-gate pp = page_get_mnode_cachelist( 15887c478bd9Sstevel@tonic-gate bin, flags, mnode, mtype); 15897c478bd9Sstevel@tonic-gate } 15907c478bd9Sstevel@tonic-gate } else { 15917c478bd9Sstevel@tonic-gate pp = page_get_mnode_anylist(bin, szc, 15927c478bd9Sstevel@tonic-gate flags, mnode, mtype, dma_attr); 15937c478bd9Sstevel@tonic-gate } 15947c478bd9Sstevel@tonic-gate if (pp != NULL) { 15957c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pga_allocok); 15967c478bd9Sstevel@tonic-gate check_dma(dma_attr, pp, 1); 15977c478bd9Sstevel@tonic-gate return (pp); 15987c478bd9Sstevel@tonic-gate } 15997c478bd9Sstevel@tonic-gate } 16007c478bd9Sstevel@tonic-gate if (!local_failed_stat) { 16017c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1); 16027c478bd9Sstevel@tonic-gate local_failed_stat = 1; 16037c478bd9Sstevel@tonic-gate } 16047c478bd9Sstevel@tonic-gate } 16057c478bd9Sstevel@tonic-gate VM_STAT_ADD(pga_vmstats.pga_allocfailed); 16067c478bd9Sstevel@tonic-gate 16077c478bd9Sstevel@tonic-gate return (NULL); 16087c478bd9Sstevel@tonic-gate } 16097c478bd9Sstevel@tonic-gate 16107c478bd9Sstevel@tonic-gate /* 16117c478bd9Sstevel@tonic-gate * page_create_io() 16127c478bd9Sstevel@tonic-gate * 16137c478bd9Sstevel@tonic-gate * This function is a copy of page_create_va() with an additional 16147c478bd9Sstevel@tonic-gate * argument 'mattr' that specifies DMA memory requirements to 16157c478bd9Sstevel@tonic-gate * the page list functions. This function is used by the segkmem 16167c478bd9Sstevel@tonic-gate * allocator so it is only to create new pages (i.e PG_EXCL is 16177c478bd9Sstevel@tonic-gate * set). 16187c478bd9Sstevel@tonic-gate * 16197c478bd9Sstevel@tonic-gate * Note: This interface is currently used by x86 PSM only and is 16207c478bd9Sstevel@tonic-gate * not fully specified so the commitment level is only for 16217c478bd9Sstevel@tonic-gate * private interface specific to x86. This interface uses PSM 16227c478bd9Sstevel@tonic-gate * specific page_get_anylist() interface. 16237c478bd9Sstevel@tonic-gate */ 16247c478bd9Sstevel@tonic-gate 16257c478bd9Sstevel@tonic-gate #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 16267c478bd9Sstevel@tonic-gate for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \ 16277c478bd9Sstevel@tonic-gate if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 16287c478bd9Sstevel@tonic-gate break; \ 16297c478bd9Sstevel@tonic-gate } \ 16307c478bd9Sstevel@tonic-gate } 16317c478bd9Sstevel@tonic-gate 16327c478bd9Sstevel@tonic-gate 16337c478bd9Sstevel@tonic-gate page_t * 16347c478bd9Sstevel@tonic-gate page_create_io( 16357c478bd9Sstevel@tonic-gate struct vnode *vp, 16367c478bd9Sstevel@tonic-gate u_offset_t off, 16377c478bd9Sstevel@tonic-gate uint_t bytes, 16387c478bd9Sstevel@tonic-gate uint_t flags, 16397c478bd9Sstevel@tonic-gate struct as *as, 16407c478bd9Sstevel@tonic-gate caddr_t vaddr, 16417c478bd9Sstevel@tonic-gate ddi_dma_attr_t *mattr) /* DMA memory attributes if any */ 16427c478bd9Sstevel@tonic-gate { 16437c478bd9Sstevel@tonic-gate page_t *plist = NULL; 16447c478bd9Sstevel@tonic-gate uint_t plist_len = 0; 16457c478bd9Sstevel@tonic-gate pgcnt_t npages; 16467c478bd9Sstevel@tonic-gate page_t *npp = NULL; 16477c478bd9Sstevel@tonic-gate uint_t pages_req; 16487c478bd9Sstevel@tonic-gate page_t *pp; 16497c478bd9Sstevel@tonic-gate kmutex_t *phm = NULL; 16507c478bd9Sstevel@tonic-gate uint_t index; 16517c478bd9Sstevel@tonic-gate 16527c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START, 16537c478bd9Sstevel@tonic-gate "page_create_start:vp %p off %llx bytes %u flags %x", 16547c478bd9Sstevel@tonic-gate vp, off, bytes, flags); 16557c478bd9Sstevel@tonic-gate 16567c478bd9Sstevel@tonic-gate ASSERT((flags & ~(PG_EXCL | PG_WAIT | PG_PHYSCONTIG)) == 0); 16577c478bd9Sstevel@tonic-gate 16587c478bd9Sstevel@tonic-gate pages_req = npages = mmu_btopr(bytes); 16597c478bd9Sstevel@tonic-gate 16607c478bd9Sstevel@tonic-gate /* 16617c478bd9Sstevel@tonic-gate * Do the freemem and pcf accounting. 16627c478bd9Sstevel@tonic-gate */ 16637c478bd9Sstevel@tonic-gate if (!page_create_wait(npages, flags)) { 16647c478bd9Sstevel@tonic-gate return (NULL); 16657c478bd9Sstevel@tonic-gate } 16667c478bd9Sstevel@tonic-gate 16677c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS, 16687c478bd9Sstevel@tonic-gate "page_create_success:vp %p off %llx", 16697c478bd9Sstevel@tonic-gate vp, off); 16707c478bd9Sstevel@tonic-gate 16717c478bd9Sstevel@tonic-gate /* 16727c478bd9Sstevel@tonic-gate * If satisfying this request has left us with too little 16737c478bd9Sstevel@tonic-gate * memory, start the wheels turning to get some back. The 16747c478bd9Sstevel@tonic-gate * first clause of the test prevents waking up the pageout 16757c478bd9Sstevel@tonic-gate * daemon in situations where it would decide that there's 16767c478bd9Sstevel@tonic-gate * nothing to do. 16777c478bd9Sstevel@tonic-gate */ 16787c478bd9Sstevel@tonic-gate if (nscan < desscan && freemem < minfree) { 16797c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 16807c478bd9Sstevel@tonic-gate "pageout_cv_signal:freemem %ld", freemem); 16817c478bd9Sstevel@tonic-gate cv_signal(&proc_pageout->p_cv); 16827c478bd9Sstevel@tonic-gate } 16837c478bd9Sstevel@tonic-gate 16847c478bd9Sstevel@tonic-gate if (flags & PG_PHYSCONTIG) { 16857c478bd9Sstevel@tonic-gate 16867c478bd9Sstevel@tonic-gate plist = page_get_contigpage(&npages, mattr, 1); 16877c478bd9Sstevel@tonic-gate if (plist == NULL) { 16887c478bd9Sstevel@tonic-gate page_create_putback(npages); 16897c478bd9Sstevel@tonic-gate return (NULL); 16907c478bd9Sstevel@tonic-gate } 16917c478bd9Sstevel@tonic-gate 16927c478bd9Sstevel@tonic-gate pp = plist; 16937c478bd9Sstevel@tonic-gate 16947c478bd9Sstevel@tonic-gate do { 16957c478bd9Sstevel@tonic-gate if (!page_hashin(pp, vp, off, NULL)) { 16967c478bd9Sstevel@tonic-gate panic("pg_creat_io: hashin failed %p %p %llx", 16977c478bd9Sstevel@tonic-gate (void *)pp, (void *)vp, off); 16987c478bd9Sstevel@tonic-gate } 16997c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_new); 17007c478bd9Sstevel@tonic-gate off += MMU_PAGESIZE; 17017c478bd9Sstevel@tonic-gate PP_CLRFREE(pp); 17027c478bd9Sstevel@tonic-gate PP_CLRAGED(pp); 17037c478bd9Sstevel@tonic-gate page_set_props(pp, P_REF); 17047c478bd9Sstevel@tonic-gate pp = pp->p_next; 17057c478bd9Sstevel@tonic-gate } while (pp != plist); 17067c478bd9Sstevel@tonic-gate 17077c478bd9Sstevel@tonic-gate if (!npages) { 17087c478bd9Sstevel@tonic-gate check_dma(mattr, plist, pages_req); 17097c478bd9Sstevel@tonic-gate return (plist); 17107c478bd9Sstevel@tonic-gate } else { 17117c478bd9Sstevel@tonic-gate vaddr += (pages_req - npages) << MMU_PAGESHIFT; 17127c478bd9Sstevel@tonic-gate } 17137c478bd9Sstevel@tonic-gate 17147c478bd9Sstevel@tonic-gate /* 17157c478bd9Sstevel@tonic-gate * fall-thru: 17167c478bd9Sstevel@tonic-gate * 17177c478bd9Sstevel@tonic-gate * page_get_contigpage returns when npages <= sgllen. 17187c478bd9Sstevel@tonic-gate * Grab the rest of the non-contig pages below from anylist. 17197c478bd9Sstevel@tonic-gate */ 17207c478bd9Sstevel@tonic-gate } 17217c478bd9Sstevel@tonic-gate 17227c478bd9Sstevel@tonic-gate /* 17237c478bd9Sstevel@tonic-gate * Loop around collecting the requested number of pages. 17247c478bd9Sstevel@tonic-gate * Most of the time, we have to `create' a new page. With 17257c478bd9Sstevel@tonic-gate * this in mind, pull the page off the free list before 17267c478bd9Sstevel@tonic-gate * getting the hash lock. This will minimize the hash 17277c478bd9Sstevel@tonic-gate * lock hold time, nesting, and the like. If it turns 17287c478bd9Sstevel@tonic-gate * out we don't need the page, we put it back at the end. 17297c478bd9Sstevel@tonic-gate */ 17307c478bd9Sstevel@tonic-gate while (npages--) { 17317c478bd9Sstevel@tonic-gate phm = NULL; 17327c478bd9Sstevel@tonic-gate 17337c478bd9Sstevel@tonic-gate index = PAGE_HASH_FUNC(vp, off); 17347c478bd9Sstevel@tonic-gate top: 17357c478bd9Sstevel@tonic-gate ASSERT(phm == NULL); 17367c478bd9Sstevel@tonic-gate ASSERT(index == PAGE_HASH_FUNC(vp, off)); 17377c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 17387c478bd9Sstevel@tonic-gate 17397c478bd9Sstevel@tonic-gate if (npp == NULL) { 17407c478bd9Sstevel@tonic-gate /* 17417c478bd9Sstevel@tonic-gate * Try to get the page of any color either from 17427c478bd9Sstevel@tonic-gate * the freelist or from the cache list. 17437c478bd9Sstevel@tonic-gate */ 17447c478bd9Sstevel@tonic-gate npp = page_get_anylist(vp, off, as, vaddr, MMU_PAGESIZE, 17457c478bd9Sstevel@tonic-gate flags & ~PG_MATCH_COLOR, mattr, NULL); 17467c478bd9Sstevel@tonic-gate if (npp == NULL) { 17477c478bd9Sstevel@tonic-gate if (mattr == NULL) { 17487c478bd9Sstevel@tonic-gate /* 17497c478bd9Sstevel@tonic-gate * Not looking for a special page; 17507c478bd9Sstevel@tonic-gate * panic! 17517c478bd9Sstevel@tonic-gate */ 17527c478bd9Sstevel@tonic-gate panic("no page found %d", (int)npages); 17537c478bd9Sstevel@tonic-gate } 17547c478bd9Sstevel@tonic-gate /* 17557c478bd9Sstevel@tonic-gate * No page found! This can happen 17567c478bd9Sstevel@tonic-gate * if we are looking for a page 17577c478bd9Sstevel@tonic-gate * within a specific memory range 17587c478bd9Sstevel@tonic-gate * for DMA purposes. If PG_WAIT is 17597c478bd9Sstevel@tonic-gate * specified then we wait for a 17607c478bd9Sstevel@tonic-gate * while and then try again. The 17617c478bd9Sstevel@tonic-gate * wait could be forever if we 17627c478bd9Sstevel@tonic-gate * don't get the page(s) we need. 17637c478bd9Sstevel@tonic-gate * 17647c478bd9Sstevel@tonic-gate * Note: XXX We really need a mechanism 17657c478bd9Sstevel@tonic-gate * to wait for pages in the desired 17667c478bd9Sstevel@tonic-gate * range. For now, we wait for any 17677c478bd9Sstevel@tonic-gate * pages and see if we can use it. 17687c478bd9Sstevel@tonic-gate */ 17697c478bd9Sstevel@tonic-gate 17707c478bd9Sstevel@tonic-gate if ((mattr != NULL) && (flags & PG_WAIT)) { 17717c478bd9Sstevel@tonic-gate delay(10); 17727c478bd9Sstevel@tonic-gate goto top; 17737c478bd9Sstevel@tonic-gate } 17747c478bd9Sstevel@tonic-gate 17757c478bd9Sstevel@tonic-gate goto fail; /* undo accounting stuff */ 17767c478bd9Sstevel@tonic-gate } 17777c478bd9Sstevel@tonic-gate 17787c478bd9Sstevel@tonic-gate if (PP_ISAGED(npp) == 0) { 17797c478bd9Sstevel@tonic-gate /* 17807c478bd9Sstevel@tonic-gate * Since this page came from the 17817c478bd9Sstevel@tonic-gate * cachelist, we must destroy the 17827c478bd9Sstevel@tonic-gate * old vnode association. 17837c478bd9Sstevel@tonic-gate */ 17847c478bd9Sstevel@tonic-gate page_hashout(npp, (kmutex_t *)NULL); 17857c478bd9Sstevel@tonic-gate } 17867c478bd9Sstevel@tonic-gate } 17877c478bd9Sstevel@tonic-gate 17887c478bd9Sstevel@tonic-gate /* 17897c478bd9Sstevel@tonic-gate * We own this page! 17907c478bd9Sstevel@tonic-gate */ 17917c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(npp)); 17927c478bd9Sstevel@tonic-gate ASSERT(npp->p_vnode == NULL); 17937c478bd9Sstevel@tonic-gate ASSERT(!hat_page_is_mapped(npp)); 17947c478bd9Sstevel@tonic-gate PP_CLRFREE(npp); 17957c478bd9Sstevel@tonic-gate PP_CLRAGED(npp); 17967c478bd9Sstevel@tonic-gate 17977c478bd9Sstevel@tonic-gate /* 17987c478bd9Sstevel@tonic-gate * Here we have a page in our hot little mits and are 17997c478bd9Sstevel@tonic-gate * just waiting to stuff it on the appropriate lists. 18007c478bd9Sstevel@tonic-gate * Get the mutex and check to see if it really does 18017c478bd9Sstevel@tonic-gate * not exist. 18027c478bd9Sstevel@tonic-gate */ 18037c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(index); 18047c478bd9Sstevel@tonic-gate mutex_enter(phm); 18057c478bd9Sstevel@tonic-gate PAGE_HASH_SEARCH(index, pp, vp, off); 18067c478bd9Sstevel@tonic-gate if (pp == NULL) { 18077c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_new); 18087c478bd9Sstevel@tonic-gate pp = npp; 18097c478bd9Sstevel@tonic-gate npp = NULL; 18107c478bd9Sstevel@tonic-gate if (!page_hashin(pp, vp, off, phm)) { 18117c478bd9Sstevel@tonic-gate /* 18127c478bd9Sstevel@tonic-gate * Since we hold the page hash mutex and 18137c478bd9Sstevel@tonic-gate * just searched for this page, page_hashin 18147c478bd9Sstevel@tonic-gate * had better not fail. If it does, that 18157c478bd9Sstevel@tonic-gate * means somethread did not follow the 18167c478bd9Sstevel@tonic-gate * page hash mutex rules. Panic now and 18177c478bd9Sstevel@tonic-gate * get it over with. As usual, go down 18187c478bd9Sstevel@tonic-gate * holding all the locks. 18197c478bd9Sstevel@tonic-gate */ 18207c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(phm)); 18217c478bd9Sstevel@tonic-gate panic("page_create: hashin fail %p %p %llx %p", 18227c478bd9Sstevel@tonic-gate (void *)pp, (void *)vp, off, (void *)phm); 18237c478bd9Sstevel@tonic-gate 18247c478bd9Sstevel@tonic-gate } 18257c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(phm)); 18267c478bd9Sstevel@tonic-gate mutex_exit(phm); 18277c478bd9Sstevel@tonic-gate phm = NULL; 18287c478bd9Sstevel@tonic-gate 18297c478bd9Sstevel@tonic-gate /* 18307c478bd9Sstevel@tonic-gate * Hat layer locking need not be done to set 18317c478bd9Sstevel@tonic-gate * the following bits since the page is not hashed 18327c478bd9Sstevel@tonic-gate * and was on the free list (i.e., had no mappings). 18337c478bd9Sstevel@tonic-gate * 18347c478bd9Sstevel@tonic-gate * Set the reference bit to protect 18357c478bd9Sstevel@tonic-gate * against immediate pageout 18367c478bd9Sstevel@tonic-gate * 18377c478bd9Sstevel@tonic-gate * XXXmh modify freelist code to set reference 18387c478bd9Sstevel@tonic-gate * bit so we don't have to do it here. 18397c478bd9Sstevel@tonic-gate */ 18407c478bd9Sstevel@tonic-gate page_set_props(pp, P_REF); 18417c478bd9Sstevel@tonic-gate } else { 18427c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(phm)); 18437c478bd9Sstevel@tonic-gate mutex_exit(phm); 18447c478bd9Sstevel@tonic-gate phm = NULL; 18457c478bd9Sstevel@tonic-gate /* 18467c478bd9Sstevel@tonic-gate * NOTE: This should not happen for pages associated 18477c478bd9Sstevel@tonic-gate * with kernel vnode 'kvp'. 18487c478bd9Sstevel@tonic-gate */ 18497c478bd9Sstevel@tonic-gate /* XX64 - to debug why this happens! */ 18507c478bd9Sstevel@tonic-gate ASSERT(vp != &kvp); 18517c478bd9Sstevel@tonic-gate if (vp == &kvp) 18527c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, 18537c478bd9Sstevel@tonic-gate "page_create: page not expected " 18547c478bd9Sstevel@tonic-gate "in hash list for kernel vnode - pp 0x%p", 18557c478bd9Sstevel@tonic-gate (void *)pp); 18567c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_exists); 18577c478bd9Sstevel@tonic-gate goto fail; 18587c478bd9Sstevel@tonic-gate } 18597c478bd9Sstevel@tonic-gate 18607c478bd9Sstevel@tonic-gate /* 18617c478bd9Sstevel@tonic-gate * Got a page! It is locked. Acquire the i/o 18627c478bd9Sstevel@tonic-gate * lock since we are going to use the p_next and 18637c478bd9Sstevel@tonic-gate * p_prev fields to link the requested pages together. 18647c478bd9Sstevel@tonic-gate */ 18657c478bd9Sstevel@tonic-gate page_io_lock(pp); 18667c478bd9Sstevel@tonic-gate page_add(&plist, pp); 18677c478bd9Sstevel@tonic-gate plist = plist->p_next; 18687c478bd9Sstevel@tonic-gate off += MMU_PAGESIZE; 18697c478bd9Sstevel@tonic-gate vaddr += MMU_PAGESIZE; 18707c478bd9Sstevel@tonic-gate } 18717c478bd9Sstevel@tonic-gate 18727c478bd9Sstevel@tonic-gate check_dma(mattr, plist, pages_req); 18737c478bd9Sstevel@tonic-gate return (plist); 18747c478bd9Sstevel@tonic-gate 18757c478bd9Sstevel@tonic-gate fail: 18767c478bd9Sstevel@tonic-gate if (npp != NULL) { 18777c478bd9Sstevel@tonic-gate /* 18787c478bd9Sstevel@tonic-gate * Did not need this page after all. 18797c478bd9Sstevel@tonic-gate * Put it back on the free list. 18807c478bd9Sstevel@tonic-gate */ 18817c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_putbacks); 18827c478bd9Sstevel@tonic-gate PP_SETFREE(npp); 18837c478bd9Sstevel@tonic-gate PP_SETAGED(npp); 18847c478bd9Sstevel@tonic-gate npp->p_offset = (u_offset_t)-1; 18857c478bd9Sstevel@tonic-gate page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL); 18867c478bd9Sstevel@tonic-gate page_unlock(npp); 18877c478bd9Sstevel@tonic-gate } 18887c478bd9Sstevel@tonic-gate 18897c478bd9Sstevel@tonic-gate /* 18907c478bd9Sstevel@tonic-gate * Give up the pages we already got. 18917c478bd9Sstevel@tonic-gate */ 18927c478bd9Sstevel@tonic-gate while (plist != NULL) { 18937c478bd9Sstevel@tonic-gate pp = plist; 18947c478bd9Sstevel@tonic-gate page_sub(&plist, pp); 18957c478bd9Sstevel@tonic-gate page_io_unlock(pp); 18967c478bd9Sstevel@tonic-gate plist_len++; 18977c478bd9Sstevel@tonic-gate /*LINTED: constant in conditional ctx*/ 18987c478bd9Sstevel@tonic-gate VN_DISPOSE(pp, B_INVAL, 0, kcred); 18997c478bd9Sstevel@tonic-gate } 19007c478bd9Sstevel@tonic-gate 19017c478bd9Sstevel@tonic-gate /* 19027c478bd9Sstevel@tonic-gate * VN_DISPOSE does freemem accounting for the pages in plist 19037c478bd9Sstevel@tonic-gate * by calling page_free. So, we need to undo the pcf accounting 19047c478bd9Sstevel@tonic-gate * for only the remaining pages. 19057c478bd9Sstevel@tonic-gate */ 19067c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_putbacks); 19077c478bd9Sstevel@tonic-gate page_create_putback(pages_req - plist_len); 19087c478bd9Sstevel@tonic-gate 19097c478bd9Sstevel@tonic-gate return (NULL); 19107c478bd9Sstevel@tonic-gate } 19117c478bd9Sstevel@tonic-gate 19127c478bd9Sstevel@tonic-gate 19137c478bd9Sstevel@tonic-gate /* 19147c478bd9Sstevel@tonic-gate * Copy the data from the physical page represented by "frompp" to 19157c478bd9Sstevel@tonic-gate * that represented by "topp". ppcopy uses CPU->cpu_caddr1 and 19167c478bd9Sstevel@tonic-gate * CPU->cpu_caddr2. It assumes that no one uses either map at interrupt 19177c478bd9Sstevel@tonic-gate * level and no one sleeps with an active mapping there. 19187c478bd9Sstevel@tonic-gate * 19197c478bd9Sstevel@tonic-gate * Note that the ref/mod bits in the page_t's are not affected by 19207c478bd9Sstevel@tonic-gate * this operation, hence it is up to the caller to update them appropriately. 19217c478bd9Sstevel@tonic-gate */ 19227c478bd9Sstevel@tonic-gate void 19237c478bd9Sstevel@tonic-gate ppcopy(page_t *frompp, page_t *topp) 19247c478bd9Sstevel@tonic-gate { 19257c478bd9Sstevel@tonic-gate caddr_t pp_addr1; 19267c478bd9Sstevel@tonic-gate caddr_t pp_addr2; 19277c478bd9Sstevel@tonic-gate void *pte1; 19287c478bd9Sstevel@tonic-gate void *pte2; 19297c478bd9Sstevel@tonic-gate kmutex_t *ppaddr_mutex; 19307c478bd9Sstevel@tonic-gate 19317c478bd9Sstevel@tonic-gate ASSERT_STACK_ALIGNED(); 19327c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(frompp)); 19337c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(topp)); 19347c478bd9Sstevel@tonic-gate 19357c478bd9Sstevel@tonic-gate if (kpm_enable) { 19367c478bd9Sstevel@tonic-gate pp_addr1 = hat_kpm_page2va(frompp, 0); 19377c478bd9Sstevel@tonic-gate pp_addr2 = hat_kpm_page2va(topp, 0); 19387c478bd9Sstevel@tonic-gate kpreempt_disable(); 19397c478bd9Sstevel@tonic-gate } else { 19407c478bd9Sstevel@tonic-gate /* 19417c478bd9Sstevel@tonic-gate * disable pre-emption so that CPU can't change 19427c478bd9Sstevel@tonic-gate */ 19437c478bd9Sstevel@tonic-gate kpreempt_disable(); 19447c478bd9Sstevel@tonic-gate 19457c478bd9Sstevel@tonic-gate pp_addr1 = CPU->cpu_caddr1; 19467c478bd9Sstevel@tonic-gate pp_addr2 = CPU->cpu_caddr2; 19477c478bd9Sstevel@tonic-gate pte1 = (void *)CPU->cpu_caddr1pte; 19487c478bd9Sstevel@tonic-gate pte2 = (void *)CPU->cpu_caddr2pte; 19497c478bd9Sstevel@tonic-gate 19507c478bd9Sstevel@tonic-gate ppaddr_mutex = &CPU->cpu_ppaddr_mutex; 19517c478bd9Sstevel@tonic-gate mutex_enter(ppaddr_mutex); 19527c478bd9Sstevel@tonic-gate 19537c478bd9Sstevel@tonic-gate hat_mempte_remap(page_pptonum(frompp), pp_addr1, pte1, 19547c478bd9Sstevel@tonic-gate PROT_READ | HAT_STORECACHING_OK, HAT_LOAD_NOCONSIST); 19557c478bd9Sstevel@tonic-gate hat_mempte_remap(page_pptonum(topp), pp_addr2, pte2, 19567c478bd9Sstevel@tonic-gate PROT_READ | PROT_WRITE | HAT_STORECACHING_OK, 19577c478bd9Sstevel@tonic-gate HAT_LOAD_NOCONSIST); 19587c478bd9Sstevel@tonic-gate } 19597c478bd9Sstevel@tonic-gate 19607c478bd9Sstevel@tonic-gate if (use_sse_pagecopy) 19617c478bd9Sstevel@tonic-gate hwblkpagecopy(pp_addr1, pp_addr2); 19627c478bd9Sstevel@tonic-gate else 19637c478bd9Sstevel@tonic-gate bcopy(pp_addr1, pp_addr2, PAGESIZE); 19647c478bd9Sstevel@tonic-gate 19657c478bd9Sstevel@tonic-gate if (!kpm_enable) 19667c478bd9Sstevel@tonic-gate mutex_exit(ppaddr_mutex); 19677c478bd9Sstevel@tonic-gate kpreempt_enable(); 19687c478bd9Sstevel@tonic-gate } 19697c478bd9Sstevel@tonic-gate 19707c478bd9Sstevel@tonic-gate /* 19717c478bd9Sstevel@tonic-gate * Zero the physical page from off to off + len given by `pp' 19727c478bd9Sstevel@tonic-gate * without changing the reference and modified bits of page. 19737c478bd9Sstevel@tonic-gate * 19747c478bd9Sstevel@tonic-gate * We use this using CPU private page address #2, see ppcopy() for more info. 19757c478bd9Sstevel@tonic-gate * pagezero() must not be called at interrupt level. 19767c478bd9Sstevel@tonic-gate */ 19777c478bd9Sstevel@tonic-gate void 19787c478bd9Sstevel@tonic-gate pagezero(page_t *pp, uint_t off, uint_t len) 19797c478bd9Sstevel@tonic-gate { 19807c478bd9Sstevel@tonic-gate caddr_t pp_addr2; 19817c478bd9Sstevel@tonic-gate void *pte2; 19827c478bd9Sstevel@tonic-gate kmutex_t *ppaddr_mutex; 19837c478bd9Sstevel@tonic-gate 19847c478bd9Sstevel@tonic-gate ASSERT_STACK_ALIGNED(); 19857c478bd9Sstevel@tonic-gate ASSERT(len <= MMU_PAGESIZE); 19867c478bd9Sstevel@tonic-gate ASSERT(off <= MMU_PAGESIZE); 19877c478bd9Sstevel@tonic-gate ASSERT(off + len <= MMU_PAGESIZE); 19887c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(pp)); 19897c478bd9Sstevel@tonic-gate 19907c478bd9Sstevel@tonic-gate if (kpm_enable) { 19917c478bd9Sstevel@tonic-gate pp_addr2 = hat_kpm_page2va(pp, 0); 19927c478bd9Sstevel@tonic-gate kpreempt_disable(); 19937c478bd9Sstevel@tonic-gate } else { 19947c478bd9Sstevel@tonic-gate kpreempt_disable(); 19957c478bd9Sstevel@tonic-gate 19967c478bd9Sstevel@tonic-gate pp_addr2 = CPU->cpu_caddr2; 19977c478bd9Sstevel@tonic-gate pte2 = (void *)CPU->cpu_caddr2pte; 19987c478bd9Sstevel@tonic-gate 19997c478bd9Sstevel@tonic-gate ppaddr_mutex = &CPU->cpu_ppaddr_mutex; 20007c478bd9Sstevel@tonic-gate mutex_enter(ppaddr_mutex); 20017c478bd9Sstevel@tonic-gate 20027c478bd9Sstevel@tonic-gate hat_mempte_remap(page_pptonum(pp), pp_addr2, pte2, 20037c478bd9Sstevel@tonic-gate PROT_READ | PROT_WRITE | HAT_STORECACHING_OK, 20047c478bd9Sstevel@tonic-gate HAT_LOAD_NOCONSIST); 20057c478bd9Sstevel@tonic-gate } 20067c478bd9Sstevel@tonic-gate 20077c478bd9Sstevel@tonic-gate if (use_sse_pagezero) 20087c478bd9Sstevel@tonic-gate hwblkclr(pp_addr2 + off, len); 20097c478bd9Sstevel@tonic-gate else 20107c478bd9Sstevel@tonic-gate bzero(pp_addr2 + off, len); 20117c478bd9Sstevel@tonic-gate 20127c478bd9Sstevel@tonic-gate if (!kpm_enable) 20137c478bd9Sstevel@tonic-gate mutex_exit(ppaddr_mutex); 20147c478bd9Sstevel@tonic-gate kpreempt_enable(); 20157c478bd9Sstevel@tonic-gate } 20167c478bd9Sstevel@tonic-gate 20177c478bd9Sstevel@tonic-gate /* 20187c478bd9Sstevel@tonic-gate * Platform-dependent page scrub call. 20197c478bd9Sstevel@tonic-gate */ 20207c478bd9Sstevel@tonic-gate void 20217c478bd9Sstevel@tonic-gate pagescrub(page_t *pp, uint_t off, uint_t len) 20227c478bd9Sstevel@tonic-gate { 20237c478bd9Sstevel@tonic-gate /* 20247c478bd9Sstevel@tonic-gate * For now, we rely on the fact that pagezero() will 20257c478bd9Sstevel@tonic-gate * always clear UEs. 20267c478bd9Sstevel@tonic-gate */ 20277c478bd9Sstevel@tonic-gate pagezero(pp, off, len); 20287c478bd9Sstevel@tonic-gate } 20297c478bd9Sstevel@tonic-gate 20307c478bd9Sstevel@tonic-gate /* 20317c478bd9Sstevel@tonic-gate * set up two private addresses for use on a given CPU for use in ppcopy() 20327c478bd9Sstevel@tonic-gate */ 20337c478bd9Sstevel@tonic-gate void 20347c478bd9Sstevel@tonic-gate setup_vaddr_for_ppcopy(struct cpu *cpup) 20357c478bd9Sstevel@tonic-gate { 20367c478bd9Sstevel@tonic-gate void *addr; 20377c478bd9Sstevel@tonic-gate void *pte; 20387c478bd9Sstevel@tonic-gate 20397c478bd9Sstevel@tonic-gate addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP); 20407c478bd9Sstevel@tonic-gate pte = hat_mempte_setup(addr); 20417c478bd9Sstevel@tonic-gate cpup->cpu_caddr1 = addr; 20427c478bd9Sstevel@tonic-gate cpup->cpu_caddr1pte = (pteptr_t)pte; 20437c478bd9Sstevel@tonic-gate 20447c478bd9Sstevel@tonic-gate addr = vmem_alloc(heap_arena, mmu_ptob(1), VM_SLEEP); 20457c478bd9Sstevel@tonic-gate pte = hat_mempte_setup(addr); 20467c478bd9Sstevel@tonic-gate cpup->cpu_caddr2 = addr; 20477c478bd9Sstevel@tonic-gate cpup->cpu_caddr2pte = (pteptr_t)pte; 20487c478bd9Sstevel@tonic-gate 20497c478bd9Sstevel@tonic-gate mutex_init(&cpup->cpu_ppaddr_mutex, NULL, MUTEX_DEFAULT, NULL); 20507c478bd9Sstevel@tonic-gate } 20517c478bd9Sstevel@tonic-gate 20527c478bd9Sstevel@tonic-gate 20537c478bd9Sstevel@tonic-gate /* 20547c478bd9Sstevel@tonic-gate * Create the pageout scanner thread. The thread has to 20557c478bd9Sstevel@tonic-gate * start at procedure with process pp and priority pri. 20567c478bd9Sstevel@tonic-gate */ 20577c478bd9Sstevel@tonic-gate void 20587c478bd9Sstevel@tonic-gate pageout_init(void (*procedure)(), proc_t *pp, pri_t pri) 20597c478bd9Sstevel@tonic-gate { 20607c478bd9Sstevel@tonic-gate (void) thread_create(NULL, 0, procedure, NULL, 0, pp, TS_RUN, pri); 20617c478bd9Sstevel@tonic-gate } 20627c478bd9Sstevel@tonic-gate 20637c478bd9Sstevel@tonic-gate /* 20647c478bd9Sstevel@tonic-gate * Function for flushing D-cache when performing module relocations 20657c478bd9Sstevel@tonic-gate * to an alternate mapping. Unnecessary on Intel / AMD platforms. 20667c478bd9Sstevel@tonic-gate */ 20677c478bd9Sstevel@tonic-gate void 20687c478bd9Sstevel@tonic-gate dcache_flushall() 20697c478bd9Sstevel@tonic-gate {} 2070